Merge branch 'fix_missing_node_py_changes' of https://github.com/AntelopeIO/leap into fix_missing_node_py_changes

fix noisy `ignoring packed attribute` warning on `IR::ControlStructureImm`

CMakeLists.txt

@@ -14,9 +14,9 @@ set( CMAKE_CXX_EXTENSIONS ON )
 set( CXX_STANDARD_REQUIRED ON)
 
 set(VERSION_MAJOR 4)
-set(VERSION_MINOR 0)
+set(VERSION_MINOR 1)
 set(VERSION_PATCH 0)
-set(VERSION_SUFFIX rc2)
+set(VERSION_SUFFIX dev)
 
 if(VERSION_SUFFIX)
     set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}")

libraries/chain/CMakeLists.txt

@@ -100,7 +100,6 @@ add_library( eosio_chain
 
               wast_to_wasm.cpp
               wasm_interface.cpp
-              wasm_module_cache.cpp
               wasm_eosio_validation.cpp
               wasm_eosio_injection.cpp
               wasm_config.cpp

libraries/chain/block_log.cpp

@@ -1533,7 +1533,7 @@ namespace eosio { namespace chain {
       ilog("blocks.log and blocks.index agree on number of blocks");
 
       if (interval == 0) {
-         interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1);
+         interval = std::max((log_bundle.log_index.num_blocks() + 7u) >> 3, 1u);
       }
       uint32_t expected_block_num = log_bundle.log_data.first_block_num();
 

libraries/chain/controller.cpp

@@ -246,10 +246,11 @@ struct controller_impl {
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
    thread_local static vm::wasm_allocator wasm_alloc; // a copy for main thread and each read-only thread
 #endif
+   // Ideally wasmif should be thread_local which must be a static.
+   // Unittests can create multiple controller objects (testers) at the same time,
+   // which overwrites the same static wasmif, is used for eosvmoc too.
    wasm_interface  wasmif;  // used by main thread and all threads for EOSVMOC
-   std::mutex threaded_wasmifs_mtx;
-   std::unordered_map<std::thread::id, std::unique_ptr<wasm_interface>> threaded_wasmifs; // one for each read-only thread, used by eos-vm and eos-vm-jit
-   wasm_module_cache wasm_mod_cache;
+   thread_local static std::unique_ptr<wasm_interface> wasmif_thread_local; // a copy for each read-only thread, used by eos-vm and eos-vm-jit
 
    typedef pair<scope_name,action_name>                   handler_key;
    map< account_name, map<handler_key, apply_handler> >   apply_handlers;
@@ -336,13 +337,7 @@ struct controller_impl {
       set_activation_handler<builtin_protocol_feature_t::crypto_primitives>();
 
       self.irreversible_block.connect([this](const block_state_ptr& bsp) {
-         // producer_plugin has already asserted irreversible_block signal is called in
-         // write window
-         wasmif.current_lib(bsp->block_num);
-         for (auto& ele: threaded_wasmifs) {
-            ele.second->current_lib(bsp->block_num);
-         }
-         wasm_mod_cache.current_lib(bsp->block_num);
+         get_wasm_interface().current_lib(bsp->block_num);
       });
 
 
@@ -502,10 +497,9 @@ struct controller_impl {
 
    void replay(std::function<bool()> check_shutdown) {
       auto blog_head = blog.head();
-      if( !fork_db.root() ) {
+      if( !blog_head && !fork_db.root() ) {
          fork_db.reset( *head );
-         if (!blog_head)
-            return;
+         return;
       }
 
       replaying = true;
@@ -2686,53 +2680,28 @@ struct controller_impl {
    // only called from non-main threads (read-only trx execution threads)
    // when producer_plugin starts them
    void init_thread_local_data() {
-      EOS_ASSERT( !is_on_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
+      EOS_ASSERT( !is_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if ( is_eos_vm_oc_enabled() )
          // EOSVMOC needs further initialization of its thread local data
          wasmif.init_thread_local_data();
       else
 #endif
-      {
          // Non-EOSVMOC needs a wasmif per thread
-         std::lock_guard g(threaded_wasmifs_mtx);
-         threaded_wasmifs[std::this_thread::get_id()] = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
-      }
+         wasmif_thread_local = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
    }
 
-   bool is_on_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+   bool is_main_thread() { return main_thread_id == std::this_thread::get_id(); };
 
-   wasm_module_cache& get_wasm_module_cache() {
-      return wasm_mod_cache;
-   }
-
-   void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
-      // The caller of this function apply_eosio_setcode has already asserted that
-      // the transaction is not a read-only trx, which implies we are in now
-      // in write window. Safe to call threaded_wasmifs's code_block_num_last_used
-      wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
-      for (auto& ele: threaded_wasmifs) {
-         ele.second->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
-      }
-      wasm_mod_cache.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
-   }
-
-   void wasm_interface_exit() {
-      // exit all running wasmifs
-      wasmif.exit();
-      for (auto& ele: threaded_wasmifs) {
-         ele.second->exit();
-      }
-   }
    wasm_interface& get_wasm_interface() {
-      if ( is_on_main_thread()
+      if ( is_main_thread()
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
           || is_eos_vm_oc_enabled()
 #endif
          )
          return wasmif;
       else
-         return *threaded_wasmifs[std::this_thread::get_id()];
+         return *wasmif_thread_local;
    }
 
    block_state_ptr fork_db_head() const;
@@ -2742,6 +2711,7 @@ thread_local platform_timer controller_impl::timer;
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
 thread_local eosio::vm::wasm_allocator controller_impl::wasm_alloc;
 #endif
+thread_local std::unique_ptr<wasm_interface> controller_impl::wasmif_thread_local;
 
 const resource_limits_manager&   controller::get_resource_limits_manager()const
 {
@@ -3716,22 +3686,6 @@ void controller::init_thread_local_data() {
    my->init_thread_local_data();
 }
 
-bool controller::is_on_main_thread() const {
-  return my->is_on_main_thread();
-}
-
-wasm_module_cache& controller::get_wasm_module_cache() {
-   return my->get_wasm_module_cache();
-}
-
-void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
-   return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
-}
-
-void controller::wasm_interface_exit() {
-   return my->wasm_interface_exit();
-}
-
 /// Protocol feature activation handlers:
 
 template<>

libraries/chain/eosio_contract.cpp

@@ -160,7 +160,7 @@ void apply_eosio_setcode(apply_context& context) {
       old_size  = (int64_t)old_code_entry.code.size() * config::setcode_ram_bytes_multiplier;
       if( old_code_entry.code_ref_count == 1 ) {
          db.remove(old_code_entry);
-         context.control.code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
+         context.control.get_wasm_interface().code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
       } else {
          db.modify(old_code_entry, [](code_object& o) {
             --o.code_ref_count;

libraries/chain/include/eosio/chain/controller.hpp

@@ -10,7 +10,6 @@
 #include <eosio/chain/account_object.hpp>
 #include <eosio/chain/snapshot.hpp>
 #include <eosio/chain/protocol_feature_manager.hpp>
-#include <eosio/chain/wasm_module_cache.hpp>
 #include <eosio/chain/webassembly/eos-vm-oc/config.hpp>
 
 namespace chainbase {
@@ -374,10 +373,6 @@ namespace eosio { namespace chain {
       void set_db_read_only_mode();
       void unset_db_read_only_mode();
       void init_thread_local_data();
-      bool is_on_main_thread() const;
-      wasm_module_cache& get_wasm_module_cache();
-      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
-      void wasm_interface_exit();
 
       private:
          friend class apply_context;

libraries/chain/include/eosio/chain/log_index.hpp

@@ -32,7 +32,7 @@ class log_index {
    bool is_open() const { return file_.is_open(); }
 
    uint64_t back() { return nth_block_position(num_blocks()-1); }
-   int      num_blocks() const { return num_blocks_; }
+   unsigned num_blocks() const { return num_blocks_; }
    uint64_t nth_block_position(uint32_t n) {
       file_.seek(n*sizeof(uint64_t));
       uint64_t r;

libraries/chain/include/eosio/chain/wasm_interface_private.hpp

@@ -37,12 +37,14 @@ namespace eosio { namespace chain {
    struct wasm_interface_impl {
       struct wasm_cache_entry {
          digest_type                                          code_hash;
+         uint32_t                                             first_block_num_used;
          uint32_t                                             last_block_num_used;
          std::unique_ptr<wasm_instantiated_module_interface>  module;
          uint8_t                                              vm_type = 0;
          uint8_t                                              vm_version = 0;
       };
       struct by_hash;
+      struct by_first_block_num;
       struct by_last_block_num;
 
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
@@ -155,6 +157,7 @@ namespace eosio { namespace chain {
 
             it = wasm_instantiation_cache.emplace( wasm_interface_impl::wasm_cache_entry{
                                                       .code_hash = code_hash,
+                                                      .first_block_num_used = codeobject->first_block_used,
                                                       .last_block_num_used = UINT32_MAX,
                                                       .module = nullptr,
                                                       .vm_type = vm_type,
@@ -170,10 +173,21 @@ namespace eosio { namespace chain {
                trx_context.resume_billing_timer();
             });
             trx_context.pause_billing_timer();
+            IR::Module module;
             std::vector<U8> bytes = {
                 (const U8*)codeobject->code.data(),
                 (const U8*)codeobject->code.data() + codeobject->code.size()};
-            IR::Module module = trx_context.control.get_wasm_module_cache().get_module(code_hash, vm_type, vm_version, bytes);
+            try {
+               Serialization::MemoryInputStream stream((const U8*)bytes.data(),
+                                                       bytes.size());
+               WASM::scoped_skip_checks no_check;
+               WASM::serialize(stream, module);
+               module.userSections.clear();
+            } catch (const Serialization::FatalSerializationException& e) {
+               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
+            } catch (const IR::ValidationException& e) {
+               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
+            }
 
             wasm_instantiation_cache.modify(it, [&](auto& c) {
                c.module = runtime_interface->instantiate_module((const char*)bytes.data(), bytes.size(), parse_initial_memory(module), code_hash, vm_type, vm_version);
@@ -195,6 +209,7 @@ namespace eosio { namespace chain {
                   member<wasm_cache_entry, uint8_t,     &wasm_cache_entry::vm_version>
                >
             >,
+            ordered_non_unique<tag<by_first_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::first_block_num_used>>,
             ordered_non_unique<tag<by_last_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::last_block_num_used>>
          >
       > wasm_cache_index;

libraries/chain/include/eosio/chain/wasm_module_cache.hpp

@@ -1,44 +0,0 @@
-#pragma once
-
-#include <eosio/chain/types.hpp>  // for digest_type
-#include "IR/Module.h"
-#include <eosio/chain/multi_index_includes.hpp>
-
-#include <mutex>
-
-namespace eosio { namespace chain {
-   struct wasm_module_cache {
-      const IR::Module& get_module( const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, std::vector<U8>& bytes );
-      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
-      void current_lib(uint32_t lib);
-
-      struct module_entry {
-         digest_type   code_hash;
-         uint8_t       vm_type = 0;
-         uint8_t       vm_version = 0;
-         uint32_t      last_block_num_used = UINT32_MAX;
-         IR::Module    module;
-      };
-      struct by_hash;
-      struct by_last_block_num;
-
-   private:
-      typedef boost::multi_index_container<
-         module_entry,
-         indexed_by<
-            ordered_unique<tag<by_hash>,
-               composite_key<module_entry,
-                  member<module_entry, digest_type, &module_entry::code_hash>,
-                  member<module_entry, uint8_t,     &module_entry::vm_type>,
-                  member<module_entry, uint8_t,     &module_entry::vm_version>
-               >
-            >,
-            ordered_non_unique<tag<by_last_block_num>, member<module_entry, uint32_t, &module_entry::last_block_num_used>>
-         >
-      > module_cache_index;
-
-      std::mutex mtx;
-      module_cache_index module_cache;
-   };
-
-} } // eosio::chain

libraries/chain/wasm_module_cache.cpp

@@ -1,55 +0,0 @@
-#include <eosio/chain/wasm_module_cache.hpp>
-#include <eosio/chain/exceptions.hpp>
-#include "WAST/WAST.h"
-#include "IR/Validate.h"
-
-namespace eosio { namespace chain {
-   const IR::Module& wasm_module_cache::get_module( const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, std::vector<U8>& bytes )
-   {
-      std::lock_guard g(mtx);
-      module_cache_index::iterator it = module_cache.find( boost::make_tuple(code_hash, vm_type, vm_version) );
-      if(it == module_cache.end()) {
-         IR::Module module;
-         try {
-            Serialization::MemoryInputStream stream((const U8*)bytes.data(), bytes.size());
-            WASM::scoped_skip_checks no_check;
-            WASM::serialize(stream, module);
-            module.userSections.clear();
-         } catch (const Serialization::FatalSerializationException& e) {
-            EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-         } catch (const IR::ValidationException& e) {
-            EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-         }
-
-         it = module_cache.emplace( module_entry {
-                                      .code_hash = code_hash,
-                                      .vm_type = vm_type,
-                                      .vm_version = vm_version,
-                                      .module = module
-                                   } ).first;
-      }
-
-      return it->module;
-   }
-
-   // apply_eosio_setcode asserted this was not called by non-read-only trxs.
-   // This implies it is in write window and no read-only threads are running.
-   // Safe to update module_cache without mutex.
-   void wasm_module_cache::code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
-      auto it = module_cache.find(boost::make_tuple(code_hash,vm_type, vm_version));
-      if(it != module_cache.end())
-         module_cache.modify(it, [block_num](module_entry& e) {
-            e.last_block_num_used = block_num;
-         });
-      }
-
-   // producer_plugin has already asserted irreversible_block signal is called
-   // in write window. No read-only threads are running. Safe to update
-   // module_cache without mutex.
-   void wasm_module_cache::current_lib(uint32_t lib) {
-      //anything last used before or on the LIB can be evicted
-      const auto first_it = module_cache.get<by_last_block_num>().begin();
-      const auto last_it  = module_cache.get<by_last_block_num>().upper_bound(lib);
-      module_cache.get<by_last_block_num>().erase(first_it, last_it);
-   }
-} } // eosio::chain

libraries/chain/webassembly/cf_system.cpp

@@ -46,6 +46,6 @@ namespace eosio { namespace chain { namespace webassembly {
    }
 
    void interface::eosio_exit( int32_t code ) const {
-      context.control.wasm_interface_exit();
+      context.control.get_wasm_interface().exit();
    }
 }}} // ns eosio::chain::webassembly

libraries/custom_appbase/include/eosio/chain/application.hpp

@@ -1,33 +1,30 @@
 #pragma once
 
 #include <appbase/application_base.hpp>
-#include <eosio/chain/exec_pri_queue.hpp>
-#include <mutex>
+#include <appbase/execution_priority_queue.hpp>
 
 /*
- * Customize appbase to support two-queue executor.
+ * Custmomize appbase to support two-queue exector.
 */
 namespace appbase { 
 
 enum class exec_window {
-   read,  // the window during which operations from read_only queue
-          // can be executed in parallel in the read-only thread pool
-          // as well as in the app thread.
-   write, // the window during which operations from both read_write and
-          // parallel queues can be executed in app thread,
-          // while read-only operations are not executed in read-only
-          // thread pool. The read-only thread pool is not active; only
-          // the main app thread is active.
+   read,  // the window during which operations from read_only_trx_safe queue
+          // can be executed in app thread in parallel with read-only transactions
+          // in read-only transaction excuting threads.
+   write, // the window during which operations from both general and
+          // read_only_trx_safe queues can be executed in app thread,
+          // while read-only transactions are not executed in read-only
+          // transaction excuting threads.
 };
 
 enum class exec_queue {
-   read_only,          // the queue storing tasks which are safe to execute
-                       // in parallel with other read-only tasks in the read-only
-                       // thread pool as well as on the main app thread.
-                       // Multi-thread safe as long as nothing is executed from the read_write queue.
-   read_write          // the queue storing tasks which can be only executed
-                       // on the app thread while read-only tasks are
-                       // not being executed in read-only threads. Single threaded.
+   read_only_trx_safe, // the queue storing operations which are safe to execute
+                       // on app thread in parallel with read-only transactions
+                       // in read-only transaction excuting threads.
+   general             // the queue storing operations which can be only executed
+                       // on the app thread while read-only transactions are
+                       // not being executed in read-only threads
 };
 
 class two_queue_executor {
@@ -35,64 +32,56 @@ public:
 
    template <typename Func>
    auto post( int priority, exec_queue q, Func&& func ) {
-      if ( q == exec_queue::read_write )
-         return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      if ( q == exec_queue::general )
+         return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
       else
-         return boost::asio::post( io_serv_, read_only_queue_.wrap( priority, --order_, std::forward<Func>( func)));
+         return boost::asio::post(io_serv_, read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Func>(func)));
    }
 
    // Legacy and deprecated. To be removed after cleaning up its uses in base appbase
    template <typename Func>
    auto post( int priority, Func&& func ) {
-      // safer to use read_write queue for unknown type of operation since operations
-      // from read_write queue are not executed in parallel with read-only operations
-      return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      // safer to use general queue for unknown type of operation since operations
+      // from general queue are not executed in parallel with read-only transactions
+      return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
    }
 
    boost::asio::io_service& get_io_service() { return io_serv_; }
 
    bool execute_highest() {
       if ( exec_window_ == exec_window::write ) {
-         // During write window only main thread is accessing anything in two_queue_executor, no locking required
-         if( !read_write_queue_.empty() && (read_only_queue_.empty() || *read_only_queue_.top() < *read_write_queue_.top()) )  {
-            // read_write_queue_'s top function's priority greater than read_only_queue_'s top function's, or read_only_queue_ empty
-            read_write_queue_.execute_highest();
-         } else if( !read_only_queue_.empty() ) {
-            read_only_queue_.execute_highest();
+         if( !general_queue_.empty() && ( read_only_trx_safe_queue_.empty() || *read_only_trx_safe_queue_.top() < *general_queue_.top()) )  {
+            // general_queue_'s top function's priority greater than read_only_trx_safe_queue_'s top function's, or general_queue_ empty
+            general_queue_.execute_highest();
+         } else if( !read_only_trx_safe_queue_.empty() ) {
+            read_only_trx_safe_queue_.execute_highest();
          }
-         return !read_only_queue_.empty() || !read_write_queue_.empty();
+         return !read_only_trx_safe_queue_.empty() || !general_queue_.empty();
       } else {
-         // When in read window, multiple threads including main app thread are accessing two_queue_executor, locking required
-         return read_only_queue_.execute_highest_locked(false);
+         return read_only_trx_safe_queue_.execute_highest();
       }
    }
 
-   bool execute_highest_read_only() {
-      return read_only_queue_.execute_highest_locked(true);
-   }
-
    template <typename Function>
-   boost::asio::executor_binder<Function, appbase::exec_pri_queue::executor>
+   boost::asio::executor_binder<Function, appbase::execution_priority_queue::executor>
    wrap(int priority, exec_queue q, Function&& func ) {
-      if ( q == exec_queue::read_write )
-         return read_write_queue_.wrap(priority, --order_, std::forward<Function>(func));
+      if ( q == exec_queue::general )
+         return general_queue_.wrap(priority, --order_, std::forward<Function>(func));
       else
-         return read_only_queue_.wrap( priority, --order_, std::forward<Function>( func));
+         return read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Function>(func));
    }
      
    void clear() {
-      read_only_queue_.clear();
-      read_write_queue_.clear();
+      read_only_trx_safe_queue_.clear();
+      general_queue_.clear();
    }
 
-   void set_to_read_window(uint32_t num_threads, std::function<bool()> should_exit) {
+   void set_to_read_window() {
       exec_window_ = exec_window::read;
-      read_only_queue_.enable_locking(num_threads, std::move(should_exit));
    }
 
    void set_to_write_window() {
       exec_window_ = exec_window::write;
-      read_only_queue_.disable_locking();
    }
 
    bool is_read_window() const {
@@ -103,15 +92,15 @@ public:
       return exec_window_ == exec_window::write;
    }
 
-   auto& read_only_queue() { return read_only_queue_; }
+   auto& read_only_trx_safe_queue() { return read_only_trx_safe_queue_; }
 
-   auto& read_write_queue() { return read_write_queue_; }
+   auto& general_queue() { return general_queue_; }
 
    // members are ordered taking into account that the last one is destructed first
 private:
    boost::asio::io_service            io_serv_;
-   appbase::exec_pri_queue            read_only_queue_;
-   appbase::exec_pri_queue            read_write_queue_;
+   appbase::execution_priority_queue  read_only_trx_safe_queue_;
+   appbase::execution_priority_queue  general_queue_;
    std::atomic<std::size_t>           order_ { std::numeric_limits<size_t>::max() }; // to maintain FIFO ordering in both queues within priority
    exec_window                        exec_window_ { exec_window::write };
 };

libraries/custom_appbase/include/eosio/chain/exec_pri_queue.hpp

@@ -1,233 +0,0 @@
-#pragma once
-#include <boost/asio.hpp>
-
-#include <condition_variable>
-#include <mutex>
-#include <queue>
-
-namespace appbase {
-// adapted from: https://www.boost.org/doc/libs/1_69_0/doc/html/boost_asio/example/cpp11/invocation/prioritised_handlers.cpp
-
-// Locking has to be coordinated by caller, use with care.
-class exec_pri_queue : public boost::asio::execution_context
-{
-public:
-
-   void enable_locking(uint32_t num_threads, std::function<bool()> should_exit) {
-      assert(num_threads > 0 && num_waiting_ == 0);
-      lock_enabled_ = true;
-      max_waiting_ = num_threads;
-      should_exit_ = std::move(should_exit);
-      exiting_blocking_ = false;
-   }
-
-   void disable_locking() {
-      lock_enabled_ = false;
-      should_exit_ = [](){ assert(false); return true; }; // should not be called when locking is disabled
-   }
-
-   // called from appbase::application_base::exec poll_one() or run_one()
-   template <typename Function>
-   void add(int priority, size_t order, Function function)
-   {
-      std::unique_ptr<queued_handler_base> handler(new queued_handler<Function>(priority, order, std::move(function)));
-      if (lock_enabled_) {
-         std::lock_guard g( mtx_ );
-         handlers_.push( std::move( handler ) );
-         if (num_waiting_)
-            cond_.notify_one();
-      } else {
-         handlers_.push( std::move( handler ) );
-      }
-   }
-
-   // only call when no lock required
-   void clear()
-   {
-      handlers_ = prio_queue();
-   }
-
-   // only call when no lock required
-   bool execute_highest()
-   {
-      if( !handlers_.empty() ) {
-         handlers_.top()->execute();
-         handlers_.pop();
-      }
-
-      return !handlers_.empty();
-   }
-
-private:
-   // has to be defined before use, auto return type
-   auto pop() {
-      auto t = std::move(const_cast<std::unique_ptr<queued_handler_base>&>(handlers_.top()));
-      handlers_.pop();
-      return t;
-   }
-
-public:
-
-   bool execute_highest_locked(bool should_block) {
-      std::unique_lock g(mtx_);
-      if (should_block) {
-         ++num_waiting_;
-         cond_.wait(g, [this](){
-            bool exit = exiting_blocking_ || should_exit_();
-            bool empty = handlers_.empty();
-            if (empty || exit) {
-               if (((empty && num_waiting_ == max_waiting_) || exit) && !exiting_blocking_) {
-                  cond_.notify_all();
-                  exiting_blocking_ = true;
-               }
-               return exit || exiting_blocking_; // same as calling should_exit(), but faster
-            }
-            return true;
-         });
-         --num_waiting_;
-         if (exiting_blocking_ || should_exit_())
-            return false;
-      }
-      if( handlers_.empty() )
-         return false;
-      auto t = pop();
-      g.unlock();
-      t->execute();
-      return true;
-   }
-
-   // Only call when locking disabled
-   size_t size() const { return handlers_.size(); }
-
-   // Only call when locking disabled
-   bool empty() const { return handlers_.empty(); }
-
-   // Only call when locking disabled
-   const auto& top() const { return handlers_.top(); }
-
-   class executor
-   {
-   public:
-      executor(exec_pri_queue& q, int p, size_t o)
-            : context_(q), priority_(p), order_(o)
-      {
-      }
-
-      exec_pri_queue& context() const noexcept
-      {
-         return context_;
-      }
-
-      template <typename Function, typename Allocator>
-      void dispatch(Function f, const Allocator&) const
-      {
-         context_.add(priority_, order_, std::move(f));
-      }
-
-      template <typename Function, typename Allocator>
-      void post(Function f, const Allocator&) const
-      {
-         context_.add(priority_, order_, std::move(f));
-      }
-
-      template <typename Function, typename Allocator>
-      void defer(Function f, const Allocator&) const
-      {
-         context_.add(priority_, order_, std::move(f));
-      }
-
-      void on_work_started() const noexcept {}
-      void on_work_finished() const noexcept {}
-
-      bool operator==(const executor& other) const noexcept
-      {
-         return order_ == other.order_ && &context_ == &other.context_ && priority_ == other.priority_;
-      }
-
-      bool operator!=(const executor& other) const noexcept
-      {
-         return !operator==(other);
-      }
-
-   private:
-      exec_pri_queue& context_;
-      int priority_;
-      size_t order_;
-   };
-
-   template <typename Function>
-   boost::asio::executor_binder<Function, executor>
-   wrap(int priority, size_t order, Function&& func)
-   {
-      return boost::asio::bind_executor( executor(*this, priority, order), std::forward<Function>(func) );
-   }
-
-private:
-   class queued_handler_base
-   {
-   public:
-      queued_handler_base( int p, size_t order )
-            : priority_( p )
-            , order_( order )
-      {
-      }
-
-      virtual ~queued_handler_base() = default;
-
-      virtual void execute() = 0;
-
-      int priority() const { return priority_; }
-      // C++20
-      // friend std::weak_ordering operator<=>(const queued_handler_base&,
-      //                                       const queued_handler_base&) noexcept = default;
-      friend bool operator<(const queued_handler_base& a,
-                            const queued_handler_base& b) noexcept
-      {
-         return std::tie( a.priority_, a.order_ ) < std::tie( b.priority_, b.order_ );
-      }
-
-   private:
-      int priority_;
-      size_t order_;
-   };
-
-   template <typename Function>
-   class queued_handler : public queued_handler_base
-   {
-   public:
-      queued_handler(int p, size_t order, Function f)
-            : queued_handler_base( p, order )
-            , function_( std::move(f) )
-      {
-      }
-
-      void execute() override
-      {
-         function_();
-      }
-
-   private:
-      Function function_;
-   };
-
-   struct deref_less
-   {
-      template<typename Pointer>
-      bool operator()(const Pointer& a, const Pointer& b) noexcept(noexcept(*a < *b))
-      {
-         return *a < *b;
-      }
-   };
-
-   bool lock_enabled_ = false;
-   std::mutex mtx_;
-   std::condition_variable cond_;
-   uint32_t num_waiting_{0};
-   uint32_t max_waiting_{0};
-   bool exiting_blocking_{false};
-   std::function<bool()> should_exit_; // called holding mtx_
-   using prio_queue = std::priority_queue<std::unique_ptr<queued_handler_base>, std::deque<std::unique_ptr<queued_handler_base>>, deref_less>;
-   prio_queue handlers_;
-};
-
-} // appbase

libraries/custom_appbase/tests/custom_appbase_tests.cpp

@@ -27,28 +27,28 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
 
    // Stop app. Use the lowest priority to make sure this function to execute the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
-      // read_only_queue should only contain the current lambda function,
-      // and read_write_queue should have executed all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_only_trx_safe_queue should only contain the current lambda function,
+      // and general_queue should have execute all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
 
    // exactly number of both queues' functions processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 8 );
@@ -64,40 +64,40 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    BOOST_CHECK_LT( rslts[6], rslts[7] );
 }
 
-// verify functions only from read_only queue are processed during read window
+// verify functions only from read_only_trx_safe queue are processed during read window
 BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only queue only
-   app->executor().set_to_read_window(1, [](){return false;});
+   // set to run functions from read_only_trx_safe queue only
+   app->executor().set_to_read_window();
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
-      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 4 );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_queue should have current function and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 4 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 6 );
@@ -109,40 +109,40 @@ BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    BOOST_CHECK_LT( rslts[3], rslts[4] );
 }
 
-// verify no functions are executed during  read window if read_only queue is empty
+// verify no functions are executed during  read window if read_only_trx_safe queue is empty
 BOOST_AUTO_TEST_CASE( execute_from_empty_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only queue only
-   app->executor().set_to_read_window(1, [](){return false;});
+   // set to run functions from read_only_trx_safe queue only
+   app->executor().set_to_read_window();
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // Stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
-      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 10 );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_queue should have current function and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 10 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
 
    // no results
    BOOST_REQUIRE_EQUAL( rslts.size(), 0 );
@@ -159,32 +159,32 @@ BOOST_AUTO_TEST_CASE( execute_from_both_queues ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[9]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[10]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[11]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[10]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[11]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
       // read_queue should have current function and write_queue's functions are all executed 
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
       app->quit();
       } );
 
    app_thread.join();
 
    // queues are emptied after quit
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 12 );

libraries/libfc/CMakeLists.txt

@@ -57,6 +57,8 @@ set( fc_sources
      src/crypto/blake2.cpp
      src/crypto/k1_recover.cpp
      src/network/ip.cpp
+     src/network/resolve.cpp
+     src/network/udp_socket.cpp
      src/network/url.cpp
      src/network/http/http_client.cpp
      src/compress/zlib.cpp

libraries/libfc/include/fc/network/resolve.hpp

@@ -0,0 +1,9 @@
+#pragma once
+#include <fc/vector.hpp>
+#include <fc/network/ip.hpp>
+
+namespace fc
+{
+  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
+                                                      const std::string& host, uint16_t port);
+}

libraries/libfc/include/fc/network/udp_socket.hpp

@@ -0,0 +1,39 @@
+#pragma once
+#include <fc/utility.hpp>
+#include <memory>
+
+#include <boost/asio.hpp>
+
+namespace fc {
+  namespace ip {
+    class endpoint;
+    class address;
+  }
+
+  /**
+   *  The udp_socket class has reference semantics, all copies will
+   *  refer to the same underlying socket.
+   */
+  class udp_socket {
+    public:
+      udp_socket();
+      udp_socket( const udp_socket& s );
+      ~udp_socket();
+
+      void initialize(boost::asio::io_service &);
+      void open();
+      void send_to(const char* b, size_t l, boost::asio::ip::udp::endpoint &to);
+      void send_to(const std::shared_ptr<const char>& b, size_t l, boost::asio::ip::udp::endpoint &to);
+      void close();
+
+      void set_reuse_address(bool);
+
+      void connect(const boost::asio::ip::udp::endpoint& e);
+      const boost::asio::ip::udp::endpoint local_endpoint() const;
+
+    private:
+      class                impl;
+      std::shared_ptr<impl> my;
+  };
+
+}

libraries/libfc/src/log/gelf_appender.cpp

@@ -1,3 +1,6 @@
+#include <fc/network/udp_socket.hpp>
+#include <fc/network/ip.hpp>
+#include <fc/network/resolve.hpp>
 #include <fc/exception/exception.hpp>
 #include <fc/log/gelf_appender.hpp>
 #include <fc/reflect/variant.hpp>
@@ -7,7 +10,6 @@
 #include <fc/compress/zlib.hpp>
 #include <fc/log/logger_config.hpp>
 
-#include <boost/asio.hpp>
 #include <boost/lexical_cast.hpp>
 #include <iomanip>
 #include <iostream>
@@ -17,6 +19,14 @@
 
 namespace fc
 {
+  namespace detail
+  {
+    boost::asio::ip::udp::endpoint to_asio_ep( const fc::ip::endpoint& e )
+    {
+      return boost::asio::ip::udp::endpoint(boost::asio::ip::address_v4(e.get_address()), e.port() );
+    }
+  }
+
   const std::vector<std::string> gelf_appender::config::reserved_field_names  = {
       "_id",            // per GELF specification
       "_timestamp_ns",  // Remaining names all populated by appender
@@ -26,21 +36,23 @@ namespace fc
       "_method_name",
       "_thread_name",
       "_task_name"
-  };
+   };
 
-  const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
+   const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
 
   class gelf_appender::impl
   {
   public:
     using work_guard_t = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
     config                                         cfg;
+    std::optional<boost::asio::ip::udp::endpoint>  gelf_endpoint;
     std::thread                                    thread;
     boost::asio::io_context                        io_context;
     work_guard_t work_guard =                      boost::asio::make_work_guard(io_context);
-    boost::asio::ip::udp::socket                   gelf_socket;
+    udp_socket                                     gelf_socket;
 
-    impl(const variant& c) : gelf_socket(io_context)
+
+    impl(const variant& c)
     {
       mutable_variant_object mvo;
       from_variant(c, mvo);
@@ -72,35 +84,6 @@ namespace fc
           thread.join();
        }
     }
-
-    static std::shared_ptr<std::vector<char>> make_new_bufer(boost::asio::const_buffer buf) {
-      const char* p = static_cast<const char*>(buf.data());
-      return std::make_shared<std::vector<char>>(p, p+buf.size());
-    }
-
-    static std::shared_ptr<std::vector<char>> make_new_bufer(const std::array<boost::asio::const_buffer, 2>& bufs) {
-      auto new_buf = std::make_shared<std::vector<char>>();
-      new_buf->reserve(bufs[0].size() + bufs[1].size());
-      for (int i = 0; i < 2; ++i) {
-          const char* p = static_cast<const char*>(bufs[i].data());
-          new_buf->insert(new_buf->end(), p, p + bufs[i].size());
-      }
-      return new_buf;
-    }
-
-    template <typename Buffers>
-    void send(Buffers&& bufs) {
-      boost::system::error_code ec;
-      gelf_socket.send(std::forward<Buffers>(bufs), 0, ec);
-      if (ec == boost::asio::error::would_block) {
-          auto new_buf = make_new_bufer(std::forward<Buffers>(bufs));
-          gelf_socket.async_send(boost::asio::buffer(*new_buf),
-                                    [new_buf](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/) {
-                                        // Swallow errors.  Currently only used for GELF logging, so depend on local
-                                        // log to catch anything that doesn't make it across the network.
-                                    });
-      }
-    }
   };
 
   gelf_appender::gelf_appender(const variant& args) :
@@ -112,40 +95,58 @@ namespace fc
   {
     try
     {
-      if (my->cfg.endpoint.empty()) {
-        fprintf(stderr, "The logging destination is not specified\n");
-        return;
+      try
+      {
+        // if it's a numeric address:port, this will parse it
+        my->gelf_endpoint = detail::to_asio_ep(ip::endpoint::from_string(my->cfg.endpoint));
+      }
+      catch (...)
+      {
+      }
+      if (!my->gelf_endpoint)
+      {
+        // couldn't parse as a numeric ip address, try resolving as a DNS name.
+        // This can yield, so don't do it in the catch block above
+        string::size_type colon_pos = my->cfg.endpoint.find(':');
+        try
+        {
+          FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
+          string port = my->cfg.endpoint.substr(colon_pos + 1);
+
+          string hostname = my->cfg.endpoint.substr( 0, colon_pos );
+
+          boost::asio::ip::udp::resolver resolver{ my->io_context };
+          auto endpoints = resolver.resolve(hostname, port);
+
+          if (endpoints.empty())
+              FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
+                                 ("hostname", hostname));
+
+          my->gelf_endpoint = *endpoints.begin();
+        }
+        catch (const boost::bad_lexical_cast&)
+        {
+          FC_THROW("Bad port: ${port}", ("port", my->cfg.endpoint.substr(colon_pos + 1, my->cfg.endpoint.size())));
+        }
       }
 
-      std::string_view endpoint = my->cfg.endpoint;
-      string::size_type colon_pos = endpoint.rfind(':');
-      FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
-      auto port = endpoint.substr(colon_pos + 1);
-
-      auto hostname = (endpoint[0] == '[' && colon_pos >= 2)  ? endpoint.substr( 1, colon_pos-2 ) : endpoint.substr( 0, colon_pos );
-
-      boost::asio::ip::udp::resolver resolver{ my->io_context };
-      auto endpoints = resolver.resolve(hostname, port);
-
-      if (endpoints.empty())
-          FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
-                              ("hostname", std::string(hostname)));
-
-      my->gelf_socket.connect(*endpoints.begin());
-      std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
-      my->gelf_socket.non_blocking(true);
-
-      my->thread = std::thread([this] {
-        try {
-          fc::set_os_thread_name("gelf");
-          my->io_context.run();
-        } catch (std::exception& ex) {
-          fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
-        } catch (...) {
-          fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
-        }
-      });
+      if (my->gelf_endpoint)
+      {
+        my->gelf_socket.initialize(my->io_context);
+        my->gelf_socket.open();
+        std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
 
+        my->thread = std::thread([this] {
+          try {
+            fc::set_os_thread_name("gelf");
+            my->io_context.run();
+          } catch (std::exception& ex) {
+            fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
+          } catch (...) {
+            fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
+          }
+        });
+      }
     }
     catch (...)
     {
@@ -220,44 +221,57 @@ namespace fc
 
     if (gelf_message_as_string.size() <= max_payload_size)
     {
-      my->send(boost::asio::buffer(gelf_message_as_string));
+      // no need to split
+      std::shared_ptr<char> send_buffer(new char[gelf_message_as_string.size()],
+                                        [](char* p){ delete[] p; });
+      memcpy(send_buffer.get(), gelf_message_as_string.c_str(),
+             gelf_message_as_string.size());
+
+      my->gelf_socket.send_to(send_buffer, gelf_message_as_string.size(),
+                              *my->gelf_endpoint);
     }
     else
     {
       // split the message
-      struct gelf_header {
-        uint8_t magic[2] = { 0x1e, 0x0f};
-        uint64_t message_id;
-        uint8_t seq = 0;
-        uint8_t count = 0;
-      } header;
-
       // we need to generate an 8-byte ID for this message.
       // city hash should do
-      header.message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
-      const unsigned body_length = max_payload_size - sizeof(header);
-      header.count = (gelf_message_as_string.size() + body_length - 1) / body_length;
+      uint64_t message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
+      const unsigned header_length = 2 /* magic */ + 8 /* msg id */ + 1 /* seq */ + 1 /* count */;
+      const unsigned body_length = max_payload_size - header_length;
+      unsigned total_number_of_packets = (gelf_message_as_string.size() + body_length - 1) / body_length;
       unsigned bytes_sent = 0;
-
+      unsigned number_of_packets_sent = 0;
       while (bytes_sent < gelf_message_as_string.size())
       {
         unsigned bytes_to_send = std::min((unsigned)gelf_message_as_string.size() - bytes_sent,
                                           body_length);
 
-        std::array<boost::asio::const_buffer,2> bufs = {
-          boost::asio::const_buffer(&header, sizeof(header)),
-          boost::asio::const_buffer(gelf_message_as_string.c_str() + bytes_sent, bytes_to_send)
-        };
-        my->send(bufs);
-        ++header.seq;
+        std::shared_ptr<char> send_buffer(new char[max_payload_size],
+                                          [](char* p){ delete[] p; });
+        char* ptr = send_buffer.get();
+        // magic number for chunked message
+        *(unsigned char*)ptr++ = 0x1e;
+        *(unsigned char*)ptr++ = 0x0f;
+
+        // message id
+        memcpy(ptr, (char*)&message_id, sizeof(message_id));
+        ptr += sizeof(message_id);
+
+        *(unsigned char*)(ptr++) = number_of_packets_sent;
+        *(unsigned char*)(ptr++) = total_number_of_packets;
+        memcpy(ptr, gelf_message_as_string.c_str() + bytes_sent,
+               bytes_to_send);
+        my->gelf_socket.send_to(send_buffer, header_length + bytes_to_send,
+                                *my->gelf_endpoint);
+        ++number_of_packets_sent;
         bytes_sent += bytes_to_send;
       }
-      FC_ASSERT(header.seq == header.count);
+      FC_ASSERT(number_of_packets_sent == total_number_of_packets);
     }
   }
 
   void gelf_appender::log(const log_message& message) {
-    if (!my->thread.joinable())
+    if (!my->gelf_endpoint)
       return;
 
     // use now() instead of context.get_timestamp() because log_message construction can include user provided long running calls

libraries/libfc/src/network/resolve.cpp

@@ -0,0 +1,33 @@
+#include <boost/asio.hpp>
+#include <fc/exception/exception.hpp>
+
+namespace fc
+{
+  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
+                                                     const std::string& host, uint16_t port)
+  {
+    using q = boost::asio::ip::udp::resolver::query;
+    using b = boost::asio::ip::resolver_query_base;
+    boost::asio::ip::udp::resolver res(io_service);
+    boost::system::error_code ec;
+    auto ep = res.resolve(q(host, std::to_string(uint64_t(port)),
+                            b::address_configured | b::numeric_service), ec);
+    if(!ec)
+    {
+      std::vector<boost::asio::ip::udp::endpoint> eps;
+      while(ep != boost::asio::ip::udp::resolver::iterator())
+      {
+        if(ep->endpoint().address().is_v4())
+        {
+         eps.push_back(*ep);
+        }
+        // TODO: add support for v6
+        ++ep;
+      }
+      return eps;
+    }
+    FC_THROW_EXCEPTION(unknown_host_exception,
+                       "name resolution failed: ${reason}",
+                       ("reason", ec.message()));
+  }
+}

libraries/libfc/src/network/udp_socket.cpp

@@ -0,0 +1,111 @@
+#include <fc/network/udp_socket.hpp>
+#include <fc/network/ip.hpp>
+
+
+namespace fc
+{
+  class udp_socket::impl
+  {
+    public:
+      impl(){}
+      ~impl(){}
+
+      std::shared_ptr<boost::asio::ip::udp::socket> _sock;
+  };
+
+  udp_socket::udp_socket()
+    : my(new impl())
+  {
+  }
+
+  void udp_socket::initialize(boost::asio::io_service& service)
+  {
+    my->_sock.reset(new boost::asio::ip::udp::socket(service));
+  }
+
+  udp_socket::~udp_socket() 
+  {
+    try 
+    {
+      if(my->_sock)
+        my->_sock->close(); //close boost socket to make any pending reads run their completion handler
+    }
+    catch (...) //avoid destructor throw and likely this is just happening because socket wasn't open.
+    {
+    }
+  }
+
+  void udp_socket::send_to(const char* buffer, size_t length, boost::asio::ip::udp::endpoint& to)
+  {
+    try 
+    {
+      my->_sock->send_to(boost::asio::buffer(buffer, length), to);
+      return;
+    } 
+    catch(const boost::system::system_error& e)
+    {
+      if(e.code() == boost::asio::error::would_block)
+      {
+          auto send_buffer_ptr = std::make_shared<std::vector<char>>(buffer, buffer+length);
+          my->_sock->async_send_to(boost::asio::buffer(send_buffer_ptr.get(), length), to,
+                                   [send_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
+          {
+            // Swallow errors.  Currently only used for GELF logging, so depend on local
+            // log to catch anything that doesn't make it across the network.
+          });
+       }
+       // All other exceptions ignored.
+    }
+  }
+
+  void udp_socket::send_to(const std::shared_ptr<const char>& buffer, size_t length,
+                           boost::asio::ip::udp::endpoint& to)
+  {
+    try
+    {
+      my->_sock->send_to(boost::asio::buffer(buffer.get(), length), to);
+      return;
+    } 
+    catch(const boost::system::system_error& e)
+    {
+      if(e.code() == boost::asio::error::would_block)
+      {
+          auto preserved_buffer_ptr = buffer;
+          my->_sock->async_send_to(boost::asio::buffer(preserved_buffer_ptr.get(), length), to,
+                                   [preserved_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
+          {
+            // Swallow errors.  Currently only used for GELF logging, so depend on local
+            // log to catch anything that doesn't make it across the network.
+          });
+       }
+       // All other exceptions ignored.
+    }
+  }
+
+  void udp_socket::open()
+  {
+    my->_sock->open(boost::asio::ip::udp::v4());
+    my->_sock->non_blocking(true);
+  }
+
+  void udp_socket::close()
+  {
+    my->_sock->close();
+  }
+
+  const boost::asio::ip::udp::endpoint udp_socket::local_endpoint() const
+  {
+    return my->_sock->local_endpoint();
+  }
+
+  void udp_socket::connect(const boost::asio::ip::udp::endpoint& e)
+  {
+    my->_sock->connect(e);
+  }
+
+  void udp_socket::set_reuse_address( bool s )
+  {
+    my->_sock->set_option( boost::asio::ip::udp::socket::reuse_address(s) );
+  }
+
+}

libraries/state_history/include/eosio/state_history/log.hpp

@@ -550,9 +550,6 @@ class state_history_log {
          fc::raw::pack(log, num_blocks_in_log);
       }
 
-      log.flush();
-      index.flush();
-
       auto partition_config = std::get_if<state_history::partition_config>(&config);
       if (partition_config && block_num % partition_config->stride == 0) {
          split_log();

libraries/wasm-jit/Include/IR/Operators.h

@@ -12,10 +12,11 @@ namespace IR
 	struct NoImm {};
 	struct MemoryImm {};
 
+	PACKED_STRUCT(
 	struct ControlStructureImm
 	{
 		ResultType resultType{};
-	};
+	});
 
 	struct BranchImm
 	{
@@ -675,4 +676,19 @@ namespace IR
 	};
 
 	IR_API const char* getOpcodeName(Opcode opcode);
-}
+}
+
+//paranoia for future platforms
+static_assert(sizeof(IR::OpcodeAndImm<IR::NoImm>) == 2);
+static_assert(sizeof(IR::OpcodeAndImm<IR::MemoryImm>) == 2);
+static_assert(sizeof(IR::OpcodeAndImm<IR::ControlStructureImm>) == 3);
+static_assert(sizeof(IR::OpcodeAndImm<IR::BranchImm>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::BranchTableImm>) == 18);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<I32>>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<I64>>) == 10);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<F32>>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<F64>>) == 10);
+static_assert(sizeof(IR::OpcodeAndImm<IR::GetOrSetVariableImm<false>>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::CallImm>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::CallIndirectImm>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LoadOrStoreImm<0>>) == 10);

plugins/chain_api_plugin/chain_api_plugin.cpp

@@ -75,7 +75,7 @@ void chain_api_plugin::plugin_startup() {
    ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() );
 
    _http_plugin.add_api( {
-      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only, appbase::priority::medium_high);
+      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
    _http_plugin.add_api({
       CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params),
       CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required),
@@ -96,19 +96,19 @@ void chain_api_plugin::plugin_startup() {
       CHAIN_RO_CALL(get_scheduled_transactions, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_required_keys, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_transaction_id, 200, http_params_types::params_required),
-      // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue
       CHAIN_RO_CALL_ASYNC(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required),
       CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required),
+      // transaction related APIs will be posted to general queue after keys are recovered
       CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_only);
+   }, appbase::exec_queue::read_only_trx_safe);
 
    // Not safe to run in parallel with read-only transactions
    _http_plugin.add_api({
       CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_write, appbase::priority::medium_low );
+   }, appbase::exec_queue::general, appbase::priority::medium_low );
 
    if (chain.account_queries_enabled()) {
       _http_plugin.add_async_api({
@@ -124,7 +124,7 @@ void chain_api_plugin::plugin_startup() {
    if (chain.transaction_finality_status_enabled()) {
       _http_plugin.add_api({
          CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required),
-      }, appbase::exec_queue::read_only);
+      }, appbase::exec_queue::read_only_trx_safe);
    }
 
    _http_plugin.add_api({
@@ -160,7 +160,7 @@ void chain_api_plugin::plugin_startup() {
            }
         }
       }
-   }, appbase::exec_queue::read_only);
+   }, appbase::exec_queue::read_only_trx_safe);
 }
 
 void chain_api_plugin::plugin_shutdown() {}

plugins/db_size_api_plugin/db_size_api_plugin.cpp

@@ -28,7 +28,7 @@ using namespace eosio;
 void db_size_api_plugin::plugin_startup() {
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(db_size, this, get,  INVOKE_R_V(this, get), 200),
-   }, appbase::exec_queue::read_only);
+   }, appbase::exec_queue::read_only_trx_safe);
 }
 
 db_size_stats db_size_api_plugin::get() {

plugins/http_plugin/http_plugin.cpp

@@ -334,7 +334,7 @@ namespace eosio {
                      handle_exception("node", "get_supported_apis", body.empty() ? "{}" : body, cb);
                   }
                }
-            }}, appbase::exec_queue::read_only);
+            }}, appbase::exec_queue::read_only_trx_safe);
             
          } catch (...) {
             fc_elog(logger(), "http_plugin startup fails, shutting down");

plugins/http_plugin/tests/unit_tests.cpp

@@ -52,7 +52,7 @@ public:
                   cb(200, fc::time_point::maximum(), fc::variant(ok ? string("yes") : string("no")));
                }
             },
-         }, appbase::exec_queue::read_write);
+         }, appbase::exec_queue::general);
    }
 
 private:

plugins/net_api_plugin/net_api_plugin.cpp

@@ -53,7 +53,11 @@ void net_api_plugin::plugin_startup() {
    ilog("starting net_api_plugin");
    // lifetime of plugin is lifetime of application
    auto& net_mgr = app().get_plugin<net_plugin>();
-   app().get_plugin<http_plugin>().add_async_api({
+   app().get_plugin<http_plugin>().add_api({
+    //   CALL(net, net_mgr, set_timeout,
+    //        INVOKE_V_R(net_mgr, set_timeout, int64_t), 200),
+    //   CALL(net, net_mgr, sign_transaction,
+    //        INVOKE_R_R_R_R(net_mgr, sign_transaction, chain::signed_transaction, flat_set<public_key_type>, chain::chain_id_type), 201),
        CALL_WITH_400(net, net_mgr, connect,
             INVOKE_R_R(net_mgr, connect, std::string), 201),
        CALL_WITH_400(net, net_mgr, disconnect,
@@ -62,7 +66,9 @@ void net_api_plugin::plugin_startup() {
             INVOKE_R_R(net_mgr, status, std::string), 201),
        CALL_WITH_400(net, net_mgr, connections,
             INVOKE_R_V(net_mgr, connections), 201),
-   } );
+    //   CALL(net, net_mgr, open,
+    //        INVOKE_V_R(net_mgr, open, std::string), 200),
+   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
 }
 
 void net_api_plugin::plugin_initialize(const variables_map& options) {

plugins/net_plugin/net_plugin.cpp

@@ -3225,23 +3225,22 @@ namespace eosio {
             return;
          }
 
+         bool valid_block_header = !!bsp;
 
-         uint32_t block_num = bsp ? bsp->block_num : 0;
-
-         if( block_num != 0 ) {
+         if( valid_block_header ) {
             fc_dlog( logger, "validated block header, broadcasting immediately, connection ${cid}, blk num = ${num}, id = ${id}",
-                     ("cid", cid)("num", block_num)("id", bsp->id) );
+                     ("cid", cid)("num", bsp->block_num)("id", bsp->id) );
             my_impl->dispatcher->add_peer_block( bsp->id, cid ); // no need to send back to sender
             my_impl->dispatcher->bcast_block( bsp->block, bsp->id );
          }
 
-         app().executor().post(priority::medium, exec_queue::read_write, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
+         app().executor().post(priority::medium, exec_queue::general, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
             c->process_signed_block( id, std::move(ptr), std::move(bsp) );
          });
 
-         if( block_num != 0 ) {
+         if( valid_block_header ) {
             // ready to process immediately, so signal producer to interrupt start_block
-            my_impl->producer_plug->received_block(block_num);
+            my_impl->producer_plug->received_block();
          }
       });
    }

plugins/producer_api_plugin/producer_api_plugin.cpp

@@ -118,11 +118,9 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_R(producer, get_account_ram_corrections, producer_plugin::get_account_ram_corrections_params), 201),
        CALL_WITH_400(producer, producer, get_unapplied_transactions,
                      INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200),
-       CALL_WITH_400(producer, producer, get_snapshot_requests,
-                     INVOKE_R_V(producer, get_snapshot_requests), 201),
-   }, appbase::exec_queue::read_only, appbase::priority::medium_high);
+   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
 
-   // Not safe to run in parallel
+   // Not safe to run in parallel with read-only transactions
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(producer, producer, pause,
             INVOKE_V_V(producer, pause), 201),
@@ -140,13 +138,15 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
        CALL_WITH_400(producer, producer, schedule_snapshot,
             INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+            INVOKE_R_V(producer, get_snapshot_requests), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,
             INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
        CALL_WITH_400(producer, producer, get_integrity_hash,
             INVOKE_R_V(producer, get_integrity_hash), 201),
        CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
             INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201),
-   }, appbase::exec_queue::read_write, appbase::priority::medium_high);
+   }, appbase::exec_queue::general, appbase::priority::medium_high);
 }
 
 void producer_api_plugin::plugin_initialize(const variables_map& options) {

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -202,14 +202,11 @@ public:
    void register_metrics_listener(metrics_listener listener);
 
    // thread-safe, called when a new block is received
-   void received_block(uint32_t block_num);
+   void received_block();
 
-   const std::set<account_name>& producer_accounts() const;
+const std::set<account_name>& producer_accounts() const;
 
-   static void set_test_mode(bool m) { test_mode_ = m; }
  private:
-   inline static bool test_mode_{false}; // to be moved into appbase (application_base)
-   
    std::shared_ptr<class producer_plugin_impl> my;
 };
 

plugins/producer_plugin/producer_plugin.cpp

@@ -14,7 +14,6 @@
 #include <fc/io/json.hpp>
 #include <fc/log/logger_config.hpp>
 #include <fc/scoped_exit.hpp>
-#include <fc/time.hpp>
 
 #include <boost/asio.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -28,6 +27,7 @@
 #include <boost/multi_index/member.hpp>
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/random_access_index.hpp>
 #include <boost/signals2/connection.hpp>
 
 namespace bmi = boost::multi_index;
@@ -256,53 +256,39 @@ struct block_time_tracker {
       block_idle_time += idle;
    }
 
-   void add_fail_time( const fc::microseconds& fail_time, bool is_transient ) {
-      if( is_transient ) {
-         // transient time includes both success and fail time
-         transient_trx_time += fail_time;
-         ++transient_trx_num;
-      } else {
-         trx_fail_time += fail_time;
-         ++trx_fail_num;
-      }
+   void add_fail_time( const fc::microseconds& fail_time ) {
+      trx_fail_time += fail_time;
+      ++trx_fail_num;
    }
 
-   void add_success_time( const fc::microseconds& time, bool is_transient ) {
-      if( is_transient ) {
-         transient_trx_time += time;
-         ++transient_trx_num;
-      } else {
-         trx_success_time += time;
-         ++trx_success_num;
-      }
+   void add_success_time( const fc::microseconds& time ) {
+      trx_success_time += time;
+      ++trx_success_num;
    }
 
    void report( const fc::time_point& idle_trx_time, uint32_t block_num ) {
       if( _log.is_enabled( fc::log_level::debug ) ) {
          auto now = fc::time_point::now();
          add_idle_time( now - idle_trx_time );
-         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, transient: ${trans_trx_num}, ${trans_trx_time}us, other: ${o}us",
+         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, other: ${o}us",
                   ("n", block_num)
                   ("i", block_idle_time)("t", now - clear_time)("sn", trx_success_num)("s", trx_success_time)
                   ("fn", trx_fail_num)("f", trx_fail_time)
-                  ("trans_trx_num", transient_trx_num)("trans_trx_time", transient_trx_time)
-                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time - transient_trx_time) );
+                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time) );
       }
    }
 
    void clear() {
-      block_idle_time = trx_fail_time = trx_success_time = transient_trx_time = fc::microseconds{};
-      trx_fail_num = trx_success_num = transient_trx_num = 0;
+      block_idle_time = trx_fail_time = trx_success_time = fc::microseconds{};
+      trx_fail_num = trx_success_num = 0;
       clear_time = fc::time_point::now();
    }
 
    fc::microseconds block_idle_time;
    uint32_t trx_success_num = 0;
    uint32_t trx_fail_num = 0;
-   uint32_t transient_trx_num = 0;
    fc::microseconds trx_success_time;
    fc::microseconds trx_fail_time;
-   fc::microseconds transient_trx_time;
    fc::time_point clear_time{fc::time_point::now()};
 };
 
@@ -313,7 +299,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       producer_plugin_impl(boost::asio::io_service& io)
       :_timer(io)
       ,_transaction_ack_channel(app().get_channel<compat::channels::transaction_ack>())
-      ,_ro_timer(io)
+      ,_ro_write_window_timer(io)
+      ,_ro_read_window_timer(io)
       {
       }
 
@@ -337,9 +324,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       push_result push_transaction( const fc::time_point& block_deadline,
                                     const transaction_metadata_ptr& trx,
                                     bool api_trx, bool return_failure_trace,
-                                    const next_function<transaction_trace_ptr>& next );
+                                    next_function<transaction_trace_ptr> next );
       push_result handle_push_result( const transaction_metadata_ptr& trx,
-                                      const next_function<transaction_trace_ptr>& next,
+                                      next_function<transaction_trace_ptr> next,
                                       const fc::time_point& start,
                                       const chain::controller& chain,
                                       const transaction_trace_ptr& trace,
@@ -369,7 +356,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       named_thread_pool<struct prod>                            _thread_pool;
 
       std::atomic<int32_t>                                      _max_transaction_time_ms; // modified by app thread, read by net_plugin thread pool
-      std::atomic<uint32_t>                                     _received_block{0}; // modified by net_plugin thread pool
+      std::atomic<bool>                                         _received_block{false}; // modified by net_plugin thread pool and app thread
       fc::microseconds                                          _max_irreversible_block_age_us;
       int32_t                                                   _produce_time_offset_us = 0;
       int32_t                                                   _last_block_time_offset_us = 0;
@@ -427,60 +414,39 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       
       // ro for read-only
       struct ro_trx_t {
-         transaction_metadata_ptr trx;
-         next_func_t              next;
+         packed_transaction_ptr  trx;
+         next_func_t             next;
       };
-      // The queue storing previously exhausted read-only transactions to be re-executed by read-only threads
-      // thread-safe
-      class ro_trx_queue_t {
-      public:
-         void push_front(ro_trx_t&& t) {
-            std::lock_guard g(mtx);
-            queue.push_front(std::move(t));
-         }
-
-         bool empty() const {
-            std::lock_guard g(mtx);
-            return queue.empty();
-         }
-
-         bool pop_front(ro_trx_t& t) {
-            std::unique_lock g(mtx);
-            if (queue.empty())
-               return false;
-            t = queue.front();
-            queue.pop_front();
-            return true;
-         }
-
-      private:
-         mutable std::mutex      mtx;
-         deque<ro_trx_t>         queue;  // boost deque which is faster than std::deque
+      // The queue storing read-only transactions to be executed by read-only threads
+      struct ro_trx_queue_t {
+         std::mutex              mtx;
+         std::deque<ro_trx_t>    queue;
       };
-
-      uint32_t                        _ro_thread_pool_size{ 0 };
-      static constexpr uint32_t       _ro_max_eos_vm_oc_threads_allowed{ 8 }; // Due to uncertainty to get total virtual memory size on a 5-level paging system, set a hard limit
+      uint16_t                        _ro_thread_pool_size{ 0 };
+      static constexpr uint16_t       _ro_max_eos_vm_oc_threads_allowed{ 8 }; // Due to uncertainty to get total virtual memory size on a 5-level paging system, set a hard limit
       named_thread_pool<struct read>  _ro_thread_pool;
       fc::microseconds                _ro_write_window_time_us{ 200000 };
       fc::microseconds                _ro_read_window_time_us{ 60000 };
       static constexpr fc::microseconds _ro_read_window_minimum_time_us{ 10000 };
       fc::microseconds                _ro_read_window_effective_time_us{ 0 }; // calculated during option initialization
-      std::atomic<int64_t>            _ro_all_threads_exec_time_us; // total time spent by all threads executing transactions. use atomic for simplicity and performance
-      fc::time_point                  _ro_read_window_start_time;
-      fc::time_point                  _ro_window_deadline; // only modified on app thread, read-window deadline or write-window deadline
-      boost::asio::deadline_timer     _ro_timer; // only accessible from the main thread
+      boost::asio::deadline_timer     _ro_write_window_timer;
+      boost::asio::deadline_timer     _ro_read_window_timer;
       fc::microseconds                _ro_max_trx_time_us{ 0 }; // calculated during option initialization
-      ro_trx_queue_t                  _ro_exhausted_trx_queue;
-      std::atomic<uint32_t>           _ro_num_active_exec_tasks{ 0 };
-      std::vector<std::future<bool>>  _ro_exec_tasks_fut;
+      ro_trx_queue_t                  _ro_trx_queue;
+      std::atomic<uint32_t>            _ro_num_active_trx_exec_tasks{ 0 };
+      std::vector<std::future<bool>>  _ro_trx_exec_tasks_fut;
 
       void start_write_window();
       void switch_to_write_window();
       void switch_to_read_window();
-      bool read_only_execution_task(uint32_t pending_block_num);
-      void repost_exhausted_transactions(const fc::time_point& deadline);
-      bool push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next);
-
+      bool read_only_trx_execution_task();
+      bool process_read_only_transaction(const packed_transaction_ptr& trx,
+                                         const next_function<transaction_trace_ptr>& next,
+                                         const fc::time_point& read_window_start_time);
+      bool push_read_only_transaction(const transaction_metadata_ptr& trx,
+                                      next_function<transaction_trace_ptr> next,
+                                      const fc::time_point& read_window_start_time);
+      
       void consider_new_watermark( account_name producer, uint32_t block_num, block_timestamp_type timestamp) {
          auto itr = _producer_watermarks.find( producer );
          if( itr != _producer_watermarks.end() ) {
@@ -514,7 +480,6 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       }
 
       void on_irreversible_block( const signed_block_ptr& lib ) {
-         EOS_ASSERT(app().executor().is_write_window(), producer_exception, "write window is expected for on_irreversible_block signal");
          _irreversible_block_time = lib->timestamp.to_time_point();
          const chain::controller& chain = chain_plug->chain();
 
@@ -674,12 +639,12 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                          transaction_metadata::trx_type trx_type,
                                          bool return_failure_traces,
                                          next_function<transaction_trace_ptr> next) {
-         if ( trx_type == transaction_metadata::trx_type::read_only ) {
-            // Post all read only trxs to read_only queue for execution.
-            auto trx_metadata = transaction_metadata::create_no_recover_keys( trx, transaction_metadata::trx_type::read_only );
-            app().executor().post(priority::low, exec_queue::read_only, [this, trx{std::move(trx_metadata)}, next{std::move(next)}]() mutable {
-               push_read_only_transaction( std::move(trx), std::move(next) );
-            } );
+         if ( trx_type == transaction_metadata::trx_type::read_only && _ro_thread_pool_size > 0 ) {
+            // Parallel read-only trx execution enabled.
+            // Store the transaction in read-only-trx queue so that it is
+            // executed in read window
+            std::lock_guard<std::mutex> g( _ro_trx_queue.mtx );
+            _ro_trx_queue.queue.push_back({trx, std::move(next)});
             return;
          }
 
@@ -712,7 +677,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                                           next{std::move(next)}, trx=trx]() mutable {
             if( future.valid() ) {
                future.wait();
-               app().executor().post( priority::low, exec_queue::read_write, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
+               app().executor().post( priority::low, exec_queue::general, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
                   auto start = fc::time_point::now();
                   auto idle_time = start - self->_idle_trx_time;
                   self->_time_tracker.add_idle_time( idle_time );
@@ -723,8 +688,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                      self->log_trx_results( trx, nullptr, ex, 0, start, is_transient );
                      next( std::move(ex) );
                      self->_idle_trx_time = fc::time_point::now();
-                     auto dur = self->_idle_trx_time - start;
-                     self->_time_tracker.add_fail_time(dur, is_transient);
+                     self->_time_tracker.add_fail_time(self->_idle_trx_time - start);
                   };
                   try {
                      auto result = future.get();
@@ -745,7 +709,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       bool process_incoming_transaction_async(const transaction_metadata_ptr& trx,
                                               bool api_trx,
                                               bool return_failure_trace,
-                                              const next_function<transaction_trace_ptr>& next) {
+                                              next_function<transaction_trace_ptr> next) {
          bool exhausted = false;
          chain::controller& chain = chain_plug->chain();
          try {
@@ -825,7 +789,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          exhausted
       };
 
-      inline bool should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const;
+      inline bool should_interrupt_start_block( const fc::time_point& deadline ) const;
       start_block_result start_block();
 
       fc::time_point calculate_pending_block_time() const;
@@ -928,12 +892,12 @@ void producer_plugin::set_program_options(
           "Number of worker threads in producer thread pool")
          ("snapshots-dir", bpo::value<bfs::path>()->default_value("snapshots"),
           "the location of the snapshots directory (absolute path or relative to application data dir)")
-         ("read-only-threads", bpo::value<uint32_t>(),
-          "Number of worker threads in read-only execution thread pool")
+         ("read-only-threads", bpo::value<uint16_t>()->default_value(my->_ro_thread_pool_size),
+          "Number of worker threads in read-only transaction execution thread pool")
          ("read-only-write-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_write_window_time_us.count()),
-          "Time in microseconds the write window lasts.")
+          "time in microseconds the write window lasts")
          ("read-only-read-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_read_window_time_us.count()),
-          "Time in microseconds the read window lasts.")
+          "time in microseconds the read window lasts")
          ;
    config_file_options.add(producer_options);
 }
@@ -977,7 +941,6 @@ if( options.count(op_name) ) { \
    } \
 }
 
-using namespace std::chrono_literals;
 void producer_plugin::plugin_initialize(const boost::program_options::variables_map& options)
 { try {
    handle_sighup(); // Sets loggers
@@ -1110,22 +1073,11 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
-   if ( options.count( "read-only-threads" ) ) {
-      my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint32_t>();
-   } else if ( my->_producers.empty() ) {
-      if( options.count( "plugin" ) ) {
-         const auto& v = options.at( "plugin" ).as<std::vector<std::string>>();
-         auto i = std::find_if( v.cbegin(), v.cend(), []( const std::string& p ) { return p == "eosio::chain_api_plugin"; } );
-         if( i != v.cend() ) {
-            // default to 3 threads for non producer nodes running chain_api_plugin if not specified
-            my->_ro_thread_pool_size = 3;
-            ilog( "chain_api_plugin configured, defaulting read-only-threads to ${t}", ("t", my->_ro_thread_pool_size) );
-         }
-      }
-   }
-   EOS_ASSERT( test_mode_ || my->_ro_thread_pool_size == 0 || my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
+   my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint16_t>();
    // only initialize other read-only options when read-only thread pool is enabled
    if ( my->_ro_thread_pool_size > 0 ) {
+      EOS_ASSERT( my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
+
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if (chain.is_eos_vm_oc_enabled()) {
          // EOS VM OC requires 4.2TB Virtual for each executing thread. Make sure the memory
@@ -1146,36 +1098,36 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
          }
 
          EOS_ASSERT( vm_total_kb > 0, plugin_config_exception, "Unable to get system virtual memory size (not a Linux?), therefore cannot determine if the system has enough virtual memory for multi-threaded read-only transactions on EOS VM OC");
-         EOS_ASSERT( vm_total_kb > vm_used_kb, plugin_config_exception, "vm total (${t}) must be greater than vm used (${u})", ("t", vm_total_kb)("u", vm_used_kb));
-         uint32_t num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000;
          // reserve 1 for the app thread, 1 for anything else which might use VM
-         EOS_ASSERT( num_threads_supported > 2, plugin_config_exception, "With the EOS VM OC configured, there is not enough system virtual memory to support the required minimum of 3 threads (1 for main thread, 1 for read-only, and 1 for anything else), vm total: ${t}, vm used: ${u}", ("t", vm_total_kb)("u", vm_used_kb));
-         num_threads_supported -= 2;
-         auto actual_threads_allowed = std::min(my->_ro_max_eos_vm_oc_threads_allowed, num_threads_supported);
-         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}, max allowed: ${max}, actual allowed: ${actual}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported) ("max", my->_ro_max_eos_vm_oc_threads_allowed)("actual", actual_threads_allowed));
-         EOS_ASSERT( my->_ro_thread_pool_size <= actual_threads_allowed, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads allowed for EOS VM OC (${allowed})", ("th", my->_ro_thread_pool_size) ("allowed", actual_threads_allowed) );
+         int num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000 - 2;
+         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported));
+         EOS_ASSERT( num_threads_supported >= my->_ro_thread_pool_size, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads supported for EOS VM OC (${supp}) by the system virtual memory size", ("th", my->_ro_thread_pool_size) ("supp", num_threads_supported) );
+
+         if ( my->_ro_thread_pool_size > my->_ro_max_eos_vm_oc_threads_allowed ) {
+            wlog("--read-only-threads (${th}) greater than maximum number of threads allowed (${allowed}) for EOS Vm OC. Set it to ${allowed}", ("th", my->_ro_thread_pool_size) ("allowed", my->_ro_max_eos_vm_oc_threads_allowed));
+            my->_ro_thread_pool_size = my->_ro_max_eos_vm_oc_threads_allowed;
+         }
       }
 #endif
-   }
 
-   my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
-   my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
-   EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
-   my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
+      my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
+      my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
+      EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
+      my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
 
-   // Make sure a read-only transaction can finish within the read
-   // window if scheduled at the very beginning of the window.
-   // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
-   // for safety margin
-   if ( my->_max_transaction_time_ms.load() > 0 ) {
-      EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
-      my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
-   } else {
-      // _max_transaction_time_ms can be set to negative in testing (for unlimited)
-      my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
+      // Make sure a read-only transaction can finish within the read
+      // window if scheduled at the very beginning of the window.
+      // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
+      // for safety marging
+      if ( my->_max_transaction_time_ms.load() > 0 ) {
+         EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
+         my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
+      } else {
+         // _max_transaction_time_ms can be set to negative in testing (for unlimited)
+         my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
+      }
+      ilog("_ro_thread_pool_size ${s},  _ro_write_window_time_us ${ww}, _ro_read_window_time_us ${rw}, _ro_max_trx_time_us ${t}, _ro_read_window_effective_time_us ${w}", ("s", my->_ro_thread_pool_size) ("ww", my->_ro_write_window_time_us) ("rw", my->_ro_read_window_time_us) ("t", my->_ro_max_trx_time_us) ("w", my->_ro_read_window_effective_time_us));
    }
-   ilog("ro_thread_pool_size ${s}, ro_write_window_time_us ${ww}, ro_read_window_time_us ${rw}, ro_max_trx_time_us ${t}, ro_read_window_effective_time_us ${w}",
-        ("s", my->_ro_thread_pool_size)("ww", my->_ro_write_window_time_us)("rw", my->_ro_read_window_time_us)("t", my->_ro_max_trx_time_us)("w", my->_ro_read_window_effective_time_us));
 
    my->_incoming_block_sync_provider = app().get_method<incoming::methods::block_sync>().register_provider(
          [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
@@ -1222,6 +1174,7 @@ void producer_plugin::plugin_startup()
       app().quit();
    } );
 
+
    chain::controller& chain = my->chain_plug->chain();
    EOS_ASSERT( my->_producers.empty() || chain.get_read_mode() != chain::db_read_mode::IRREVERSIBLE, plugin_config_exception,
               "node cannot have any producer-name configured because block production is impossible when read_mode is \"irreversible\"" );
@@ -1256,27 +1209,15 @@ void producer_plugin::plugin_startup()
    }
 
    if ( my->_ro_thread_pool_size > 0 ) {
-      std::atomic<uint32_t> num_threads_started = 0;
       my->_ro_thread_pool.start( my->_ro_thread_pool_size,
          []( const fc::exception& e ) {
-            fc_elog( _log, "Exception in read-only thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
+            fc_elog( _log, "Exception in read-only transaction thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
             app().quit();
          },
          [&]() {
             chain.init_thread_local_data();
-            ++num_threads_started;
          });
 
-      // This will be changed with std::latch or std::atomic<>::wait
-      // when C++20 is used.
-      auto time_slept_ms = 0;
-      constexpr auto max_time_slept_ms = 1000;
-      while ( num_threads_started.load() < my->_ro_thread_pool_size && time_slept_ms < max_time_slept_ms ) {
-         std::this_thread::sleep_for( 1ms );
-         ++time_slept_ms;
-      }
-      EOS_ASSERT(num_threads_started.load() == my->_ro_thread_pool_size, producer_exception, "read-only threads failed to start. num_threads_started: ${n}, time_slept_ms: ${t}ms", ("n", num_threads_started.load())("t", time_slept_ms));
-
       my->start_write_window();
    }
 
@@ -1836,12 +1777,12 @@ fc::time_point producer_plugin_impl::calculate_block_deadline( const fc::time_po
    }
 }
 
-bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const {
+bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline ) const {
    if( _pending_block_mode == pending_block_mode::producing ) {
       return deadline <= fc::time_point::now();
    }
    // if we can produce then honor deadline so production starts on time
-   return (!_producers.empty() && deadline <= fc::time_point::now()) || (_received_block >= pending_block_num);
+   return (!_producers.empty() && deadline <= fc::time_point::now()) || _received_block;
 }
 
 producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
@@ -1861,7 +1802,6 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
    const fc::time_point now = fc::time_point::now();
    const fc::time_point block_time = calculate_pending_block_time();
-   const uint32_t pending_block_num = hbs->block_num + 1;
    const fc::time_point preprocess_deadline = calculate_block_deadline(block_time);
 
    const pending_block_mode previous_pending_mode = _pending_block_mode;
@@ -1927,7 +1867,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    if (_pending_block_mode == pending_block_mode::producing) {
       const auto start_block_time = block_time - fc::microseconds( config::block_interval_us );
       if( now < start_block_time ) {
-         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", pending_block_num)("bt", block_time) );
+         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", hbs->block_num + 1)("bt", block_time) );
          // start_block_time instead of block_time because schedule_delayed_production_loop calculates next block time from given time
          schedule_delayed_production_loop(weak_from_this(), calculate_producer_wake_up_time(start_block_time));
          return start_block_result::waiting_for_production;
@@ -1941,7 +1881,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    }
 
    fc_dlog(_log, "Starting block #${n} at ${time} producer ${p}",
-           ("n", pending_block_num)("time", now)("p", scheduled_producer.producer_name));
+           ("n", hbs->block_num + 1)("time", now)("p", scheduled_producer.producer_name));
 
    try {
       uint16_t blocks_to_confirm = 0;
@@ -2005,7 +1945,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             std::swap( features_to_activate, protocol_features_to_activate );
             _protocol_features_signaled = true;
             ilog( "signaling activation of the following protocol features in block ${num}: ${features_to_activate}",
-                  ("num", pending_block_num)("features_to_activate", features_to_activate) );
+                  ("num", hbs->block_num + 1)("features_to_activate", features_to_activate) );
          }
       }
 
@@ -2031,7 +1971,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
          if( !remove_expired_blacklisted_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
          if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(),
-                                                  [&](){ return should_interrupt_start_block( preprocess_deadline, pending_block_num ); } ) ) {
+                                                  [&](){ return should_interrupt_start_block( preprocess_deadline ); } ) ) {
             return start_block_result::exhausted;
          }
 
@@ -2054,11 +1994,9 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             process_scheduled_and_incoming_trxs( scheduled_trx_deadline, incoming_itr );
          }
 
-         repost_exhausted_transactions( preprocess_deadline );
-
          if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit()
             return start_block_result::failed;
-         if ( should_interrupt_start_block( preprocess_deadline, pending_block_num ) || block_is_exhausted() ) {
+         if ( should_interrupt_start_block( preprocess_deadline ) || block_is_exhausted() ) {
             return start_block_result::exhausted;
          }
 
@@ -2085,12 +2023,11 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
 {
    chain::controller& chain = chain_plug->chain();
    auto pending_block_time = chain.pending_block_time();
-   auto pending_block_num = chain.pending_block_num();
 
    // remove all expired transactions
    size_t num_expired = 0;
    size_t orig_count = _unapplied_transactions.size();
-   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline, pending_block_num); },
+   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline); },
          [&num_expired]( const packed_transaction_ptr& packed_trx_ptr, trx_enum_type trx_type ) {
             // expired exception is logged as part of next() call
             ++num_expired;
@@ -2114,13 +2051,12 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point
    if(!blacklist_by_expiry.empty()) {
       const chain::controller& chain = chain_plug->chain();
       const auto lib_time = chain.last_irreversible_block_time();
-      const auto pending_block_num = chain.pending_block_num();
 
       int num_expired = 0;
       int orig_count = _blacklisted_transactions.size();
 
       while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= lib_time) {
-         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2259,10 +2195,9 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
                                         const transaction_metadata_ptr& trx,
                                         bool api_trx,
                                         bool return_failure_trace,
-                                        const next_function<transaction_trace_ptr>& next )
+                                        next_function<transaction_trace_ptr> next )
 {
    auto start = fc::time_point::now();
-   EOS_ASSERT(!trx->is_read_only(), producer_exception, "Unexpected read-only trx");
 
    bool disable_subjective_enforcement = (api_trx && _disable_subjective_api_billing)
                                          || (!api_trx && _disable_subjective_p2p_billing)
@@ -2279,7 +2214,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
          log_trx_results( trx, except_ptr );
          next( except_ptr );
       }
-      _time_tracker.add_fail_time(fc::time_point::now() - start, trx->is_transient());
+      _time_tracker.add_fail_time(fc::time_point::now() - start);
       return push_result{.failed = true};
    }
 
@@ -2299,6 +2234,14 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
       }
    }
 
+   // Make sure db_read_only_mode always to be unset
+   auto db_read_only_mode_guard = fc::make_scoped_exit([trx, &chain]{
+      if( trx->is_read_only() )
+         chain.unset_db_read_only_mode();
+   });
+   if( trx->is_read_only() )
+      chain.set_db_read_only_mode();
+
    auto trace = chain.push_transaction( trx, block_deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );
 
    return handle_push_result(trx, next, start, chain, trace, return_failure_trace, disable_subjective_enforcement, first_auth, sub_bill, prev_billed_cpu_time_us);
@@ -2306,7 +2249,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
 
 producer_plugin_impl::push_result
 producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
-                                          const next_function<transaction_trace_ptr>& next,
+                                          next_function<transaction_trace_ptr> next,
                                           const fc::time_point& start,
                                           const chain::controller& chain,
                                           const transaction_trace_ptr& trace,
@@ -2318,14 +2261,8 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    auto end = fc::time_point::now();
    push_result pr;
    if( trace->except ) {
-      // Transient trxs are dry-run or read-only.
-      // Dry-run trxs only run in write window. Read-only trxs can run in
-      // both write and read windows; time spent in read window is counted
-      // by read window summary.
-      if ( app().executor().is_write_window() ) {
-         auto dur = end - start;
-         _time_tracker.add_fail_time(dur, trx->is_transient());
-      }
+      if ( !trx->is_read_only() )
+         _time_tracker.add_fail_time(end - start);
       if( exception_is_exhausted( *trace->except ) ) {
          if( _pending_block_mode == pending_block_mode::producing ) {
             fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} COULD NOT FIT, tx: ${txid} RETRYING ",
@@ -2365,14 +2302,8 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    } else {
       fc_tlog( _log, "Subjective bill for success ${a}: ${b} elapsed ${t}us, time ${r}us",
                ("a",first_auth)("b",sub_bill)("t",trace->elapsed)("r", end - start));
-      // Transient trxs are dry-run or read-only.
-      // Dry-run trxs only run in write window. Read-only trxs can run in
-      // both write and read windows; time spent in read window is counted
-      // by read window summary.
-      if ( app().executor().is_write_window() ) {
-         auto dur = end - start;
-         _time_tracker.add_success_time(dur, trx->is_transient());
-      }
+      if ( !trx->is_read_only() )
+         _time_tracker.add_success_time(end - start);
       log_trx_results( trx, trace, start );
       // if producing then trx is in objective cpu account billing
       if (!disable_subjective_enforcement && _pending_block_mode != pending_block_mode::producing) {
@@ -2388,14 +2319,12 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
 {
    bool exhausted = false;
    if( !_unapplied_transactions.empty() ) {
-      const chain::controller& chain = chain_plug->chain();
-      const auto pending_block_num = chain.pending_block_num();
       int num_applied = 0, num_failed = 0, num_processed = 0;
       auto unapplied_trxs_size = _unapplied_transactions.size();
       auto itr     = _unapplied_transactions.unapplied_begin();
       auto end_itr = _unapplied_transactions.unapplied_end();
       while( itr != end_itr ) {
-         if( should_interrupt_start_block( deadline, pending_block_num ) ) {
+         if( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2517,7 +2446,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
          auto trace = chain.push_scheduled_transaction(trx_id, deadline, max_trx_time, 0, false);
          auto end = fc::time_point::now();
          if (trace->except) {
-            _time_tracker.add_fail_time(end - start, false); // delayed transaction cannot be transient
+            _time_tracker.add_fail_time(end - start);
             if (exception_is_exhausted(*trace->except)) {
                if( block_is_exhausted() ) {
                   exhausted = true;
@@ -2537,7 +2466,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
                num_failed++;
             }
          } else {
-            _time_tracker.add_success_time(end - start, false); // delayed transaction cannot be transient
+            _time_tracker.add_success_time(end - start);
             fc_dlog(_trx_successful_trace_log,
                     "[TRX_TRACE] Block ${block_num} for producer ${prod} is ACCEPTING scheduled tx: ${txid}, time: ${r}, auth: ${a}, cpu: ${cpu}",
                     ("block_num", chain.head_block_num() + 1)("prod", get_pending_block_producer())
@@ -2570,10 +2499,8 @@ bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline
    if( itr != end ) {
       size_t processed = 0;
       fc_dlog( _log, "Processing ${n} pending transactions", ("n", _unapplied_transactions.incoming_size()) );
-      const chain::controller& chain = chain_plug->chain();
-      const auto pending_block_num = chain.pending_block_num();
       while( itr != end ) {
-         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2616,6 +2543,7 @@ bool producer_plugin_impl::block_is_exhausted() const {
 // -> Idle
 // --> Start block B (block time y.000) at time x.500
 void producer_plugin_impl::schedule_production_loop() {
+   _received_block = false;
    _timer.cancel();
 
    auto result = start_block();
@@ -2627,7 +2555,7 @@ void producer_plugin_impl::schedule_production_loop() {
       _timer.expires_from_now( boost::posix_time::microseconds( config::block_interval_us  / 10 ));
 
       // we failed to start a block, so try again later?
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
           [weak_this = weak_from_this(), cid = ++_timer_corelation_id]( const boost::system::error_code& ec ) {
              auto self = weak_this.lock();
              if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2680,7 +2608,7 @@ void producer_plugin_impl::schedule_maybe_produce_block( bool exhausted ) {
                ("num", chain.head_block_num() + 1)("desc", block_is_exhausted() ? "Exhausted" : "Deadline exceeded") );
    }
 
-   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
+   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
          [&chain, weak_this = weak_from_this(), cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2722,7 +2650,7 @@ void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr<
       fc_dlog(_log, "Scheduling Speculative/Production Change at ${time}", ("time", wake_up_time));
       static const boost::posix_time::ptime epoch(boost::gregorian::date(1970, 1, 1));
       _timer.expires_at(epoch + boost::posix_time::microseconds(wake_up_time->time_since_epoch().count()));
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
          [weak_this,cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2823,8 +2751,8 @@ void producer_plugin_impl::produce_block() {
         ("confs", new_bs->header.confirmed));
 }
 
-void producer_plugin::received_block(uint32_t block_num) {
-   my->_received_block = block_num;
+void producer_plugin::received_block() {
+   my->_received_block = true;
 }
 
 void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id, const packed_transaction_ptr& packed_trx_ptr, const char* reason) const {
@@ -2836,42 +2764,34 @@ void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id,
             ("entire_trx", packed_trx_ptr ? my->chain_plug->get_log_trx(packed_trx_ptr->get_transaction()) : fc::variant{trx_id}));
 }
 
-// Called from only one read_only thread
+// Called from app thread
 void producer_plugin_impl::switch_to_write_window() {
-   if ( _log.is_enabled( fc::log_level::debug ) ) {
-      auto now = fc::time_point::now();
-      fc_dlog( _log, "Read-only threads ${n}, read window ${r}us, total all threads ${t}us",
-               ("n", _ro_thread_pool_size)
-               ("r", now - _ro_read_window_start_time)
-               ("t", _ro_all_threads_exec_time_us.load()));
-   }
-
    // this method can be called from multiple places. it is possible
    // we are already in write window.
    if ( app().executor().is_write_window() ) {
       return;
    }
 
-   EOS_ASSERT(_ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
+   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
+   _ro_read_window_timer.cancel();
+   _ro_write_window_timer.cancel();
 
    start_write_window();
 }
 
-// Called from app thread on plugin_startup
-// Called from only one read_only thread & called from app thread, but not concurrently
+// Called from app thread
 void producer_plugin_impl::start_write_window() {
    chain::controller& chain = chain_plug->chain();
 
    app().executor().set_to_write_window();
    chain.unset_db_read_only_mode();
-   _idle_trx_time = _ro_window_deadline = fc::time_point::now();
+   _idle_trx_time = fc::time_point::now();
 
-   _ro_window_deadline += _ro_write_window_time_us; // not allowed on block producers, so no need to limit to block deadline
    auto expire_time = boost::posix_time::microseconds(_ro_write_window_time_us.count());
-   _ro_timer.expires_from_now( expire_time );
-   _ro_timer.async_wait( app().executor().wrap(  // stay on app thread
+   _ro_write_window_timer.expires_from_now( expire_time );
+   _ro_write_window_timer.async_wait( app().executor().wrap(  // stay on app thread
       priority::high,
-      exec_queue::read_write, // placed in read_write so only called from main thread
+      exec_queue::read_only_trx_safe, // placed in read_only_trx_safe queue so it is ensured to be executed in either window
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
@@ -2880,154 +2800,144 @@ void producer_plugin_impl::start_write_window() {
       }));
 }
 
-// Called only from app thread
+// Called from app thread
 void producer_plugin_impl::switch_to_read_window() {
    EOS_ASSERT(app().executor().is_write_window(),  producer_exception, "expected to be in write window");
-   EOS_ASSERT( _ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "_ro_exec_tasks_fut expected to be empty" );
+   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(),  producer_exception, "_ro_trx_exec_tasks_fut expected to be empty" );
 
+   _ro_write_window_timer.cancel();
+   _ro_read_window_timer.cancel();
    _time_tracker.add_idle_time( fc::time_point::now() - _idle_trx_time );
 
    // we are in write window, so no read-only trx threads are processing transactions.
-   if ( app().executor().read_only_queue().empty() ) { // no read-only tasks to process. stay in write window
+   // _ro_trx_queue is not being accessed. No need to lock.
+   if ( _ro_trx_queue.queue.empty() ) { // no read-only trxs to process. stay in write window
       start_write_window(); // restart write window timer for next round
       return;
    }
 
+   app().executor().set_to_read_window();
+   chain_plug->chain().set_db_read_only_mode();
+   _received_block = false;
 
-   auto& chain = chain_plug->chain();
-   uint32_t pending_block_num = chain.head_block_num() + 1;
-   _ro_read_window_start_time = fc::time_point::now();
-   _ro_window_deadline = _ro_read_window_start_time + _ro_read_window_effective_time_us;
-   app().executor().set_to_read_window(_ro_thread_pool_size,
-      [received_block=&_received_block, pending_block_num, ro_window_deadline=_ro_window_deadline]() {
-         return fc::time_point::now() >= ro_window_deadline || (received_block->load() >= pending_block_num); // should_exit()
-      });
-   chain.set_db_read_only_mode();
-   _ro_all_threads_exec_time_us = 0;
-
-   // start a read-only execution task in each thread in the thread pool
-   _ro_num_active_exec_tasks = _ro_thread_pool_size;
-   _ro_exec_tasks_fut.resize(0);
-   for (uint32_t i = 0; i < _ro_thread_pool_size; ++i ) {
-      _ro_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this, pending_block_num] () {
-         return self->read_only_execution_task(pending_block_num);
+   // start a read-only transaction execution task in each thread in the thread pool
+   _ro_num_active_trx_exec_tasks = _ro_thread_pool_size;
+   for (auto i = 0; i < _ro_thread_pool_size; ++i ) {
+      _ro_trx_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this] () {
+         return self->read_only_trx_execution_task();
       }) );
    }
 
    auto expire_time = boost::posix_time::microseconds(_ro_read_window_time_us.count());
-   _ro_timer.expires_from_now( expire_time );
-   // Needs to be on read_only because that is what is being processed until switch_to_write_window().
-   _ro_timer.async_wait( app().executor().wrap(
+   _ro_read_window_timer.expires_from_now( expire_time );
+   _ro_read_window_timer.async_wait( app().executor().wrap(  // stay on app thread
       priority::high,
-      exec_queue::read_only,
+      exec_queue::read_only_trx_safe,
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
             // use future to make sure all read-only tasks finished before switching to write window
-            for ( auto& task: self->_ro_exec_tasks_fut ) {
+            for ( auto& task: self->_ro_trx_exec_tasks_fut ) {
                task.get();
             }
-            self->_ro_exec_tasks_fut.clear();
-            // will be executed from the main app thread because all read-only threads are idle now
+            self->_ro_trx_exec_tasks_fut.clear();
             self->switch_to_write_window();
           } else if ( self ) {
-             self->_ro_exec_tasks_fut.clear();
+             self->_ro_trx_exec_tasks_fut.clear();
           }
        }));
 }
 
-// Called from a read only thread. Run in parallel with app and other read only threads
-bool producer_plugin_impl::read_only_execution_task(uint32_t pending_block_num) {
-   // We have 3 ways to break out the while loop:
+// Called from a read only trx thread. Run in parallel with app and other read only trx threads
+bool producer_plugin_impl::read_only_trx_execution_task() {
+   auto start = fc::time_point::now();
+   auto read_window_deadline = start + _ro_read_window_effective_time_us;
+   // We have 4 ways to break out the while loop:
    // 1. pass read window deadline
-   // 2. net_plugin receives a block
-   // 3. no read-only tasks to execute
-   while ( fc::time_point::now() < _ro_window_deadline && _received_block < pending_block_num ) {
-      bool more = app().executor().execute_highest_read_only(); // blocks until all read only threads are idle
-      if ( !more ) {
+   // 2. Net_plugin receives a block
+   // 3. No more transactions in the read-only trx queue
+   // 4. A transaction execution is exhaused
+   while ( fc::time_point::now() < read_window_deadline && !_received_block ) {
+      std::unique_lock<std::mutex> lck( _ro_trx_queue.mtx );
+      if ( _ro_trx_queue.queue.empty() ) {
+         break;
+      }
+      auto trx = _ro_trx_queue.queue.front();
+      _ro_trx_queue.queue.pop_front();
+      lck.unlock();
+      
+      auto retry = process_read_only_transaction( trx.trx, trx.next, start );
+      if ( retry ) {
+         lck.lock();
+         _ro_trx_queue.queue.push_front(trx);
+         // Do not schedule new execution
          break;
       }
    }
 
    // If all tasks are finished, do not wait until end of read window; switch to write window now.
-   if ( --_ro_num_active_exec_tasks == 0 ) {
-      // Needs to be on read_only because that is what is being processed until switch_to_write_window().
-      app().executor().post( priority::high, exec_queue::read_only, [self=this]() {
-         self->_ro_exec_tasks_fut.clear();
-         // will be executed from the main app thread because all read-only threads are idle now
+   if ( --_ro_num_active_trx_exec_tasks == 0 ) {
+      // Do switching on app thread to serialize
+      app().executor().post( priority::high, exec_queue::read_only_trx_safe, [self=this]() {
+         self->_ro_trx_exec_tasks_fut.clear();
          self->switch_to_write_window();
       } );
-      // last thread post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
-      ro_trx_t t;
-      while( _ro_exhausted_trx_queue.pop_front(t) ) {
-         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
-            push_read_only_transaction( std::move(trx), std::move(next) );
-         } );
-      }
    }
 
    return true;
 }
 
-// Called from app thread during start block.
-// Reschedule any exhausted read-only transactions from the last block
-void producer_plugin_impl::repost_exhausted_transactions(const fc::time_point& deadline) {
-   if ( !_ro_exhausted_trx_queue.empty() ) {
-      chain::controller& chain = chain_plug->chain();
-      uint32_t pending_block_num = chain.pending_block_num();
-      // post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
-      ro_trx_t t;
-      while( !should_interrupt_start_block( deadline, pending_block_num ) && _ro_exhausted_trx_queue.pop_front(t) ) {
-         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
-            push_read_only_transaction( std::move(trx), std::move(next) );
-         } );
-      }
-   }
+// Called from a read only trx thread. Run in parallel with app and other read only trx threads
+// Return whether the trx needs to be retried in next read window
+bool producer_plugin_impl::process_read_only_transaction(const packed_transaction_ptr& trx, const next_function<transaction_trace_ptr>& next, const fc::time_point& read_window_start_time) {
+   chain::controller& chain = chain_plug->chain();
+   auto future = transaction_metadata::start_recover_keys( trx, _thread_pool.get_executor(),
+                                                           chain.get_chain_id(), fc::microseconds::maximum(),
+                                                           transaction_metadata::trx_type::read_only,
+                                                           chain.configured_subjective_signature_length_limit() );
+   auto exception_handler = [&next](fc::exception_ptr ex) {
+      next( std::move(ex) );
+      return false;
+   };
+   future.wait();
+   try {
+      auto trx_metadata = future.get();
+      return push_read_only_transaction( trx_metadata, next, read_window_start_time );
+   } CATCH_AND_CALL(exception_handler);
+   return false;
 }
 
-// Called from a read_only_trx execution thread, or from app thread when executing exclusively
+// Called from a read_only_trx execution thread
 // Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next) {
+bool producer_plugin_impl::push_read_only_transaction(
+            const transaction_metadata_ptr& trx,
+            next_function<transaction_trace_ptr> next,
+            const fc::time_point& read_window_start_time) {
    auto retry = false;
+   chain::controller& chain = chain_plug->chain();
+
+   if( !chain.is_building_block() ) {
+      // try next round
+      return true;
+   }
 
    try {
+      const auto block_deadline = calculate_block_deadline( chain.pending_block_time() );
       auto start = fc::time_point::now();
-      chain::controller& chain = chain_plug->chain();
-      if ( !chain.is_building_block() ) {
-         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
+      auto time_used_in_read_window_us = ( start - read_window_start_time );
+      if ( time_used_in_read_window_us >= _ro_read_window_time_us ) {
+         // already passed read-window deadline, try next round
          return true;
       }
 
-      // When executing a read-only trx on the main thread while in the write window,
-      // need to switch db mode to read only.
-      auto db_read_only_mode_guard = fc::make_scoped_exit([&]{
-         if( app().executor().is_write_window() )
-            chain.unset_db_read_only_mode();
-      });
-
-      if ( app().executor().is_write_window() ) {
-         chain.set_db_read_only_mode();
-         auto idle_time = fc::time_point::now() - _idle_trx_time;
-         _time_tracker.add_idle_time( idle_time );
-      }
-
-      // use read-window/write-window deadline if there are read/write windows, otherwise use block_deadline if only the app thead
-      auto window_deadline = (_ro_thread_pool_size != 0) ? _ro_window_deadline : calculate_block_deadline( chain.pending_block_time() );
-
-      // Ensure the trx to finish by the end of read-window or write-window or block_deadline depending on
+      auto remaining_time_in_read_window_us = _ro_read_window_time_us - time_used_in_read_window_us;
+      // Ensure the trx to finish by the end of read-window.
+      auto window_deadline = std::min( start + remaining_time_in_read_window_us, block_deadline );
       auto trace = chain.push_transaction( trx, window_deadline, _ro_max_trx_time_us, 0, false, 0 );
-      _ro_all_threads_exec_time_us += (fc::time_point::now() - start).count();
       auto pr = handle_push_result(trx, next, start, chain, trace, true /*return_failure_trace*/, true /*disable_subjective_enforcement*/, {} /*first_auth*/, 0 /*sub_bill*/, 0 /*prev_billed_cpu_time_us*/);
       // If a transaction was exhausted, that indicates we are close to
       // the end of read window. Retry in next round.
       retry = pr.trx_exhausted;
-      if( retry ) {
-         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
-      }
-
-      if ( app().executor().is_write_window() ) {
-         _idle_trx_time = fc::time_point::now();
-      }
    } catch ( const guard_exception& e ) {
       chain_plugin::handle_guard_exception(e);
    } catch ( boost::interprocess::bad_alloc& ) {

plugins/producer_plugin/test/test_read_only_trx.cpp

@@ -79,7 +79,6 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    appbase::scoped_app app;
    fc::temp_directory temp;
    auto temp_dir_str = temp.path().string();
-   producer_plugin::set_test_mode(true);
    
    std::promise<std::tuple<producer_plugin*, chain_plugin*>> plugin_promise;
    std::future<std::tuple<producer_plugin*, chain_plugin*>> plugin_fut = plugin_promise.get_future();
@@ -97,19 +96,14 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    auto chain_id = chain_plug->get_chain_id();
 
    std::atomic<size_t> next_calls = 0;
-   std::atomic<size_t> num_get_account_calls = 0;
    std::atomic<size_t> num_posts = 0;
    std::atomic<size_t> trace_with_except = 0;
    std::atomic<bool> trx_match = true;
-   const size_t num_pushes = 4242;
+   const size_t num_pushes = 2558;
 
    for( size_t i = 1; i <= num_pushes; ++i ) {
       auto ptrx = make_unique_trx( chain_id );
-      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug, &num_get_account_calls]() {
-         chain_plug->get_read_only_api(fc::seconds(90)).get_account(chain_apis::read_only::get_account_params{.account_name=config::system_account_name}, fc::time_point::now()+fc::seconds(90));
-         ++num_get_account_calls;
-      });
-      app->executor().post( priority::low, exec_queue::read_write, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
+      app->executor().post( priority::low, exec_queue::general, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
          ++num_posts;
          bool return_failure_traces = true;
          app->get_method<plugin_interface::incoming::methods::transaction_async>()(ptrx,
@@ -132,15 +126,12 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
                ++next_calls;
         });
       });
-      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug]() {
-         chain_plug->get_read_only_api(fc::seconds(90)).get_consensus_parameters(chain_apis::read_only::get_consensus_parameters_params{}, fc::time_point::now()+fc::seconds(90));
-      });
    }
 
    // Wait long enough such that all transactions are executed
    auto start = fc::time_point::now();
    auto hard_deadline = start + fc::seconds(10); // To protect against waiting forever
-   while ( (next_calls < num_pushes || num_get_account_calls < num_pushes) && fc::time_point::now() < hard_deadline ){
+   while ( next_calls < num_pushes && fc::time_point::now() < hard_deadline ){
       std::this_thread::sleep_for( 100ms );;
    }
 
@@ -150,39 +141,13 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    BOOST_CHECK_EQUAL( trace_with_except, 0 ); // should not have any traces with except in it
    BOOST_CHECK_EQUAL( num_pushes, num_posts );
    BOOST_CHECK_EQUAL( num_pushes, next_calls.load() );
-   BOOST_CHECK_EQUAL( num_pushes, num_get_account_calls.load() );
-   BOOST_CHECK( trx_match.load() );  // trace should match the transaction
+   BOOST_CHECK( trx_match.load() );  // trace should match the transaction  
 }
 
 // test read-only trxs on main thread (no --read-only-threads)
 BOOST_AUTO_TEST_CASE(no_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e", "--abi-serializer-max-time-ms=999" };
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e" };
    test_trxs_common(specific_args);
 }
 
-// test read-only trxs on 1 threads (with --read-only-threads)
-BOOST_AUTO_TEST_CASE(with_1_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
-                                              "--read-only-threads=1",
-                                              "--max-transaction-time=10",
-                                              "--abi-serializer-max-time-ms=999",
-                                              "--read-only-write-window-time-us=100000",
-                                              "--read-only-read-window-time-us=40000",
-                                              "--disable-subjective-billing=true" };
-   test_trxs_common(specific_args);
-}
-
-// test read-only trxs on 16 separate threads (with --read-only-threads)
-BOOST_AUTO_TEST_CASE(with_16_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
-                                              "--read-only-threads=16",
-                                              "--max-transaction-time=10",
-                                              "--abi-serializer-max-time-ms=999",
-                                              "--read-only-write-window-time-us=100000",
-                                              "--read-only-read-window-time-us=40000",
-                                              "--disable-subjective-billing=true" };
-   test_trxs_common(specific_args);
-}
-
-
 BOOST_AUTO_TEST_SUITE_END()

plugins/producer_plugin/test/test_snapshot_scheduler.cpp

@@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
             fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
             std::vector<const char*> argv =
                   {"test", "--data-dir", temp.c_str(), "--config-dir", temp.c_str(),
-                   "-p", "eosio", "-e", "--disable-subjective-billing=true"};
+                   "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true"};
             appbase::app().initialize<chain_plugin, producer_plugin>(argv.size(), (char**) &argv[0]);
             appbase::app().startup();
             plugin_promise.set_value(

plugins/producer_plugin/test/test_trx_full.cpp

@@ -112,7 +112,7 @@ BOOST_AUTO_TEST_CASE(producer) {
          fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
          std::vector<const char*> argv =
                {"test", "--data-dir", temp_dir_str.c_str(), "--config-dir", temp_dir_str.c_str(),
-                "-p", "eosio", "-e", "--disable-subjective-billing=true" };
+                "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true" };
          app->initialize<chain_plugin, producer_plugin>( argv.size(), (char**) &argv[0] );
          app->startup();
          plugin_promise.set_value(

plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp

@@ -103,7 +103,7 @@ public:
    void pop_entry(bool call_send = true) {
       send_queue.erase(send_queue.begin());
       sending = false;
-      if (call_send || !send_queue.empty())
+      if (call_send)
          send();
    }
 

plugins/test_control_api_plugin/test_control_api_plugin.cpp

@@ -51,7 +51,7 @@ void test_control_api_plugin::plugin_startup() {
 
    app().get_plugin<http_plugin>().add_api({
       TEST_CONTROL_RW_CALL(kill_node_on_producer, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_write);
+   }, appbase::exec_queue::general);
 }
 
 void test_control_api_plugin::plugin_shutdown() {}

plugins/wallet_api_plugin/wallet_api_plugin.cpp

@@ -115,7 +115,7 @@ void wallet_api_plugin::plugin_startup() {
             INVOKE_R_R_R(wallet_mgr, list_keys, std::string, std::string), 200),
        CALL_WITH_400(wallet, wallet_mgr, get_public_keys,
             INVOKE_R_V(wallet_mgr, get_public_keys), 200)
-   }, appbase::exec_queue::read_write);
+   }, appbase::exec_queue::general);
 }
 
 void wallet_api_plugin::plugin_initialize(const variables_map& options) {

programs/keosd/main.cpp

@@ -111,7 +111,7 @@ int main(int argc, char** argv)
                        [&a=app](string, string, url_response_callback cb) {
          cb(200, fc::time_point::maximum(), fc::variant(fc::variant_object()));
          a->quit();
-      }, appbase::exec_queue::read_write );
+      }, appbase::exec_queue::general );
       app->startup();
       app->exec();
    } catch (const fc::exception& e) {

tests/CMakeLists.txt

@@ -125,11 +125,11 @@ add_test(NAME nodeos_protocol_feature_test COMMAND tests/nodeos_protocol_feature
 set_property(TEST nodeos_protocol_feature_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME compute_transaction_test COMMAND tests/compute_transaction_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST compute_transaction_test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 0 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-basic-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-parallel-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST read-only-trx-parallel-eos-vm-oc-test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME subjective_billing_test COMMAND tests/subjective_billing_test.py -v -p 2 -n 4 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST subjective_billing_test PROPERTY LABELS nonparallelizable_tests)

tests/TestHarness/Cluster.py

@@ -22,11 +22,8 @@ from .Node import BlockType
 from .Node import Node
 from .WalletMgr import WalletMgr
 from .launch_transaction_generators import TransactionGeneratorsLauncher, TpsTrxGensConfig
-try:
-    from .libc import unshare, CLONE_NEWNET
-    from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
-except:
-    pass
+from .libc import unshare, CLONE_NEWNET
+from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
 
 # Protocol Feature Setup Policy
 class PFSetupPolicy:
@@ -1450,7 +1447,7 @@ class Cluster(object):
     def killall(self, kill=True, silent=True, allInstances=False):
         """Kill cluster nodeos instances. allInstances will kill all nodeos instances running on the system."""
         signalNum=9 if kill else 15
-        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"{sys.executable} {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
+        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"python3 {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         if 0 != subprocess.call(cmd.split(), stdout=Utils.FNull):
             if not silent: Utils.Print("Launcher failed to shut down eos cluster.")

tests/TestHarness/Node.py

@@ -61,7 +61,7 @@ class Node(Transactions):
             self.fetchBlock = lambda blockNum: self.processUrllibRequest("trace_api", "get_block", {"block_num":blockNum}, silentErrors=False, exitOnError=True)
             self.fetchKeyCommand = lambda: "[transaction][transaction_header][ref_block_num]"
             self.fetchRefBlock = lambda trans: trans["transaction_header"]["ref_block_num"]
-            self.cleosLimit = "--time-limit 999"
+            self.cleosLimit = "--time-limit 99999"
             self.fetchHeadBlock = lambda node, headBlock: node.processUrllibRequest("chain", "get_block_info", {"block_num":headBlock}, silentErrors=False, exitOnError=True)
 
     def __str__(self):

tests/TestHarness/WalletMgr.py

@@ -12,9 +12,9 @@ from .testUtils import Utils
 Wallet=namedtuple("Wallet", "name password host port")
 # pylint: disable=too-many-instance-attributes
 class WalletMgr(object):
+    __walletLogOutFile=f"{Utils.TestLogRoot}/test_keosd_out.log"
+    __walletLogErrFile=f"{Utils.TestLogRoot}/test_keosd_err.log"
     __walletDataDir=f"{Utils.TestLogRoot}/test_wallet_0"
-    __walletLogOutFile=f"{__walletDataDir}/test_keosd_out.log"
-    __walletLogErrFile=f"{__walletDataDir}/test_keosd_err.log"
     __MaxPort=9999
 
     # pylint: disable=too-many-arguments
@@ -83,9 +83,6 @@ class WalletMgr(object):
         cmd="%s --data-dir %s --config-dir %s --unlock-timeout=999999 --http-server-address=%s:%d --http-max-response-time-ms 99999 --verbose-http-errors" % (
             Utils.EosWalletPath, WalletMgr.__walletDataDir, WalletMgr.__walletDataDir, self.host, self.port)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
-        if not os.path.isdir(WalletMgr.__walletDataDir):
-            if Utils.Debug: Utils.Print(f"Creating dir {WalletMgr.__walletDataDir} in dir: {os.getcwd()}")
-            os.mkdir(WalletMgr.__walletDataDir)
         with open(WalletMgr.__walletLogOutFile, 'w') as sout, open(WalletMgr.__walletLogErrFile, 'w') as serr:
             popen=subprocess.Popen(cmd.split(), stdout=sout, stderr=serr)
             self.__walletPid=popen.pid
@@ -302,5 +299,6 @@ class WalletMgr(object):
 
     @staticmethod
     def cleanup():
-        if os.path.isdir(WalletMgr.__walletDataDir) and os.path.exists(WalletMgr.__walletDataDir):
+        dataDir=WalletMgr.__walletDataDir
+        if os.path.isdir(dataDir) and os.path.exists(dataDir):
             shutil.rmtree(WalletMgr.__walletDataDir)

tests/auto_bp_peering_test.py

@@ -35,7 +35,8 @@ args = TestHelper.parse_args({
 Utils.Debug = args.v
 killAll = args.clean_run
 dumpErrorDetails = args.dump_error_details
-dontKill = args.leave_running
+# dontKill = args.leave_running
+dontKill = True
 killEosInstances = not dontKill
 killWallet = not dontKill
 keepLogs = args.keep_logs

tests/cli_test.py

@@ -11,7 +11,7 @@ import time
 import shutil
 import signal
 
-from TestHarness import Account, Node, ReturnType, Utils, WalletMgr
+from TestHarness import Account, Cluster, Node, ReturnType, Utils, WalletMgr
 
 testSuccessful=False
 
@@ -352,7 +352,7 @@ def abi_file_with_nodeos_test():
         os.makedirs(data_dir, exist_ok=True)
         walletMgr = WalletMgr(True)
         walletMgr.launch()
-        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --max-transaction-time=-1 --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
+        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
         node.verifyAlive() # Setting node state to not alive
         node.relaunch(newChain=True, cachePopen=True)
         node.waitForBlock(1)
@@ -403,7 +403,7 @@ def abi_file_with_nodeos_test():
                     os.kill(node.pid, signal.SIGKILL)
         if testSuccessful:
             Utils.Print("Cleanup nodeos data.")
-            shutil.rmtree(Utils.DataPath)
+            shutil.rmtree(data_dir)
 
         if malicious_token_abi_path:
             if os.path.exists(malicious_token_abi_path):

tests/launcher.py

@@ -174,7 +174,7 @@ class launcher(object):
         parser.add_argument('-i', '--timestamp', help='set the timestamp for the first block.  Use "now" to indicate the current time')
         parser.add_argument('-l', '--launch', choices=['all', 'none', 'local'], help='select a subset of nodes to launch.  If not set, the default is to launch all unless an output file is named, in which case none are started.', default='all')
         parser.add_argument('-o', '--output', help='save a copy of the generated topology in this file and exit without launching', dest='topology_filename')
-        parser.add_argument('-k', '--kill', type=int, help='kill a network as specified in arguments with given signal')
+        parser.add_argument('-k', '--kill', help='retrieve the list of previously started process ids and issue a kill to each')
         parser.add_argument('--down', type=comma_separated, help='comma-separated list of node numbers that will be shut down', default=[])
         parser.add_argument('--bounce', type=comma_separated, help='comma-separated list of node numbers that will be restarted', default=[])
         parser.add_argument('--roll', type=comma_separated, help='comma-separated list of host names where the nodes will be rolled to a new version')
@@ -613,17 +613,6 @@ plugin = eosio::chain_api_plugin
                     if relaunch:
                         self.launch(node)
 
-    def kill(self, signum):
-        errorCode = 0
-        for node in self.network.nodes.values():
-            try:
-                with open(node.data_dir_name / f'{Utils.EosServerName}.pid', 'r') as f:
-                    pid = int(f.readline())
-                    self.terminate_wait_pid(pid, signum, raise_if_missing=False)
-            except FileNotFoundError as err:
-                errorCode = 1
-        return errorCode
-
     def start_all(self):
         if self.args.launch.lower() != 'none':
             for instance in self.network.nodes.values():
@@ -639,9 +628,8 @@ plugin = eosio::chain_api_plugin
                     f.write(f'"{node.dot_label}"->"{pname}" [dir="forward"];\n')
             f.write('}')
 
-    def terminate_wait_pid(self, pid, signum = signal.SIGTERM, raise_if_missing=True):
-        '''Terminate a non-child process with given signal number or with SIGTERM if not
-           provided and wait for it to exit.'''
+    def terminate_wait_pid(self, pid, raise_if_missing=True):
+        '''Terminate a non-child process with SIGTERM and wait for it to exit.'''
         if sys.version_info >= (3, 9) and platform.system() == 'Linux': # on our supported platforms, Python 3.9 accompanies a kernel > 5.3
             try:
                 fd = os.pidfd_open(pid)
@@ -652,7 +640,7 @@ plugin = eosio::chain_api_plugin
                 po = select.poll()
                 po.register(fd, select.POLLIN)
                 try:
-                    os.kill(pid, signum)
+                    os.kill(pid, signal.SIGTERM)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -675,7 +663,7 @@ plugin = eosio::chain_api_plugin
                     return min(delay * 2, 0.04)
                 delay = 0.0001
                 try:
-                    os.kill(pid, signum)
+                    os.kill(pid, signal.SIGTERM)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -688,16 +676,12 @@ plugin = eosio::chain_api_plugin
                         return
 
 if __name__ == '__main__':
-    errorCode = 0
     l = launcher(sys.argv[1:])
     if len(l.args.down):
         l.down(l.args.down)
     elif len(l.args.bounce):
         l.bounce(l.args.bounce)
-    elif l.args.kill:
-        errorCode = l.kill(l.args.kill)
     elif l.args.launch == 'all' or l.args.launch == 'local':
         l.start_all()
     for f in glob.glob(Utils.DataPath):
         shutil.rmtree(f)
-    sys.exit(errorCode)

tests/nodeos_contrl_c_test.py

@@ -115,7 +115,7 @@ try:
         errorExit("Failed to kill the seed node")
 
 finally:
-    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=False, cleanRun=True, dumpErrorDetails=True)
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=True, cleanRun=True, dumpErrorDetails=True)
 
 errorCode = 0 if testSuccessful else 1
 exit(errorCode)

tests/nodeos_snapshot_forked_test.py

@@ -171,7 +171,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(60), \
+    assert prodC.waitForLibToAdvance(), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
  
     for prodNode in prodNodes:

tests/performance_tests/performance_test_basic.py

@@ -497,17 +497,15 @@ class PerformanceTestBasic:
             traceback.print_exc()
 
         finally:
-            # Despite keepLogs being hardcoded to False, logs will still appear on test failure in TestLogs
-            # due to testSuccessful being False
             TestHelper.shutdown(
-                cluster=self.cluster,
-                walletMgr=self.walletMgr,
-                testSuccessful=testSuccessful,
-                killEosInstances=self.testHelperConfig._killEosInstances,
-                killWallet=self.testHelperConfig._killWallet,
-                keepLogs=False,
-                cleanRun=self.testHelperConfig.killAll,
-                dumpErrorDetails=self.testHelperConfig.dumpErrorDetails
+                self.cluster,
+                self.walletMgr,
+                testSuccessful,
+                self.testHelperConfig._killEosInstances,
+                self.testHelperConfig._killWallet,
+                self.testHelperConfig.keepLogs,
+                self.testHelperConfig.killAll,
+                self.testHelperConfig.dumpErrorDetails
                 )
 
             if self.ptbConfig.delPerfLogs:

tests/plugin_http_api_test.py

@@ -40,8 +40,8 @@ class PluginHttpTest(unittest.TestCase):
         self.keosd.killall(True)
         WalletMgr.cleanup()
         Node.killAllNodeos()
-        if os.path.exists(Utils.DataPath):
-            shutil.rmtree(Utils.DataPath)
+        if os.path.exists(self.data_dir):
+            shutil.rmtree(self.data_dir)
         if os.path.exists(self.config_dir):
             shutil.rmtree(self.config_dir)
         time.sleep(self.sleep_s)
@@ -56,7 +56,7 @@ class PluginHttpTest(unittest.TestCase):
                         "http_plugin", "db_size_api_plugin", "prometheus_plugin"]
         nodeos_plugins = "--plugin eosio::" +  " --plugin eosio::".join(plugin_names)
         nodeos_flags = (" --data-dir=%s --config-dir=%s --trace-dir=%s --trace-no-abis --access-control-allow-origin=%s "
-                        "--contracts-console --http-validate-host=%s --verbose-http-errors --max-transaction-time -1 --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
+                        "--contracts-console --http-validate-host=%s --verbose-http-errors --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
                         "--p2p-peer-address localhost:9011 --resource-monitor-not-shutdown-on-threshold-exceeded ") % (self.data_dir, self.config_dir, self.data_dir, "\'*\'", "false")
         start_nodeos_cmd = ("%s -e -p eosio %s %s ") % (Utils.EosServerPath, nodeos_plugins, nodeos_flags)
         self.nodeos.launchCmd(start_nodeos_cmd, self.node_id)

tests/read_only_trx_test.py

@@ -21,7 +21,6 @@ errorExit=Utils.errorExit
 
 appArgs=AppArgs()
 appArgs.add(flag="--read-only-threads", type=int, help="number of read-only threads", default=0)
-appArgs.add(flag="--num-test-runs", type=int, help="number of times to run the tests", default=1)
 appArgs.add_bool(flag="--eos-vm-oc-enable", help="enable eos-vm-oc")
 appArgs.add(flag="--wasm-runtime", type=str, help="if set to eos-vm-oc, must compile with EOSIO_EOS_VM_OC_DEVELOPER", default="eos-vm-jit")
 
@@ -45,7 +44,6 @@ dontKill=args.leave_running
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
-numTestRuns=args.num_test_runs
 
 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -93,8 +91,9 @@ try:
     specificExtraNodeosArgs={}
     # producer nodes will be mapped to 0 through pnodes-1, so the number pnodes is the no-producing API node
     specificExtraNodeosArgs[pnodes]=" --plugin eosio::net_api_plugin"
-    specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
-    specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
+    if args.read_only_threads > 0:
+        specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
+        specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
     if args.eos_vm_oc_enable:
         specificExtraNodeosArgs[pnodes]+=" --eos-vm-oc-enable"
     if args.wasm_runtime:
@@ -260,7 +259,7 @@ try:
         runReadOnlyTrxAndRpcInParallel("chain", "get_raw_code_and_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_raw_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_producers", "rows", payload = {"json":"true","lower_bound":""})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", "rows", payload = {"json":"true","code":"eosio","scope":"eosio","table":"global"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", code=500, payload = {"json":"true","table":"noauth"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_table_by_scope", fieldIn="rows", payload = {"json":"true","table":"noauth"})
         runReadOnlyTrxAndRpcInParallel("chain", "get_currency_balance", code=200, payload = {"code":"eosio.token", "account":testAccountName})
         runReadOnlyTrxAndRpcInParallel("chain", "get_currency_stats", fieldIn="SYS", payload = {"code":"eosio.token", "symbol":"SYS"})
@@ -277,26 +276,13 @@ try:
         runReadOnlyTrxAndRpcInParallel("net", "connect", code=201, payload = "localhost")
         runReadOnlyTrxAndRpcInParallel("net", "disconnect", code=201, payload = "localhost")
 
-    def runEverythingParallel():
-        threadList = []
-        threadList.append(threading.Thread(target = multiReadOnlyTests))
-        threadList.append(threading.Thread(target = chainApiTests))
-        threadList.append(threading.Thread(target = netApiTests))
-        threadList.append(threading.Thread(target = mixedOpsTest))
-        for thr in threadList:
-            thr.start()
-        for thr in threadList:
-            thr.join()
-
     basicTests()
 
     if args.read_only_threads > 0: # Save test time. No need to run other tests if multi-threaded is not enabled
-        for i in range(numTestRuns):
-            multiReadOnlyTests()
-            chainApiTests()
-            netApiTests()
-            mixedOpsTest()
-            runEverythingParallel()
+        multiReadOnlyTests()
+        chainApiTests()
+        netApiTests()
+        mixedOpsTest()
 
     testSuccessful = True
 finally:

tests/trace_plugin_test.py

@@ -8,8 +8,6 @@ import os
 
 from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr, CORE_SYMBOL
 
-testSuccessful = True
-
 class TraceApiPluginTest(unittest.TestCase):
     sleep_s = 1
     cluster=Cluster(walletd=True, defproduceraPrvtKey=None)
@@ -103,8 +101,6 @@ class TraceApiPluginTest(unittest.TestCase):
                     self.assertIn('memo', prms)
                 break
         self.assertTrue(isTrxInBlockFromTraceApi)
-        global testSuccessful
-        testSuccessful = True
 
     @classmethod
     def setUpClass(self):
@@ -113,7 +109,7 @@ class TraceApiPluginTest(unittest.TestCase):
 
     @classmethod
     def tearDownClass(self):
-        self.cleanEnv(self, shouldCleanup=testSuccessful)
+        self.cleanEnv(self, shouldCleanup=False)   # not cleanup to save log in case for further investigation
 
 if __name__ == "__main__":
     unittest.main()

tests/trx_finality_status_forked_test.py

@@ -184,7 +184,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(60), \
+    assert prodC.waitForLibToAdvance(), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
 
     retStatus = prodC.getTransactionStatus(transId)

unittests/restart_chain_tests.cpp

@@ -237,4 +237,19 @@ BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) {
    BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->digest(), other_trace->action_traces.at(1).receipt->digest());
 }
 
+namespace{
+   struct scoped_temp_path {
+      boost::filesystem::path path;
+      scoped_temp_path() {
+         path = boost::filesystem::unique_path();
+         if (boost::unit_test::framework::master_test_suite().argc >= 2) {
+            path += boost::unit_test::framework::master_test_suite().argv[1];
+         }
+      }
+      ~scoped_temp_path() {
+         boost::filesystem::remove_all(path);
+      }
+   };
+}
+
 BOOST_AUTO_TEST_SUITE_END()

unittests/snapshot_tests.cpp

@@ -412,8 +412,8 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_compatible_versions, SNAPSHOT_SUITE, snapshot
       bfs::copy_file(source_i, source_log_dir / "blocks.index", bfs::copy_option::overwrite_if_exists);
       chain.close();
    }
-   fc::temp_directory temp_dir;
-   auto config = tester::default_config(temp_dir, legacy_default_max_inline_action_size).first;
+
+   auto config = tester::default_config(fc::temp_directory(), legacy_default_max_inline_action_size).first;
    auto genesis = eosio::chain::block_log::extract_genesis_state(source_log_dir);
    bfs::create_directories(config.blocks_dir);
    bfs::copy(source_log_dir / "blocks.log", config.blocks_dir / "blocks.log");
@@ -477,8 +477,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_pending_schedule_snapshot, SNAPSHOT_SUITE, sn
    source_log_dir_str += "prod_sched";
    const auto source_log_dir = bfs::path(source_log_dir_str.c_str());
    const uint32_t legacy_default_max_inline_action_size = 4 * 1024;
-   fc::temp_directory temp_dir;
-   auto config = tester::default_config(temp_dir, legacy_default_max_inline_action_size).first;
+   auto config = tester::default_config(fc::temp_directory(), legacy_default_max_inline_action_size).first;
    auto genesis = eosio::chain::block_log::extract_genesis_state(source_log_dir);
    bfs::create_directories(config.blocks_dir);
    bfs::copy(source_log_dir / "blocks.log", config.blocks_dir / "blocks.log");