Merge branch 'release/4.0' into wasm_compile

[4.0] Correct time summary calculation for transient trxs due to change of read-only trxs windows

CMakeLists.txt

@@ -16,7 +16,7 @@ set( CXX_STANDARD_REQUIRED ON)
 set(VERSION_MAJOR 4)
 set(VERSION_MINOR 0)
 set(VERSION_PATCH 0)
-set(VERSION_SUFFIX dev)
+set(VERSION_SUFFIX rc2)
 
 if(VERSION_SUFFIX)
     set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}")
@@ -77,29 +77,9 @@ if(ENABLE_OC AND CMAKE_SIZEOF_VOID_P EQUAL 8 AND NOT WIN32)
       list(APPEND EOSIO_WASM_RUNTIMES eos-vm-oc)
       # EOS VM OC requires LLVM, but move the check up here to a central location so that the EosioTester.cmakes
       # can be created with the exact version found
-      # find_package VERSION range is only available in 3.19 and newer.
-      if(${CMAKE_VERSION} VERSION_LESS "3.19")
-         find_package(LLVM REQUIRED CONFIG)
-      else()
-         # Preferring the latest LLVM version, search through major 11..7 and minor 9..0 in reverse order.
-         # Minor versions may break API. This manifests in the LLVM config files such that a range of `11...<12`
-         # will NOT find LLVM `11.1`. We loop over minor as well as major to resolve this.
-         set(max_minor 9)
-         set(major 11)
-         set(minor ${max_minor})
-         while(NOT LLVM_FOUND AND major GREATER_EQUAL 7)
-            math(EXPR end "${major} + 1")
-            find_package(LLVM ${major}.${minor}...<${end} CONFIG QUIET)
-            if(minor GREATER 0)
-               math(EXPR minor "${minor} - 1")
-            else()
-               math(EXPR major "${major} - 1")
-               set(minor ${max_minor})
-            endif()
-         endwhile()
-      endif()
+      find_package(LLVM REQUIRED CONFIG)
       if(LLVM_VERSION_MAJOR VERSION_LESS 7 OR LLVM_VERSION_MAJOR VERSION_GREATER_EQUAL 12)
-         message(FATAL_ERROR "Leap requires an LLVM version 7 through 11")
+	      message(FATAL_ERROR "Leap requires an LLVM version 7 through 11")
       endif()
    endif()
 endif()

README.md

@@ -72,7 +72,7 @@ git clone --recursive https://github.com/AntelopeIO/leap.git
 git clone --recursive git@github.com:AntelopeIO/leap.git
 ```
 
-> ℹ️ **HTTPS vs. SSH Clone** ℹ️
+> ℹ️ **HTTPS vs. SSH Clone** ℹ️  
 Both an HTTPS or SSH git clone will yield the same result - a folder named `leap` containing our source code. It doesn't matter which type you use.
 
 Navigate into that folder:
@@ -96,13 +96,13 @@ git submodule update --init --recursive
 ### Step 3 - Build
 Select build instructions below for a [pinned build](#pinned-build) (preferred) or an [unpinned build](#unpinned-build).
 
-> ℹ️ **Pinned vs. Unpinned Build** ℹ️
+> ℹ️ **Pinned vs. Unpinned Build** ℹ️  
 We have two types of builds for Leap: "pinned" and "unpinned." The only difference is that pinned builds use specific versions for some dependencies hand-picked by the Leap engineers - they are "pinned" to those versions. In contrast, unpinned builds use the default dependency versions available on the build system at the time. We recommend performing a "pinned" build to ensure the compiler and boost versions remain the same between builds of different Leap versions. Leap requires these versions to remain the same, otherwise its state might need to be recovered from a portable snapshot or the chain needs to be replayed.
 
-> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️
+> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️  
 When building C/C++ software, often the build is performed in parallel via a command such as `make -j "$(nproc)"` which uses all available CPU threads. However, be aware that some compilation units (`*.cpp` files) in Leap will consume nearly 4GB of memory. Failures due to memory exhaustion will typically, but not always, manifest as compiler crashes. Using all available CPU threads may also prevent you from doing other things on your computer during compilation. For these reasons, consider reducing this value.
 
-> 🐋 **Docker and `sudo`** 🐋
+> 🐋 **Docker and `sudo`** 🐋  
 If you are in an Ubuntu docker container, omit `sudo` from all commands because you run as `root` by default. Most other docker containers also exclude `sudo`, especially Debian-family containers. If your shell prompt is a hash tag (`#`), omit `sudo`.
 
 #### Pinned Build
@@ -111,12 +111,14 @@ Make sure you are in the root of the `leap` repo, then run the `install_depts.sh
 sudo scripts/install_deps.sh
 ```
 
-Next, run the pinned build script. You have to give it three arguments, in the following order:
-  - A temporary folder, for all dependencies that need to be built from source.
-  - A build folder, where the binaries you need to install will be built to.
-  - The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
+Next, run the pinned build script. You have to give it three arguments in the following order:
+1. A temporary folder, for all dependencies that need to be built from source.
+1. A build folder, where the binaries you need to install will be built to.
+1. The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
 
-The following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads (Note: you don't need `sudo` for this command):
+> 🔒 You do not need to run this script with `sudo` or as root.
+
+For example, the following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads:
 ```bash
 scripts/pinned_build.sh deps build "$(nproc)"
 ```
@@ -146,11 +148,9 @@ To build, make sure you are in the root of the `leap` repo, then run the followi
 ```bash
 mkdir -p build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Release ..
+cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/usr/lib/llvm-11 ..
 make -j "$(nproc)" package
 ```
-
-If CMake fails to find a valid llvm configure file, it may be necessary to add `-DCMAKE_PREFIX_PATH=/usr/lib/llvm-11` to the cmake command.
 </details>
 
 <details> <summary>Ubuntu 18.04 Bionic</summary>

libraries/chain/CMakeLists.txt

@@ -100,6 +100,7 @@ add_library( eosio_chain
 
               wast_to_wasm.cpp
               wasm_interface.cpp
+              wasm_module_cache.cpp
               wasm_eosio_validation.cpp
               wasm_eosio_injection.cpp
               wasm_config.cpp

libraries/chain/controller.cpp

@@ -246,11 +246,10 @@ struct controller_impl {
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
    thread_local static vm::wasm_allocator wasm_alloc; // a copy for main thread and each read-only thread
 #endif
-   // Ideally wasmif should be thread_local which must be a static.
-   // Unittests can create multiple controller objects (testers) at the same time,
-   // which overwrites the same static wasmif, is used for eosvmoc too.
    wasm_interface  wasmif;  // used by main thread and all threads for EOSVMOC
-   thread_local static std::unique_ptr<wasm_interface> wasmif_thread_local; // a copy for each read-only thread, used by eos-vm and eos-vm-jit
+   std::mutex threaded_wasmifs_mtx;
+   std::unordered_map<std::thread::id, std::unique_ptr<wasm_interface>> threaded_wasmifs; // one for each read-only thread, used by eos-vm and eos-vm-jit
+   wasm_module_cache wasm_mod_cache;
 
    typedef pair<scope_name,action_name>                   handler_key;
    map< account_name, map<handler_key, apply_handler> >   apply_handlers;
@@ -337,7 +336,13 @@ struct controller_impl {
       set_activation_handler<builtin_protocol_feature_t::crypto_primitives>();
 
       self.irreversible_block.connect([this](const block_state_ptr& bsp) {
-         get_wasm_interface().current_lib(bsp->block_num);
+         // producer_plugin has already asserted irreversible_block signal is called in
+         // write window
+         wasmif.current_lib(bsp->block_num);
+         for (auto& ele: threaded_wasmifs) {
+            ele.second->current_lib(bsp->block_num);
+         }
+         wasm_mod_cache.current_lib(bsp->block_num);
       });
 
 
@@ -497,9 +502,10 @@ struct controller_impl {
 
    void replay(std::function<bool()> check_shutdown) {
       auto blog_head = blog.head();
-      if( !blog_head && !fork_db.root() ) {
+      if( !fork_db.root() ) {
          fork_db.reset( *head );
-         return;
+         if (!blog_head)
+            return;
       }
 
       replaying = true;
@@ -2680,28 +2686,53 @@ struct controller_impl {
    // only called from non-main threads (read-only trx execution threads)
    // when producer_plugin starts them
    void init_thread_local_data() {
-      EOS_ASSERT( !is_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
+      EOS_ASSERT( !is_on_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if ( is_eos_vm_oc_enabled() )
          // EOSVMOC needs further initialization of its thread local data
          wasmif.init_thread_local_data();
       else
 #endif
+      {
          // Non-EOSVMOC needs a wasmif per thread
-         wasmif_thread_local = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+         std::lock_guard g(threaded_wasmifs_mtx);
+         threaded_wasmifs[std::this_thread::get_id()] = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+      }
    }
 
-   bool is_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+   bool is_on_main_thread() { return main_thread_id == std::this_thread::get_id(); };
 
+   wasm_module_cache& get_wasm_module_cache() {
+      return wasm_mod_cache;
+   }
+
+   void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+      // The caller of this function apply_eosio_setcode has already asserted that
+      // the transaction is not a read-only trx, which implies we are in now
+      // in write window. Safe to call threaded_wasmifs's code_block_num_last_used
+      wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      for (auto& ele: threaded_wasmifs) {
+         ele.second->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      }
+      wasm_mod_cache.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+   }
+
+   void wasm_interface_exit() {
+      // exit all running wasmifs
+      wasmif.exit();
+      for (auto& ele: threaded_wasmifs) {
+         ele.second->exit();
+      }
+   }
    wasm_interface& get_wasm_interface() {
-      if ( is_main_thread()
+      if ( is_on_main_thread()
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
           || is_eos_vm_oc_enabled()
 #endif
          )
          return wasmif;
       else
-         return *wasmif_thread_local;
+         return *threaded_wasmifs[std::this_thread::get_id()];
    }
 
    block_state_ptr fork_db_head() const;
@@ -2711,7 +2742,6 @@ thread_local platform_timer controller_impl::timer;
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
 thread_local eosio::vm::wasm_allocator controller_impl::wasm_alloc;
 #endif
-thread_local std::unique_ptr<wasm_interface> controller_impl::wasmif_thread_local;
 
 const resource_limits_manager&   controller::get_resource_limits_manager()const
 {
@@ -3686,6 +3716,22 @@ void controller::init_thread_local_data() {
    my->init_thread_local_data();
 }
 
+bool controller::is_on_main_thread() const {
+  return my->is_on_main_thread();
+}
+
+wasm_module_cache& controller::get_wasm_module_cache() {
+   return my->get_wasm_module_cache();
+}
+
+void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+   return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+}
+
+void controller::wasm_interface_exit() {
+   return my->wasm_interface_exit();
+}
+
 /// Protocol feature activation handlers:
 
 template<>

libraries/chain/eosio_contract.cpp

@@ -160,7 +160,7 @@ void apply_eosio_setcode(apply_context& context) {
       old_size  = (int64_t)old_code_entry.code.size() * config::setcode_ram_bytes_multiplier;
       if( old_code_entry.code_ref_count == 1 ) {
          db.remove(old_code_entry);
-         context.control.get_wasm_interface().code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
+         context.control.code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
       } else {
          db.modify(old_code_entry, [](code_object& o) {
             --o.code_ref_count;

libraries/chain/include/eosio/chain/controller.hpp

@@ -10,6 +10,7 @@
 #include <eosio/chain/account_object.hpp>
 #include <eosio/chain/snapshot.hpp>
 #include <eosio/chain/protocol_feature_manager.hpp>
+#include <eosio/chain/wasm_module_cache.hpp>
 #include <eosio/chain/webassembly/eos-vm-oc/config.hpp>
 
 namespace chainbase {
@@ -373,6 +374,10 @@ namespace eosio { namespace chain {
       void set_db_read_only_mode();
       void unset_db_read_only_mode();
       void init_thread_local_data();
+      bool is_on_main_thread() const;
+      wasm_module_cache& get_wasm_module_cache();
+      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
+      void wasm_interface_exit();
 
       private:
          friend class apply_context;

libraries/chain/include/eosio/chain/wasm_interface_private.hpp

@@ -37,14 +37,12 @@ namespace eosio { namespace chain {
    struct wasm_interface_impl {
       struct wasm_cache_entry {
          digest_type                                          code_hash;
-         uint32_t                                             first_block_num_used;
          uint32_t                                             last_block_num_used;
          std::unique_ptr<wasm_instantiated_module_interface>  module;
          uint8_t                                              vm_type = 0;
          uint8_t                                              vm_version = 0;
       };
       struct by_hash;
-      struct by_first_block_num;
       struct by_last_block_num;
 
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
@@ -157,7 +155,6 @@ namespace eosio { namespace chain {
 
             it = wasm_instantiation_cache.emplace( wasm_interface_impl::wasm_cache_entry{
                                                       .code_hash = code_hash,
-                                                      .first_block_num_used = codeobject->first_block_used,
                                                       .last_block_num_used = UINT32_MAX,
                                                       .module = nullptr,
                                                       .vm_type = vm_type,
@@ -173,21 +170,10 @@ namespace eosio { namespace chain {
                trx_context.resume_billing_timer();
             });
             trx_context.pause_billing_timer();
-            IR::Module module;
             std::vector<U8> bytes = {
                 (const U8*)codeobject->code.data(),
                 (const U8*)codeobject->code.data() + codeobject->code.size()};
-            try {
-               Serialization::MemoryInputStream stream((const U8*)bytes.data(),
-                                                       bytes.size());
-               WASM::scoped_skip_checks no_check;
-               WASM::serialize(stream, module);
-               module.userSections.clear();
-            } catch (const Serialization::FatalSerializationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            } catch (const IR::ValidationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            }
+            IR::Module module = trx_context.control.get_wasm_module_cache().get_module(code_hash, vm_type, vm_version, bytes);
 
             wasm_instantiation_cache.modify(it, [&](auto& c) {
                c.module = runtime_interface->instantiate_module((const char*)bytes.data(), bytes.size(), parse_initial_memory(module), code_hash, vm_type, vm_version);
@@ -209,7 +195,6 @@ namespace eosio { namespace chain {
                   member<wasm_cache_entry, uint8_t,     &wasm_cache_entry::vm_version>
                >
             >,
-            ordered_non_unique<tag<by_first_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::first_block_num_used>>,
             ordered_non_unique<tag<by_last_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::last_block_num_used>>
          >
       > wasm_cache_index;

libraries/chain/include/eosio/chain/wasm_module_cache.hpp

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <eosio/chain/types.hpp>  // for digest_type
+#include "IR/Module.h"
+#include <eosio/chain/multi_index_includes.hpp>
+
+#include <mutex>
+
+namespace eosio { namespace chain {
+   struct wasm_module_cache {
+      const IR::Module& get_module( const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, std::vector<U8>& bytes );
+      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
+      void current_lib(uint32_t lib);
+
+      struct module_entry {
+         digest_type   code_hash;
+         uint8_t       vm_type = 0;
+         uint8_t       vm_version = 0;
+         uint32_t      last_block_num_used = UINT32_MAX;
+         IR::Module    module;
+      };
+      struct by_hash;
+      struct by_last_block_num;
+
+   private:
+      typedef boost::multi_index_container<
+         module_entry,
+         indexed_by<
+            ordered_unique<tag<by_hash>,
+               composite_key<module_entry,
+                  member<module_entry, digest_type, &module_entry::code_hash>,
+                  member<module_entry, uint8_t,     &module_entry::vm_type>,
+                  member<module_entry, uint8_t,     &module_entry::vm_version>
+               >
+            >,
+            ordered_non_unique<tag<by_last_block_num>, member<module_entry, uint32_t, &module_entry::last_block_num_used>>
+         >
+      > module_cache_index;
+
+      std::mutex mtx;
+      module_cache_index module_cache;
+   };
+
+} } // eosio::chain

libraries/chain/wasm_module_cache.cpp

@@ -0,0 +1,55 @@
+#include <eosio/chain/wasm_module_cache.hpp>
+#include <eosio/chain/exceptions.hpp>
+#include "WAST/WAST.h"
+#include "IR/Validate.h"
+
+namespace eosio { namespace chain {
+   const IR::Module& wasm_module_cache::get_module( const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, std::vector<U8>& bytes )
+   {
+      std::lock_guard g(mtx);
+      module_cache_index::iterator it = module_cache.find( boost::make_tuple(code_hash, vm_type, vm_version) );
+      if(it == module_cache.end()) {
+         IR::Module module;
+         try {
+            Serialization::MemoryInputStream stream((const U8*)bytes.data(), bytes.size());
+            WASM::scoped_skip_checks no_check;
+            WASM::serialize(stream, module);
+            module.userSections.clear();
+         } catch (const Serialization::FatalSerializationException& e) {
+            EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
+         } catch (const IR::ValidationException& e) {
+            EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
+         }
+
+         it = module_cache.emplace( module_entry {
+                                      .code_hash = code_hash,
+                                      .vm_type = vm_type,
+                                      .vm_version = vm_version,
+                                      .module = module
+                                   } ).first;
+      }
+
+      return it->module;
+   }
+
+   // apply_eosio_setcode asserted this was not called by non-read-only trxs.
+   // This implies it is in write window and no read-only threads are running.
+   // Safe to update module_cache without mutex.
+   void wasm_module_cache::code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+      auto it = module_cache.find(boost::make_tuple(code_hash,vm_type, vm_version));
+      if(it != module_cache.end())
+         module_cache.modify(it, [block_num](module_entry& e) {
+            e.last_block_num_used = block_num;
+         });
+      }
+
+   // producer_plugin has already asserted irreversible_block signal is called
+   // in write window. No read-only threads are running. Safe to update
+   // module_cache without mutex.
+   void wasm_module_cache::current_lib(uint32_t lib) {
+      //anything last used before or on the LIB can be evicted
+      const auto first_it = module_cache.get<by_last_block_num>().begin();
+      const auto last_it  = module_cache.get<by_last_block_num>().upper_bound(lib);
+      module_cache.get<by_last_block_num>().erase(first_it, last_it);
+   }
+} } // eosio::chain

libraries/chain/webassembly/cf_system.cpp

@@ -46,6 +46,6 @@ namespace eosio { namespace chain { namespace webassembly {
    }
 
    void interface::eosio_exit( int32_t code ) const {
-      context.control.get_wasm_interface().exit();
+      context.control.wasm_interface_exit();
    }
 }}} // ns eosio::chain::webassembly

libraries/custom_appbase/include/eosio/chain/application.hpp

@@ -1,30 +1,33 @@
 #pragma once
 
 #include <appbase/application_base.hpp>
-#include <appbase/execution_priority_queue.hpp>
+#include <eosio/chain/exec_pri_queue.hpp>
+#include <mutex>
 
 /*
- * Custmomize appbase to support two-queue exector.
+ * Customize appbase to support two-queue executor.
 */
 namespace appbase { 
 
 enum class exec_window {
-   read,  // the window during which operations from read_only_trx_safe queue
-          // can be executed in app thread in parallel with read-only transactions
-          // in read-only transaction excuting threads.
-   write, // the window during which operations from both general and
-          // read_only_trx_safe queues can be executed in app thread,
-          // while read-only transactions are not executed in read-only
-          // transaction excuting threads.
+   read,  // the window during which operations from read_only queue
+          // can be executed in parallel in the read-only thread pool
+          // as well as in the app thread.
+   write, // the window during which operations from both read_write and
+          // parallel queues can be executed in app thread,
+          // while read-only operations are not executed in read-only
+          // thread pool. The read-only thread pool is not active; only
+          // the main app thread is active.
 };
 
 enum class exec_queue {
-   read_only_trx_safe, // the queue storing operations which are safe to execute
-                       // on app thread in parallel with read-only transactions
-                       // in read-only transaction excuting threads.
-   general             // the queue storing operations which can be only executed
-                       // on the app thread while read-only transactions are
-                       // not being executed in read-only threads
+   read_only,          // the queue storing tasks which are safe to execute
+                       // in parallel with other read-only tasks in the read-only
+                       // thread pool as well as on the main app thread.
+                       // Multi-thread safe as long as nothing is executed from the read_write queue.
+   read_write          // the queue storing tasks which can be only executed
+                       // on the app thread while read-only tasks are
+                       // not being executed in read-only threads. Single threaded.
 };
 
 class two_queue_executor {
@@ -32,56 +35,64 @@ public:
 
    template <typename Func>
    auto post( int priority, exec_queue q, Func&& func ) {
-      if ( q == exec_queue::general )
-         return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      if ( q == exec_queue::read_write )
+         return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
       else
-         return boost::asio::post(io_serv_, read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+         return boost::asio::post( io_serv_, read_only_queue_.wrap( priority, --order_, std::forward<Func>( func)));
    }
 
    // Legacy and deprecated. To be removed after cleaning up its uses in base appbase
    template <typename Func>
    auto post( int priority, Func&& func ) {
-      // safer to use general queue for unknown type of operation since operations
-      // from general queue are not executed in parallel with read-only transactions
-      return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      // safer to use read_write queue for unknown type of operation since operations
+      // from read_write queue are not executed in parallel with read-only operations
+      return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
    }
 
    boost::asio::io_service& get_io_service() { return io_serv_; }
 
    bool execute_highest() {
       if ( exec_window_ == exec_window::write ) {
-         if( !general_queue_.empty() && ( read_only_trx_safe_queue_.empty() || *read_only_trx_safe_queue_.top() < *general_queue_.top()) )  {
-            // general_queue_'s top function's priority greater than read_only_trx_safe_queue_'s top function's, or general_queue_ empty
-            general_queue_.execute_highest();
-         } else if( !read_only_trx_safe_queue_.empty() ) {
-            read_only_trx_safe_queue_.execute_highest();
+         // During write window only main thread is accessing anything in two_queue_executor, no locking required
+         if( !read_write_queue_.empty() && (read_only_queue_.empty() || *read_only_queue_.top() < *read_write_queue_.top()) )  {
+            // read_write_queue_'s top function's priority greater than read_only_queue_'s top function's, or read_only_queue_ empty
+            read_write_queue_.execute_highest();
+         } else if( !read_only_queue_.empty() ) {
+            read_only_queue_.execute_highest();
          }
-         return !read_only_trx_safe_queue_.empty() || !general_queue_.empty();
+         return !read_only_queue_.empty() || !read_write_queue_.empty();
       } else {
-         return read_only_trx_safe_queue_.execute_highest();
+         // When in read window, multiple threads including main app thread are accessing two_queue_executor, locking required
+         return read_only_queue_.execute_highest_locked(false);
       }
    }
 
+   bool execute_highest_read_only() {
+      return read_only_queue_.execute_highest_locked(true);
+   }
+
    template <typename Function>
-   boost::asio::executor_binder<Function, appbase::execution_priority_queue::executor>
+   boost::asio::executor_binder<Function, appbase::exec_pri_queue::executor>
    wrap(int priority, exec_queue q, Function&& func ) {
-      if ( q == exec_queue::general )
-         return general_queue_.wrap(priority, --order_, std::forward<Function>(func));
+      if ( q == exec_queue::read_write )
+         return read_write_queue_.wrap(priority, --order_, std::forward<Function>(func));
       else
-         return read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Function>(func));
+         return read_only_queue_.wrap( priority, --order_, std::forward<Function>( func));
    }
      
    void clear() {
-      read_only_trx_safe_queue_.clear();
-      general_queue_.clear();
+      read_only_queue_.clear();
+      read_write_queue_.clear();
    }
 
-   void set_to_read_window() {
+   void set_to_read_window(uint32_t num_threads, std::function<bool()> should_exit) {
       exec_window_ = exec_window::read;
+      read_only_queue_.enable_locking(num_threads, std::move(should_exit));
    }
 
    void set_to_write_window() {
       exec_window_ = exec_window::write;
+      read_only_queue_.disable_locking();
    }
 
    bool is_read_window() const {
@@ -92,15 +103,15 @@ public:
       return exec_window_ == exec_window::write;
    }
 
-   auto& read_only_trx_safe_queue() { return read_only_trx_safe_queue_; }
+   auto& read_only_queue() { return read_only_queue_; }
 
-   auto& general_queue() { return general_queue_; }
+   auto& read_write_queue() { return read_write_queue_; }
 
    // members are ordered taking into account that the last one is destructed first
 private:
    boost::asio::io_service            io_serv_;
-   appbase::execution_priority_queue  read_only_trx_safe_queue_;
-   appbase::execution_priority_queue  general_queue_;
+   appbase::exec_pri_queue            read_only_queue_;
+   appbase::exec_pri_queue            read_write_queue_;
    std::atomic<std::size_t>           order_ { std::numeric_limits<size_t>::max() }; // to maintain FIFO ordering in both queues within priority
    exec_window                        exec_window_ { exec_window::write };
 };

libraries/custom_appbase/include/eosio/chain/exec_pri_queue.hpp

@@ -0,0 +1,233 @@
+#pragma once
+#include <boost/asio.hpp>
+
+#include <condition_variable>
+#include <mutex>
+#include <queue>
+
+namespace appbase {
+// adapted from: https://www.boost.org/doc/libs/1_69_0/doc/html/boost_asio/example/cpp11/invocation/prioritised_handlers.cpp
+
+// Locking has to be coordinated by caller, use with care.
+class exec_pri_queue : public boost::asio::execution_context
+{
+public:
+
+   void enable_locking(uint32_t num_threads, std::function<bool()> should_exit) {
+      assert(num_threads > 0 && num_waiting_ == 0);
+      lock_enabled_ = true;
+      max_waiting_ = num_threads;
+      should_exit_ = std::move(should_exit);
+      exiting_blocking_ = false;
+   }
+
+   void disable_locking() {
+      lock_enabled_ = false;
+      should_exit_ = [](){ assert(false); return true; }; // should not be called when locking is disabled
+   }
+
+   // called from appbase::application_base::exec poll_one() or run_one()
+   template <typename Function>
+   void add(int priority, size_t order, Function function)
+   {
+      std::unique_ptr<queued_handler_base> handler(new queued_handler<Function>(priority, order, std::move(function)));
+      if (lock_enabled_) {
+         std::lock_guard g( mtx_ );
+         handlers_.push( std::move( handler ) );
+         if (num_waiting_)
+            cond_.notify_one();
+      } else {
+         handlers_.push( std::move( handler ) );
+      }
+   }
+
+   // only call when no lock required
+   void clear()
+   {
+      handlers_ = prio_queue();
+   }
+
+   // only call when no lock required
+   bool execute_highest()
+   {
+      if( !handlers_.empty() ) {
+         handlers_.top()->execute();
+         handlers_.pop();
+      }
+
+      return !handlers_.empty();
+   }
+
+private:
+   // has to be defined before use, auto return type
+   auto pop() {
+      auto t = std::move(const_cast<std::unique_ptr<queued_handler_base>&>(handlers_.top()));
+      handlers_.pop();
+      return t;
+   }
+
+public:
+
+   bool execute_highest_locked(bool should_block) {
+      std::unique_lock g(mtx_);
+      if (should_block) {
+         ++num_waiting_;
+         cond_.wait(g, [this](){
+            bool exit = exiting_blocking_ || should_exit_();
+            bool empty = handlers_.empty();
+            if (empty || exit) {
+               if (((empty && num_waiting_ == max_waiting_) || exit) && !exiting_blocking_) {
+                  cond_.notify_all();
+                  exiting_blocking_ = true;
+               }
+               return exit || exiting_blocking_; // same as calling should_exit(), but faster
+            }
+            return true;
+         });
+         --num_waiting_;
+         if (exiting_blocking_ || should_exit_())
+            return false;
+      }
+      if( handlers_.empty() )
+         return false;
+      auto t = pop();
+      g.unlock();
+      t->execute();
+      return true;
+   }
+
+   // Only call when locking disabled
+   size_t size() const { return handlers_.size(); }
+
+   // Only call when locking disabled
+   bool empty() const { return handlers_.empty(); }
+
+   // Only call when locking disabled
+   const auto& top() const { return handlers_.top(); }
+
+   class executor
+   {
+   public:
+      executor(exec_pri_queue& q, int p, size_t o)
+            : context_(q), priority_(p), order_(o)
+      {
+      }
+
+      exec_pri_queue& context() const noexcept
+      {
+         return context_;
+      }
+
+      template <typename Function, typename Allocator>
+      void dispatch(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void post(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void defer(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      void on_work_started() const noexcept {}
+      void on_work_finished() const noexcept {}
+
+      bool operator==(const executor& other) const noexcept
+      {
+         return order_ == other.order_ && &context_ == &other.context_ && priority_ == other.priority_;
+      }
+
+      bool operator!=(const executor& other) const noexcept
+      {
+         return !operator==(other);
+      }
+
+   private:
+      exec_pri_queue& context_;
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   boost::asio::executor_binder<Function, executor>
+   wrap(int priority, size_t order, Function&& func)
+   {
+      return boost::asio::bind_executor( executor(*this, priority, order), std::forward<Function>(func) );
+   }
+
+private:
+   class queued_handler_base
+   {
+   public:
+      queued_handler_base( int p, size_t order )
+            : priority_( p )
+            , order_( order )
+      {
+      }
+
+      virtual ~queued_handler_base() = default;
+
+      virtual void execute() = 0;
+
+      int priority() const { return priority_; }
+      // C++20
+      // friend std::weak_ordering operator<=>(const queued_handler_base&,
+      //                                       const queued_handler_base&) noexcept = default;
+      friend bool operator<(const queued_handler_base& a,
+                            const queued_handler_base& b) noexcept
+      {
+         return std::tie( a.priority_, a.order_ ) < std::tie( b.priority_, b.order_ );
+      }
+
+   private:
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   class queued_handler : public queued_handler_base
+   {
+   public:
+      queued_handler(int p, size_t order, Function f)
+            : queued_handler_base( p, order )
+            , function_( std::move(f) )
+      {
+      }
+
+      void execute() override
+      {
+         function_();
+      }
+
+   private:
+      Function function_;
+   };
+
+   struct deref_less
+   {
+      template<typename Pointer>
+      bool operator()(const Pointer& a, const Pointer& b) noexcept(noexcept(*a < *b))
+      {
+         return *a < *b;
+      }
+   };
+
+   bool lock_enabled_ = false;
+   std::mutex mtx_;
+   std::condition_variable cond_;
+   uint32_t num_waiting_{0};
+   uint32_t max_waiting_{0};
+   bool exiting_blocking_{false};
+   std::function<bool()> should_exit_; // called holding mtx_
+   using prio_queue = std::priority_queue<std::unique_ptr<queued_handler_base>, std::deque<std::unique_ptr<queued_handler_base>>, deref_less>;
+   prio_queue handlers_;
+};
+
+} // appbase

libraries/custom_appbase/tests/custom_appbase_tests.cpp

@@ -27,28 +27,28 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
 
    // Stop app. Use the lowest priority to make sure this function to execute the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_only_trx_safe_queue should only contain the current lambda function,
-      // and general_queue should have execute all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_only_queue should only contain the current lambda function,
+      // and read_write_queue should have executed all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of both queues' functions processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 8 );
@@ -64,40 +64,40 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    BOOST_CHECK_LT( rslts[6], rslts[7] );
 }
 
-// verify functions only from read_only_trx_safe queue are processed during read window
+// verify functions only from read_only queue are processed during read window
 BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 4 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 4 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 6 );
@@ -109,40 +109,40 @@ BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    BOOST_CHECK_LT( rslts[3], rslts[4] );
 }
 
-// verify no functions are executed during  read window if read_only_trx_safe queue is empty
+// verify no functions are executed during  read window if read_only queue is empty
 BOOST_AUTO_TEST_CASE( execute_from_empty_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // Stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 10 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 10 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // no results
    BOOST_REQUIRE_EQUAL( rslts.size(), 0 );
@@ -159,32 +159,32 @@ BOOST_AUTO_TEST_CASE( execute_from_both_queues ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[9]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[10]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[11]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[10]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[11]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
       // read_queue should have current function and write_queue's functions are all executed 
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
 
    app_thread.join();
 
    // queues are emptied after quit
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 12 );

libraries/libfc/CMakeLists.txt

@@ -57,8 +57,6 @@ set( fc_sources
      src/crypto/blake2.cpp
      src/crypto/k1_recover.cpp
      src/network/ip.cpp
-     src/network/resolve.cpp
-     src/network/udp_socket.cpp
      src/network/url.cpp
      src/network/http/http_client.cpp
      src/compress/zlib.cpp

libraries/libfc/include/fc/network/resolve.hpp

@@ -1,9 +0,0 @@
-#pragma once
-#include <fc/vector.hpp>
-#include <fc/network/ip.hpp>
-
-namespace fc
-{
-  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
-                                                      const std::string& host, uint16_t port);
-}

libraries/libfc/include/fc/network/udp_socket.hpp

@@ -1,39 +0,0 @@
-#pragma once
-#include <fc/utility.hpp>
-#include <memory>
-
-#include <boost/asio.hpp>
-
-namespace fc {
-  namespace ip {
-    class endpoint;
-    class address;
-  }
-
-  /**
-   *  The udp_socket class has reference semantics, all copies will
-   *  refer to the same underlying socket.
-   */
-  class udp_socket {
-    public:
-      udp_socket();
-      udp_socket( const udp_socket& s );
-      ~udp_socket();
-
-      void initialize(boost::asio::io_service &);
-      void open();
-      void send_to(const char* b, size_t l, boost::asio::ip::udp::endpoint &to);
-      void send_to(const std::shared_ptr<const char>& b, size_t l, boost::asio::ip::udp::endpoint &to);
-      void close();
-
-      void set_reuse_address(bool);
-
-      void connect(const boost::asio::ip::udp::endpoint& e);
-      const boost::asio::ip::udp::endpoint local_endpoint() const;
-
-    private:
-      class                impl;
-      std::shared_ptr<impl> my;
-  };
-
-}

libraries/libfc/src/log/gelf_appender.cpp

@@ -1,6 +1,3 @@
-#include <fc/network/udp_socket.hpp>
-#include <fc/network/ip.hpp>
-#include <fc/network/resolve.hpp>
 #include <fc/exception/exception.hpp>
 #include <fc/log/gelf_appender.hpp>
 #include <fc/reflect/variant.hpp>
@@ -10,6 +7,7 @@
 #include <fc/compress/zlib.hpp>
 #include <fc/log/logger_config.hpp>
 
+#include <boost/asio.hpp>
 #include <boost/lexical_cast.hpp>
 #include <iomanip>
 #include <iostream>
@@ -19,14 +17,6 @@
 
 namespace fc
 {
-  namespace detail
-  {
-    boost::asio::ip::udp::endpoint to_asio_ep( const fc::ip::endpoint& e )
-    {
-      return boost::asio::ip::udp::endpoint(boost::asio::ip::address_v4(e.get_address()), e.port() );
-    }
-  }
-
   const std::vector<std::string> gelf_appender::config::reserved_field_names  = {
       "_id",            // per GELF specification
       "_timestamp_ns",  // Remaining names all populated by appender
@@ -36,23 +26,21 @@ namespace fc
       "_method_name",
       "_thread_name",
       "_task_name"
-   };
+  };
 
-   const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
+  const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
 
   class gelf_appender::impl
   {
   public:
     using work_guard_t = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
     config                                         cfg;
-    std::optional<boost::asio::ip::udp::endpoint>  gelf_endpoint;
     std::thread                                    thread;
     boost::asio::io_context                        io_context;
     work_guard_t work_guard =                      boost::asio::make_work_guard(io_context);
-    udp_socket                                     gelf_socket;
+    boost::asio::ip::udp::socket                   gelf_socket;
 
-
-    impl(const variant& c)
+    impl(const variant& c) : gelf_socket(io_context)
     {
       mutable_variant_object mvo;
       from_variant(c, mvo);
@@ -84,6 +72,35 @@ namespace fc
           thread.join();
        }
     }
+
+    static std::shared_ptr<std::vector<char>> make_new_bufer(boost::asio::const_buffer buf) {
+      const char* p = static_cast<const char*>(buf.data());
+      return std::make_shared<std::vector<char>>(p, p+buf.size());
+    }
+
+    static std::shared_ptr<std::vector<char>> make_new_bufer(const std::array<boost::asio::const_buffer, 2>& bufs) {
+      auto new_buf = std::make_shared<std::vector<char>>();
+      new_buf->reserve(bufs[0].size() + bufs[1].size());
+      for (int i = 0; i < 2; ++i) {
+          const char* p = static_cast<const char*>(bufs[i].data());
+          new_buf->insert(new_buf->end(), p, p + bufs[i].size());
+      }
+      return new_buf;
+    }
+
+    template <typename Buffers>
+    void send(Buffers&& bufs) {
+      boost::system::error_code ec;
+      gelf_socket.send(std::forward<Buffers>(bufs), 0, ec);
+      if (ec == boost::asio::error::would_block) {
+          auto new_buf = make_new_bufer(std::forward<Buffers>(bufs));
+          gelf_socket.async_send(boost::asio::buffer(*new_buf),
+                                    [new_buf](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/) {
+                                        // Swallow errors.  Currently only used for GELF logging, so depend on local
+                                        // log to catch anything that doesn't make it across the network.
+                                    });
+      }
+    }
   };
 
   gelf_appender::gelf_appender(const variant& args) :
@@ -95,58 +112,40 @@ namespace fc
   {
     try
     {
-      try
-      {
-        // if it's a numeric address:port, this will parse it
-        my->gelf_endpoint = detail::to_asio_ep(ip::endpoint::from_string(my->cfg.endpoint));
+      if (my->cfg.endpoint.empty()) {
+        fprintf(stderr, "The logging destination is not specified\n");
+        return;
       }
-      catch (...)
-      {
-      }
-      if (!my->gelf_endpoint)
-      {
-        // couldn't parse as a numeric ip address, try resolving as a DNS name.
-        // This can yield, so don't do it in the catch block above
-        string::size_type colon_pos = my->cfg.endpoint.find(':');
-        try
-        {
-          FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
-          string port = my->cfg.endpoint.substr(colon_pos + 1);
 
-          string hostname = my->cfg.endpoint.substr( 0, colon_pos );
+      std::string_view endpoint = my->cfg.endpoint;
+      string::size_type colon_pos = endpoint.rfind(':');
+      FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
+      auto port = endpoint.substr(colon_pos + 1);
 
-          boost::asio::ip::udp::resolver resolver{ my->io_context };
-          auto endpoints = resolver.resolve(hostname, port);
+      auto hostname = (endpoint[0] == '[' && colon_pos >= 2)  ? endpoint.substr( 1, colon_pos-2 ) : endpoint.substr( 0, colon_pos );
 
-          if (endpoints.empty())
-              FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
-                                 ("hostname", hostname));
+      boost::asio::ip::udp::resolver resolver{ my->io_context };
+      auto endpoints = resolver.resolve(hostname, port);
 
-          my->gelf_endpoint = *endpoints.begin();
+      if (endpoints.empty())
+          FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
+                              ("hostname", std::string(hostname)));
+
+      my->gelf_socket.connect(*endpoints.begin());
+      std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
+      my->gelf_socket.non_blocking(true);
+
+      my->thread = std::thread([this] {
+        try {
+          fc::set_os_thread_name("gelf");
+          my->io_context.run();
+        } catch (std::exception& ex) {
+          fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
+        } catch (...) {
+          fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
         }
-        catch (const boost::bad_lexical_cast&)
-        {
-          FC_THROW("Bad port: ${port}", ("port", my->cfg.endpoint.substr(colon_pos + 1, my->cfg.endpoint.size())));
-        }
-      }
+      });
 
-      if (my->gelf_endpoint)
-      {
-        my->gelf_socket.initialize(my->io_context);
-        my->gelf_socket.open();
-        std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
-
-        my->thread = std::thread([this] {
-          try {
-            fc::set_os_thread_name("gelf");
-            my->io_context.run();
-          } catch (std::exception& ex) {
-            fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
-          } catch (...) {
-            fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
-          }
-        });
-      }
     }
     catch (...)
     {
@@ -221,57 +220,44 @@ namespace fc
 
     if (gelf_message_as_string.size() <= max_payload_size)
     {
-      // no need to split
-      std::shared_ptr<char> send_buffer(new char[gelf_message_as_string.size()],
-                                        [](char* p){ delete[] p; });
-      memcpy(send_buffer.get(), gelf_message_as_string.c_str(),
-             gelf_message_as_string.size());
-
-      my->gelf_socket.send_to(send_buffer, gelf_message_as_string.size(),
-                              *my->gelf_endpoint);
+      my->send(boost::asio::buffer(gelf_message_as_string));
     }
     else
     {
       // split the message
+      struct gelf_header {
+        uint8_t magic[2] = { 0x1e, 0x0f};
+        uint64_t message_id;
+        uint8_t seq = 0;
+        uint8_t count = 0;
+      } header;
+
       // we need to generate an 8-byte ID for this message.
       // city hash should do
-      uint64_t message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
-      const unsigned header_length = 2 /* magic */ + 8 /* msg id */ + 1 /* seq */ + 1 /* count */;
-      const unsigned body_length = max_payload_size - header_length;
-      unsigned total_number_of_packets = (gelf_message_as_string.size() + body_length - 1) / body_length;
+      header.message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
+      const unsigned body_length = max_payload_size - sizeof(header);
+      header.count = (gelf_message_as_string.size() + body_length - 1) / body_length;
       unsigned bytes_sent = 0;
-      unsigned number_of_packets_sent = 0;
+
       while (bytes_sent < gelf_message_as_string.size())
       {
         unsigned bytes_to_send = std::min((unsigned)gelf_message_as_string.size() - bytes_sent,
                                           body_length);
 
-        std::shared_ptr<char> send_buffer(new char[max_payload_size],
-                                          [](char* p){ delete[] p; });
-        char* ptr = send_buffer.get();
-        // magic number for chunked message
-        *(unsigned char*)ptr++ = 0x1e;
-        *(unsigned char*)ptr++ = 0x0f;
-
-        // message id
-        memcpy(ptr, (char*)&message_id, sizeof(message_id));
-        ptr += sizeof(message_id);
-
-        *(unsigned char*)(ptr++) = number_of_packets_sent;
-        *(unsigned char*)(ptr++) = total_number_of_packets;
-        memcpy(ptr, gelf_message_as_string.c_str() + bytes_sent,
-               bytes_to_send);
-        my->gelf_socket.send_to(send_buffer, header_length + bytes_to_send,
-                                *my->gelf_endpoint);
-        ++number_of_packets_sent;
+        std::array<boost::asio::const_buffer,2> bufs = {
+          boost::asio::const_buffer(&header, sizeof(header)),
+          boost::asio::const_buffer(gelf_message_as_string.c_str() + bytes_sent, bytes_to_send)
+        };
+        my->send(bufs);
+        ++header.seq;
         bytes_sent += bytes_to_send;
       }
-      FC_ASSERT(number_of_packets_sent == total_number_of_packets);
+      FC_ASSERT(header.seq == header.count);
     }
   }
 
   void gelf_appender::log(const log_message& message) {
-    if (!my->gelf_endpoint)
+    if (!my->thread.joinable())
       return;
 
     // use now() instead of context.get_timestamp() because log_message construction can include user provided long running calls

libraries/libfc/src/network/resolve.cpp

@@ -1,33 +0,0 @@
-#include <boost/asio.hpp>
-#include <fc/exception/exception.hpp>
-
-namespace fc
-{
-  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
-                                                     const std::string& host, uint16_t port)
-  {
-    using q = boost::asio::ip::udp::resolver::query;
-    using b = boost::asio::ip::resolver_query_base;
-    boost::asio::ip::udp::resolver res(io_service);
-    boost::system::error_code ec;
-    auto ep = res.resolve(q(host, std::to_string(uint64_t(port)),
-                            b::address_configured | b::numeric_service), ec);
-    if(!ec)
-    {
-      std::vector<boost::asio::ip::udp::endpoint> eps;
-      while(ep != boost::asio::ip::udp::resolver::iterator())
-      {
-        if(ep->endpoint().address().is_v4())
-        {
-         eps.push_back(*ep);
-        }
-        // TODO: add support for v6
-        ++ep;
-      }
-      return eps;
-    }
-    FC_THROW_EXCEPTION(unknown_host_exception,
-                       "name resolution failed: ${reason}",
-                       ("reason", ec.message()));
-  }
-}

libraries/libfc/src/network/udp_socket.cpp

@@ -1,111 +0,0 @@
-#include <fc/network/udp_socket.hpp>
-#include <fc/network/ip.hpp>
-
-
-namespace fc
-{
-  class udp_socket::impl
-  {
-    public:
-      impl(){}
-      ~impl(){}
-
-      std::shared_ptr<boost::asio::ip::udp::socket> _sock;
-  };
-
-  udp_socket::udp_socket()
-    : my(new impl())
-  {
-  }
-
-  void udp_socket::initialize(boost::asio::io_service& service)
-  {
-    my->_sock.reset(new boost::asio::ip::udp::socket(service));
-  }
-
-  udp_socket::~udp_socket() 
-  {
-    try 
-    {
-      if(my->_sock)
-        my->_sock->close(); //close boost socket to make any pending reads run their completion handler
-    }
-    catch (...) //avoid destructor throw and likely this is just happening because socket wasn't open.
-    {
-    }
-  }
-
-  void udp_socket::send_to(const char* buffer, size_t length, boost::asio::ip::udp::endpoint& to)
-  {
-    try 
-    {
-      my->_sock->send_to(boost::asio::buffer(buffer, length), to);
-      return;
-    } 
-    catch(const boost::system::system_error& e)
-    {
-      if(e.code() == boost::asio::error::would_block)
-      {
-          auto send_buffer_ptr = std::make_shared<std::vector<char>>(buffer, buffer+length);
-          my->_sock->async_send_to(boost::asio::buffer(send_buffer_ptr.get(), length), to,
-                                   [send_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
-          {
-            // Swallow errors.  Currently only used for GELF logging, so depend on local
-            // log to catch anything that doesn't make it across the network.
-          });
-       }
-       // All other exceptions ignored.
-    }
-  }
-
-  void udp_socket::send_to(const std::shared_ptr<const char>& buffer, size_t length,
-                           boost::asio::ip::udp::endpoint& to)
-  {
-    try
-    {
-      my->_sock->send_to(boost::asio::buffer(buffer.get(), length), to);
-      return;
-    } 
-    catch(const boost::system::system_error& e)
-    {
-      if(e.code() == boost::asio::error::would_block)
-      {
-          auto preserved_buffer_ptr = buffer;
-          my->_sock->async_send_to(boost::asio::buffer(preserved_buffer_ptr.get(), length), to,
-                                   [preserved_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
-          {
-            // Swallow errors.  Currently only used for GELF logging, so depend on local
-            // log to catch anything that doesn't make it across the network.
-          });
-       }
-       // All other exceptions ignored.
-    }
-  }
-
-  void udp_socket::open()
-  {
-    my->_sock->open(boost::asio::ip::udp::v4());
-    my->_sock->non_blocking(true);
-  }
-
-  void udp_socket::close()
-  {
-    my->_sock->close();
-  }
-
-  const boost::asio::ip::udp::endpoint udp_socket::local_endpoint() const
-  {
-    return my->_sock->local_endpoint();
-  }
-
-  void udp_socket::connect(const boost::asio::ip::udp::endpoint& e)
-  {
-    my->_sock->connect(e);
-  }
-
-  void udp_socket::set_reuse_address( bool s )
-  {
-    my->_sock->set_option( boost::asio::ip::udp::socket::reuse_address(s) );
-  }
-
-}

libraries/state_history/include/eosio/state_history/log.hpp

@@ -550,6 +550,9 @@ class state_history_log {
          fc::raw::pack(log, num_blocks_in_log);
       }
 
+      log.flush();
+      index.flush();
+
       auto partition_config = std::get_if<state_history::partition_config>(&config);
       if (partition_config && block_num % partition_config->stride == 0) {
          split_log();

plugins/chain_api_plugin/chain_api_plugin.cpp

@@ -75,7 +75,7 @@ void chain_api_plugin::plugin_startup() {
    ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() );
 
    _http_plugin.add_api( {
-      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only, appbase::priority::medium_high);
    _http_plugin.add_api({
       CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params),
       CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required),
@@ -96,19 +96,19 @@ void chain_api_plugin::plugin_startup() {
       CHAIN_RO_CALL(get_scheduled_transactions, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_required_keys, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_transaction_id, 200, http_params_types::params_required),
+      // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue
       CHAIN_RO_CALL_ASYNC(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required),
       CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required),
-      // transaction related APIs will be posted to general queue after keys are recovered
       CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 
    // Not safe to run in parallel with read-only transactions
    _http_plugin.add_api({
       CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::general, appbase::priority::medium_low );
+   }, appbase::exec_queue::read_write, appbase::priority::medium_low );
 
    if (chain.account_queries_enabled()) {
       _http_plugin.add_async_api({
@@ -124,7 +124,7 @@ void chain_api_plugin::plugin_startup() {
    if (chain.transaction_finality_status_enabled()) {
       _http_plugin.add_api({
          CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required),
-      }, appbase::exec_queue::read_only_trx_safe);
+      }, appbase::exec_queue::read_only);
    }
 
    _http_plugin.add_api({
@@ -160,7 +160,7 @@ void chain_api_plugin::plugin_startup() {
            }
         }
       }
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 }
 
 void chain_api_plugin::plugin_shutdown() {}

plugins/db_size_api_plugin/db_size_api_plugin.cpp

@@ -28,7 +28,7 @@ using namespace eosio;
 void db_size_api_plugin::plugin_startup() {
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(db_size, this, get,  INVOKE_R_V(this, get), 200),
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 }
 
 db_size_stats db_size_api_plugin::get() {

plugins/http_plugin/http_plugin.cpp

@@ -334,7 +334,7 @@ namespace eosio {
                      handle_exception("node", "get_supported_apis", body.empty() ? "{}" : body, cb);
                   }
                }
-            }}, appbase::exec_queue::read_only_trx_safe);
+            }}, appbase::exec_queue::read_only);
             
          } catch (...) {
             fc_elog(logger(), "http_plugin startup fails, shutting down");

plugins/http_plugin/tests/unit_tests.cpp

@@ -52,7 +52,7 @@ public:
                   cb(200, fc::time_point::maximum(), fc::variant(ok ? string("yes") : string("no")));
                }
             },
-         }, appbase::exec_queue::general);
+         }, appbase::exec_queue::read_write);
    }
 
 private:

plugins/net_api_plugin/net_api_plugin.cpp

@@ -53,11 +53,7 @@ void net_api_plugin::plugin_startup() {
    ilog("starting net_api_plugin");
    // lifetime of plugin is lifetime of application
    auto& net_mgr = app().get_plugin<net_plugin>();
-   app().get_plugin<http_plugin>().add_api({
-    //   CALL(net, net_mgr, set_timeout,
-    //        INVOKE_V_R(net_mgr, set_timeout, int64_t), 200),
-    //   CALL(net, net_mgr, sign_transaction,
-    //        INVOKE_R_R_R_R(net_mgr, sign_transaction, chain::signed_transaction, flat_set<public_key_type>, chain::chain_id_type), 201),
+   app().get_plugin<http_plugin>().add_async_api({
        CALL_WITH_400(net, net_mgr, connect,
             INVOKE_R_R(net_mgr, connect, std::string), 201),
        CALL_WITH_400(net, net_mgr, disconnect,
@@ -66,9 +62,7 @@ void net_api_plugin::plugin_startup() {
             INVOKE_R_R(net_mgr, status, std::string), 201),
        CALL_WITH_400(net, net_mgr, connections,
             INVOKE_R_V(net_mgr, connections), 201),
-    //   CALL(net, net_mgr, open,
-    //        INVOKE_V_R(net_mgr, open, std::string), 200),
-   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+   } );
 }
 
 void net_api_plugin::plugin_initialize(const variables_map& options) {

plugins/net_plugin/net_plugin.cpp

@@ -3225,22 +3225,23 @@ namespace eosio {
             return;
          }
 
-         bool valid_block_header = !!bsp;
 
-         if( valid_block_header ) {
+         uint32_t block_num = bsp ? bsp->block_num : 0;
+
+         if( block_num != 0 ) {
             fc_dlog( logger, "validated block header, broadcasting immediately, connection ${cid}, blk num = ${num}, id = ${id}",
-                     ("cid", cid)("num", bsp->block_num)("id", bsp->id) );
+                     ("cid", cid)("num", block_num)("id", bsp->id) );
             my_impl->dispatcher->add_peer_block( bsp->id, cid ); // no need to send back to sender
             my_impl->dispatcher->bcast_block( bsp->block, bsp->id );
          }
 
-         app().executor().post(priority::medium, exec_queue::general, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
+         app().executor().post(priority::medium, exec_queue::read_write, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
             c->process_signed_block( id, std::move(ptr), std::move(bsp) );
          });
 
-         if( valid_block_header ) {
+         if( block_num != 0 ) {
             // ready to process immediately, so signal producer to interrupt start_block
-            my_impl->producer_plug->received_block();
+            my_impl->producer_plug->received_block(block_num);
          }
       });
    }

plugins/producer_api_plugin/producer_api_plugin.cpp

@@ -118,9 +118,11 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_R(producer, get_account_ram_corrections, producer_plugin::get_account_ram_corrections_params), 201),
        CALL_WITH_400(producer, producer, get_unapplied_transactions,
                      INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200),
-   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+                     INVOKE_R_V(producer, get_snapshot_requests), 201),
+   }, appbase::exec_queue::read_only, appbase::priority::medium_high);
 
-   // Not safe to run in parallel with read-only transactions
+   // Not safe to run in parallel
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(producer, producer, pause,
             INVOKE_V_V(producer, pause), 201),
@@ -138,15 +140,13 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
        CALL_WITH_400(producer, producer, schedule_snapshot,
             INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
-       CALL_WITH_400(producer, producer, get_snapshot_requests,
-            INVOKE_R_V(producer, get_snapshot_requests), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,
             INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
        CALL_WITH_400(producer, producer, get_integrity_hash,
             INVOKE_R_V(producer, get_integrity_hash), 201),
        CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
             INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201),
-   }, appbase::exec_queue::general, appbase::priority::medium_high);
+   }, appbase::exec_queue::read_write, appbase::priority::medium_high);
 }
 
 void producer_api_plugin::plugin_initialize(const variables_map& options) {

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -202,11 +202,14 @@ public:
    void register_metrics_listener(metrics_listener listener);
 
    // thread-safe, called when a new block is received
-   void received_block();
+   void received_block(uint32_t block_num);
 
-const std::set<account_name>& producer_accounts() const;
+   const std::set<account_name>& producer_accounts() const;
 
+   static void set_test_mode(bool m) { test_mode_ = m; }
  private:
+   inline static bool test_mode_{false}; // to be moved into appbase (application_base)
+   
    std::shared_ptr<class producer_plugin_impl> my;
 };
 

plugins/producer_plugin/producer_plugin.cpp

@@ -14,6 +14,7 @@
 #include <fc/io/json.hpp>
 #include <fc/log/logger_config.hpp>
 #include <fc/scoped_exit.hpp>
+#include <fc/time.hpp>
 
 #include <boost/asio.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -27,7 +28,6 @@
 #include <boost/multi_index/member.hpp>
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
-#include <boost/multi_index/random_access_index.hpp>
 #include <boost/signals2/connection.hpp>
 
 namespace bmi = boost::multi_index;
@@ -256,39 +256,53 @@ struct block_time_tracker {
       block_idle_time += idle;
    }
 
-   void add_fail_time( const fc::microseconds& fail_time ) {
-      trx_fail_time += fail_time;
-      ++trx_fail_num;
+   void add_fail_time( const fc::microseconds& fail_time, bool is_transient ) {
+      if( is_transient ) {
+         // transient time includes both success and fail time
+         transient_trx_time += fail_time;
+         ++transient_trx_num;
+      } else {
+         trx_fail_time += fail_time;
+         ++trx_fail_num;
+      }
    }
 
-   void add_success_time( const fc::microseconds& time ) {
-      trx_success_time += time;
-      ++trx_success_num;
+   void add_success_time( const fc::microseconds& time, bool is_transient ) {
+      if( is_transient ) {
+         transient_trx_time += time;
+         ++transient_trx_num;
+      } else {
+         trx_success_time += time;
+         ++trx_success_num;
+      }
    }
 
    void report( const fc::time_point& idle_trx_time, uint32_t block_num ) {
       if( _log.is_enabled( fc::log_level::debug ) ) {
          auto now = fc::time_point::now();
          add_idle_time( now - idle_trx_time );
-         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, other: ${o}us",
+         fc_dlog( _log, "Block #${n} trx idle: ${i}us out of ${t}us, success: ${sn}, ${s}us, fail: ${fn}, ${f}us, transient: ${trans_trx_num}, ${trans_trx_time}us, other: ${o}us",
                   ("n", block_num)
                   ("i", block_idle_time)("t", now - clear_time)("sn", trx_success_num)("s", trx_success_time)
                   ("fn", trx_fail_num)("f", trx_fail_time)
-                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time) );
+                  ("trans_trx_num", transient_trx_num)("trans_trx_time", transient_trx_time)
+                  ("o", (now - clear_time) - block_idle_time - trx_success_time - trx_fail_time - transient_trx_time) );
       }
    }
 
    void clear() {
-      block_idle_time = trx_fail_time = trx_success_time = fc::microseconds{};
-      trx_fail_num = trx_success_num = 0;
+      block_idle_time = trx_fail_time = trx_success_time = transient_trx_time = fc::microseconds{};
+      trx_fail_num = trx_success_num = transient_trx_num = 0;
       clear_time = fc::time_point::now();
    }
 
    fc::microseconds block_idle_time;
    uint32_t trx_success_num = 0;
    uint32_t trx_fail_num = 0;
+   uint32_t transient_trx_num = 0;
    fc::microseconds trx_success_time;
    fc::microseconds trx_fail_time;
+   fc::microseconds transient_trx_time;
    fc::time_point clear_time{fc::time_point::now()};
 };
 
@@ -299,8 +313,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       producer_plugin_impl(boost::asio::io_service& io)
       :_timer(io)
       ,_transaction_ack_channel(app().get_channel<compat::channels::transaction_ack>())
-      ,_ro_write_window_timer(io)
-      ,_ro_read_window_timer(io)
+      ,_ro_timer(io)
       {
       }
 
@@ -324,9 +337,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       push_result push_transaction( const fc::time_point& block_deadline,
                                     const transaction_metadata_ptr& trx,
                                     bool api_trx, bool return_failure_trace,
-                                    next_function<transaction_trace_ptr> next );
+                                    const next_function<transaction_trace_ptr>& next );
       push_result handle_push_result( const transaction_metadata_ptr& trx,
-                                      next_function<transaction_trace_ptr> next,
+                                      const next_function<transaction_trace_ptr>& next,
                                       const fc::time_point& start,
                                       const chain::controller& chain,
                                       const transaction_trace_ptr& trace,
@@ -356,7 +369,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       named_thread_pool<struct prod>                            _thread_pool;
 
       std::atomic<int32_t>                                      _max_transaction_time_ms; // modified by app thread, read by net_plugin thread pool
-      std::atomic<bool>                                         _received_block{false}; // modified by net_plugin thread pool and app thread
+      std::atomic<uint32_t>                                     _received_block{0}; // modified by net_plugin thread pool
       fc::microseconds                                          _max_irreversible_block_age_us;
       int32_t                                                   _produce_time_offset_us = 0;
       int32_t                                                   _last_block_time_offset_us = 0;
@@ -414,39 +427,60 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       
       // ro for read-only
       struct ro_trx_t {
-         packed_transaction_ptr  trx;
-         next_func_t             next;
+         transaction_metadata_ptr trx;
+         next_func_t              next;
       };
-      // The queue storing read-only transactions to be executed by read-only threads
-      struct ro_trx_queue_t {
-         std::mutex              mtx;
-         std::deque<ro_trx_t>    queue;
+      // The queue storing previously exhausted read-only transactions to be re-executed by read-only threads
+      // thread-safe
+      class ro_trx_queue_t {
+      public:
+         void push_front(ro_trx_t&& t) {
+            std::lock_guard g(mtx);
+            queue.push_front(std::move(t));
+         }
+
+         bool empty() const {
+            std::lock_guard g(mtx);
+            return queue.empty();
+         }
+
+         bool pop_front(ro_trx_t& t) {
+            std::unique_lock g(mtx);
+            if (queue.empty())
+               return false;
+            t = queue.front();
+            queue.pop_front();
+            return true;
+         }
+
+      private:
+         mutable std::mutex      mtx;
+         deque<ro_trx_t>         queue;  // boost deque which is faster than std::deque
       };
-      uint16_t                        _ro_thread_pool_size{ 0 };
-      static constexpr uint16_t       _ro_max_eos_vm_oc_threads_allowed{ 8 }; // Due to uncertainty to get total virtual memory size on a 5-level paging system, set a hard limit
+
+      uint32_t                        _ro_thread_pool_size{ 0 };
+      static constexpr uint32_t       _ro_max_eos_vm_oc_threads_allowed{ 8 }; // Due to uncertainty to get total virtual memory size on a 5-level paging system, set a hard limit
       named_thread_pool<struct read>  _ro_thread_pool;
       fc::microseconds                _ro_write_window_time_us{ 200000 };
       fc::microseconds                _ro_read_window_time_us{ 60000 };
       static constexpr fc::microseconds _ro_read_window_minimum_time_us{ 10000 };
       fc::microseconds                _ro_read_window_effective_time_us{ 0 }; // calculated during option initialization
-      boost::asio::deadline_timer     _ro_write_window_timer;
-      boost::asio::deadline_timer     _ro_read_window_timer;
+      std::atomic<int64_t>            _ro_all_threads_exec_time_us; // total time spent by all threads executing transactions. use atomic for simplicity and performance
+      fc::time_point                  _ro_read_window_start_time;
+      fc::time_point                  _ro_window_deadline; // only modified on app thread, read-window deadline or write-window deadline
+      boost::asio::deadline_timer     _ro_timer; // only accessible from the main thread
       fc::microseconds                _ro_max_trx_time_us{ 0 }; // calculated during option initialization
-      ro_trx_queue_t                  _ro_trx_queue;
-      std::atomic<uint32_t>            _ro_num_active_trx_exec_tasks{ 0 };
-      std::vector<std::future<bool>>  _ro_trx_exec_tasks_fut;
+      ro_trx_queue_t                  _ro_exhausted_trx_queue;
+      std::atomic<uint32_t>           _ro_num_active_exec_tasks{ 0 };
+      std::vector<std::future<bool>>  _ro_exec_tasks_fut;
 
       void start_write_window();
       void switch_to_write_window();
       void switch_to_read_window();
-      bool read_only_trx_execution_task();
-      bool process_read_only_transaction(const packed_transaction_ptr& trx,
-                                         const next_function<transaction_trace_ptr>& next,
-                                         const fc::time_point& read_window_start_time);
-      bool push_read_only_transaction(const transaction_metadata_ptr& trx,
-                                      next_function<transaction_trace_ptr> next,
-                                      const fc::time_point& read_window_start_time);
-      
+      bool read_only_execution_task(uint32_t pending_block_num);
+      void repost_exhausted_transactions(const fc::time_point& deadline);
+      bool push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next);
+
       void consider_new_watermark( account_name producer, uint32_t block_num, block_timestamp_type timestamp) {
          auto itr = _producer_watermarks.find( producer );
          if( itr != _producer_watermarks.end() ) {
@@ -480,6 +514,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       }
 
       void on_irreversible_block( const signed_block_ptr& lib ) {
+         EOS_ASSERT(app().executor().is_write_window(), producer_exception, "write window is expected for on_irreversible_block signal");
          _irreversible_block_time = lib->timestamp.to_time_point();
          const chain::controller& chain = chain_plug->chain();
 
@@ -639,12 +674,12 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                          transaction_metadata::trx_type trx_type,
                                          bool return_failure_traces,
                                          next_function<transaction_trace_ptr> next) {
-         if ( trx_type == transaction_metadata::trx_type::read_only && _ro_thread_pool_size > 0 ) {
-            // Parallel read-only trx execution enabled.
-            // Store the transaction in read-only-trx queue so that it is
-            // executed in read window
-            std::lock_guard<std::mutex> g( _ro_trx_queue.mtx );
-            _ro_trx_queue.queue.push_back({trx, std::move(next)});
+         if ( trx_type == transaction_metadata::trx_type::read_only ) {
+            // Post all read only trxs to read_only queue for execution.
+            auto trx_metadata = transaction_metadata::create_no_recover_keys( trx, transaction_metadata::trx_type::read_only );
+            app().executor().post(priority::low, exec_queue::read_only, [this, trx{std::move(trx_metadata)}, next{std::move(next)}]() mutable {
+               push_read_only_transaction( std::move(trx), std::move(next) );
+            } );
             return;
          }
 
@@ -677,7 +712,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                                           next{std::move(next)}, trx=trx]() mutable {
             if( future.valid() ) {
                future.wait();
-               app().executor().post( priority::low, exec_queue::general, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
+               app().executor().post( priority::low, exec_queue::read_write, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
                   auto start = fc::time_point::now();
                   auto idle_time = start - self->_idle_trx_time;
                   self->_time_tracker.add_idle_time( idle_time );
@@ -688,7 +723,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                      self->log_trx_results( trx, nullptr, ex, 0, start, is_transient );
                      next( std::move(ex) );
                      self->_idle_trx_time = fc::time_point::now();
-                     self->_time_tracker.add_fail_time(self->_idle_trx_time - start);
+                     auto dur = self->_idle_trx_time - start;
+                     self->_time_tracker.add_fail_time(dur, is_transient);
                   };
                   try {
                      auto result = future.get();
@@ -709,7 +745,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       bool process_incoming_transaction_async(const transaction_metadata_ptr& trx,
                                               bool api_trx,
                                               bool return_failure_trace,
-                                              next_function<transaction_trace_ptr> next) {
+                                              const next_function<transaction_trace_ptr>& next) {
          bool exhausted = false;
          chain::controller& chain = chain_plug->chain();
          try {
@@ -789,7 +825,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          exhausted
       };
 
-      inline bool should_interrupt_start_block( const fc::time_point& deadline ) const;
+      inline bool should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const;
       start_block_result start_block();
 
       fc::time_point calculate_pending_block_time() const;
@@ -892,12 +928,12 @@ void producer_plugin::set_program_options(
           "Number of worker threads in producer thread pool")
          ("snapshots-dir", bpo::value<bfs::path>()->default_value("snapshots"),
           "the location of the snapshots directory (absolute path or relative to application data dir)")
-         ("read-only-threads", bpo::value<uint16_t>()->default_value(my->_ro_thread_pool_size),
-          "Number of worker threads in read-only transaction execution thread pool")
+         ("read-only-threads", bpo::value<uint32_t>(),
+          "Number of worker threads in read-only execution thread pool")
          ("read-only-write-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_write_window_time_us.count()),
-          "time in microseconds the write window lasts")
+          "Time in microseconds the write window lasts.")
          ("read-only-read-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_read_window_time_us.count()),
-          "time in microseconds the read window lasts")
+          "Time in microseconds the read window lasts.")
          ;
    config_file_options.add(producer_options);
 }
@@ -941,6 +977,7 @@ if( options.count(op_name) ) { \
    } \
 }
 
+using namespace std::chrono_literals;
 void producer_plugin::plugin_initialize(const boost::program_options::variables_map& options)
 { try {
    handle_sighup(); // Sets loggers
@@ -1073,11 +1110,22 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
-   my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint16_t>();
+   if ( options.count( "read-only-threads" ) ) {
+      my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint32_t>();
+   } else if ( my->_producers.empty() ) {
+      if( options.count( "plugin" ) ) {
+         const auto& v = options.at( "plugin" ).as<std::vector<std::string>>();
+         auto i = std::find_if( v.cbegin(), v.cend(), []( const std::string& p ) { return p == "eosio::chain_api_plugin"; } );
+         if( i != v.cend() ) {
+            // default to 3 threads for non producer nodes running chain_api_plugin if not specified
+            my->_ro_thread_pool_size = 3;
+            ilog( "chain_api_plugin configured, defaulting read-only-threads to ${t}", ("t", my->_ro_thread_pool_size) );
+         }
+      }
+   }
+   EOS_ASSERT( test_mode_ || my->_ro_thread_pool_size == 0 || my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
    // only initialize other read-only options when read-only thread pool is enabled
    if ( my->_ro_thread_pool_size > 0 ) {
-      EOS_ASSERT( my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
-
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if (chain.is_eos_vm_oc_enabled()) {
          // EOS VM OC requires 4.2TB Virtual for each executing thread. Make sure the memory
@@ -1098,37 +1146,37 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
          }
 
          EOS_ASSERT( vm_total_kb > 0, plugin_config_exception, "Unable to get system virtual memory size (not a Linux?), therefore cannot determine if the system has enough virtual memory for multi-threaded read-only transactions on EOS VM OC");
+         EOS_ASSERT( vm_total_kb > vm_used_kb, plugin_config_exception, "vm total (${t}) must be greater than vm used (${u})", ("t", vm_total_kb)("u", vm_used_kb));
+         uint32_t num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000;
          // reserve 1 for the app thread, 1 for anything else which might use VM
-         int num_threads_supported = (vm_total_kb - vm_used_kb) / 4200000000 - 2;
-         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported));
-         EOS_ASSERT( num_threads_supported >= my->_ro_thread_pool_size, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads supported for EOS VM OC (${supp}) by the system virtual memory size", ("th", my->_ro_thread_pool_size) ("supp", num_threads_supported) );
-
-         if ( my->_ro_thread_pool_size > my->_ro_max_eos_vm_oc_threads_allowed ) {
-            wlog("--read-only-threads (${th}) greater than maximum number of threads allowed (${allowed}) for EOS Vm OC. Set it to ${allowed}", ("th", my->_ro_thread_pool_size) ("allowed", my->_ro_max_eos_vm_oc_threads_allowed));
-            my->_ro_thread_pool_size = my->_ro_max_eos_vm_oc_threads_allowed;
-         }
+         EOS_ASSERT( num_threads_supported > 2, plugin_config_exception, "With the EOS VM OC configured, there is not enough system virtual memory to support the required minimum of 3 threads (1 for main thread, 1 for read-only, and 1 for anything else), vm total: ${t}, vm used: ${u}", ("t", vm_total_kb)("u", vm_used_kb));
+         num_threads_supported -= 2;
+         auto actual_threads_allowed = std::min(my->_ro_max_eos_vm_oc_threads_allowed, num_threads_supported);
+         ilog("vm total in kb: ${total}, vm used in kb: ${used}, number of EOS VM OC threads supported ((vm total - vm used)/4.2 TB - 2): ${supp}, max allowed: ${max}, actual allowed: ${actual}", ("total", vm_total_kb) ("used", vm_used_kb) ("supp", num_threads_supported) ("max", my->_ro_max_eos_vm_oc_threads_allowed)("actual", actual_threads_allowed));
+         EOS_ASSERT( my->_ro_thread_pool_size <= actual_threads_allowed, plugin_config_exception, "--read-only-threads (${th}) greater than number of threads allowed for EOS VM OC (${allowed})", ("th", my->_ro_thread_pool_size) ("allowed", actual_threads_allowed) );
       }
 #endif
-
-      my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
-      my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
-      EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
-      my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
-
-      // Make sure a read-only transaction can finish within the read
-      // window if scheduled at the very beginning of the window.
-      // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
-      // for safety marging
-      if ( my->_max_transaction_time_ms.load() > 0 ) {
-         EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
-         my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
-      } else {
-         // _max_transaction_time_ms can be set to negative in testing (for unlimited)
-         my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
-      }
-      ilog("_ro_thread_pool_size ${s},  _ro_write_window_time_us ${ww}, _ro_read_window_time_us ${rw}, _ro_max_trx_time_us ${t}, _ro_read_window_effective_time_us ${w}", ("s", my->_ro_thread_pool_size) ("ww", my->_ro_write_window_time_us) ("rw", my->_ro_read_window_time_us) ("t", my->_ro_max_trx_time_us) ("w", my->_ro_read_window_effective_time_us));
    }
 
+   my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
+   my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
+   EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
+   my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
+
+   // Make sure a read-only transaction can finish within the read
+   // window if scheduled at the very beginning of the window.
+   // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
+   // for safety margin
+   if ( my->_max_transaction_time_ms.load() > 0 ) {
+      EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
+      my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
+   } else {
+      // _max_transaction_time_ms can be set to negative in testing (for unlimited)
+      my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
+   }
+   ilog("ro_thread_pool_size ${s}, ro_write_window_time_us ${ww}, ro_read_window_time_us ${rw}, ro_max_trx_time_us ${t}, ro_read_window_effective_time_us ${w}",
+        ("s", my->_ro_thread_pool_size)("ww", my->_ro_write_window_time_us)("rw", my->_ro_read_window_time_us)("t", my->_ro_max_trx_time_us)("w", my->_ro_read_window_effective_time_us));
+
    my->_incoming_block_sync_provider = app().get_method<incoming::methods::block_sync>().register_provider(
          [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
       return my->on_incoming_block(block, block_id, bsp);
@@ -1174,7 +1222,6 @@ void producer_plugin::plugin_startup()
       app().quit();
    } );
 
-
    chain::controller& chain = my->chain_plug->chain();
    EOS_ASSERT( my->_producers.empty() || chain.get_read_mode() != chain::db_read_mode::IRREVERSIBLE, plugin_config_exception,
               "node cannot have any producer-name configured because block production is impossible when read_mode is \"irreversible\"" );
@@ -1209,15 +1256,27 @@ void producer_plugin::plugin_startup()
    }
 
    if ( my->_ro_thread_pool_size > 0 ) {
+      std::atomic<uint32_t> num_threads_started = 0;
       my->_ro_thread_pool.start( my->_ro_thread_pool_size,
          []( const fc::exception& e ) {
-            fc_elog( _log, "Exception in read-only transaction thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
+            fc_elog( _log, "Exception in read-only thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
             app().quit();
          },
          [&]() {
             chain.init_thread_local_data();
+            ++num_threads_started;
          });
 
+      // This will be changed with std::latch or std::atomic<>::wait
+      // when C++20 is used.
+      auto time_slept_ms = 0;
+      constexpr auto max_time_slept_ms = 1000;
+      while ( num_threads_started.load() < my->_ro_thread_pool_size && time_slept_ms < max_time_slept_ms ) {
+         std::this_thread::sleep_for( 1ms );
+         ++time_slept_ms;
+      }
+      EOS_ASSERT(num_threads_started.load() == my->_ro_thread_pool_size, producer_exception, "read-only threads failed to start. num_threads_started: ${n}, time_slept_ms: ${t}ms", ("n", num_threads_started.load())("t", time_slept_ms));
+
       my->start_write_window();
    }
 
@@ -1777,12 +1836,12 @@ fc::time_point producer_plugin_impl::calculate_block_deadline( const fc::time_po
    }
 }
 
-bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline ) const {
+bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline, uint32_t pending_block_num ) const {
    if( _pending_block_mode == pending_block_mode::producing ) {
       return deadline <= fc::time_point::now();
    }
    // if we can produce then honor deadline so production starts on time
-   return (!_producers.empty() && deadline <= fc::time_point::now()) || _received_block;
+   return (!_producers.empty() && deadline <= fc::time_point::now()) || (_received_block >= pending_block_num);
 }
 
 producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
@@ -1802,6 +1861,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
    const fc::time_point now = fc::time_point::now();
    const fc::time_point block_time = calculate_pending_block_time();
+   const uint32_t pending_block_num = hbs->block_num + 1;
    const fc::time_point preprocess_deadline = calculate_block_deadline(block_time);
 
    const pending_block_mode previous_pending_mode = _pending_block_mode;
@@ -1867,7 +1927,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    if (_pending_block_mode == pending_block_mode::producing) {
       const auto start_block_time = block_time - fc::microseconds( config::block_interval_us );
       if( now < start_block_time ) {
-         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", hbs->block_num + 1)("bt", block_time) );
+         fc_dlog(_log, "Not producing block waiting for production window ${n} ${bt}", ("n", pending_block_num)("bt", block_time) );
          // start_block_time instead of block_time because schedule_delayed_production_loop calculates next block time from given time
          schedule_delayed_production_loop(weak_from_this(), calculate_producer_wake_up_time(start_block_time));
          return start_block_result::waiting_for_production;
@@ -1881,7 +1941,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    }
 
    fc_dlog(_log, "Starting block #${n} at ${time} producer ${p}",
-           ("n", hbs->block_num + 1)("time", now)("p", scheduled_producer.producer_name));
+           ("n", pending_block_num)("time", now)("p", scheduled_producer.producer_name));
 
    try {
       uint16_t blocks_to_confirm = 0;
@@ -1945,7 +2005,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             std::swap( features_to_activate, protocol_features_to_activate );
             _protocol_features_signaled = true;
             ilog( "signaling activation of the following protocol features in block ${num}: ${features_to_activate}",
-                  ("num", hbs->block_num + 1)("features_to_activate", features_to_activate) );
+                  ("num", pending_block_num)("features_to_activate", features_to_activate) );
          }
       }
 
@@ -1971,7 +2031,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
          if( !remove_expired_blacklisted_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
          if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(),
-                                                  [&](){ return should_interrupt_start_block( preprocess_deadline ); } ) ) {
+                                                  [&](){ return should_interrupt_start_block( preprocess_deadline, pending_block_num ); } ) ) {
             return start_block_result::exhausted;
          }
 
@@ -1994,9 +2054,11 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
             process_scheduled_and_incoming_trxs( scheduled_trx_deadline, incoming_itr );
          }
 
+         repost_exhausted_transactions( preprocess_deadline );
+
          if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit()
             return start_block_result::failed;
-         if ( should_interrupt_start_block( preprocess_deadline ) || block_is_exhausted() ) {
+         if ( should_interrupt_start_block( preprocess_deadline, pending_block_num ) || block_is_exhausted() ) {
             return start_block_result::exhausted;
          }
 
@@ -2023,11 +2085,12 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
 {
    chain::controller& chain = chain_plug->chain();
    auto pending_block_time = chain.pending_block_time();
+   auto pending_block_num = chain.pending_block_num();
 
    // remove all expired transactions
    size_t num_expired = 0;
    size_t orig_count = _unapplied_transactions.size();
-   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline); },
+   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline, pending_block_num); },
          [&num_expired]( const packed_transaction_ptr& packed_trx_ptr, trx_enum_type trx_type ) {
             // expired exception is logged as part of next() call
             ++num_expired;
@@ -2051,12 +2114,13 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point
    if(!blacklist_by_expiry.empty()) {
       const chain::controller& chain = chain_plug->chain();
       const auto lib_time = chain.last_irreversible_block_time();
+      const auto pending_block_num = chain.pending_block_num();
 
       int num_expired = 0;
       int orig_count = _blacklisted_transactions.size();
 
       while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= lib_time) {
-         if ( should_interrupt_start_block( deadline ) ) {
+         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
             exhausted = true;
             break;
          }
@@ -2195,9 +2259,10 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
                                         const transaction_metadata_ptr& trx,
                                         bool api_trx,
                                         bool return_failure_trace,
-                                        next_function<transaction_trace_ptr> next )
+                                        const next_function<transaction_trace_ptr>& next )
 {
    auto start = fc::time_point::now();
+   EOS_ASSERT(!trx->is_read_only(), producer_exception, "Unexpected read-only trx");
 
    bool disable_subjective_enforcement = (api_trx && _disable_subjective_api_billing)
                                          || (!api_trx && _disable_subjective_p2p_billing)
@@ -2214,7 +2279,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
          log_trx_results( trx, except_ptr );
          next( except_ptr );
       }
-      _time_tracker.add_fail_time(fc::time_point::now() - start);
+      _time_tracker.add_fail_time(fc::time_point::now() - start, trx->is_transient());
       return push_result{.failed = true};
    }
 
@@ -2234,14 +2299,6 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
       }
    }
 
-   // Make sure db_read_only_mode always to be unset
-   auto db_read_only_mode_guard = fc::make_scoped_exit([trx, &chain]{
-      if( trx->is_read_only() )
-         chain.unset_db_read_only_mode();
-   });
-   if( trx->is_read_only() )
-      chain.set_db_read_only_mode();
-
    auto trace = chain.push_transaction( trx, block_deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );
 
    return handle_push_result(trx, next, start, chain, trace, return_failure_trace, disable_subjective_enforcement, first_auth, sub_bill, prev_billed_cpu_time_us);
@@ -2249,7 +2306,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
 
 producer_plugin_impl::push_result
 producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
-                                          next_function<transaction_trace_ptr> next,
+                                          const next_function<transaction_trace_ptr>& next,
                                           const fc::time_point& start,
                                           const chain::controller& chain,
                                           const transaction_trace_ptr& trace,
@@ -2261,8 +2318,14 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    auto end = fc::time_point::now();
    push_result pr;
    if( trace->except ) {
-      if ( !trx->is_read_only() )
-         _time_tracker.add_fail_time(end - start);
+      // Transient trxs are dry-run or read-only.
+      // Dry-run trxs only run in write window. Read-only trxs can run in
+      // both write and read windows; time spent in read window is counted
+      // by read window summary.
+      if ( app().executor().is_write_window() ) {
+         auto dur = end - start;
+         _time_tracker.add_fail_time(dur, trx->is_transient());
+      }
       if( exception_is_exhausted( *trace->except ) ) {
          if( _pending_block_mode == pending_block_mode::producing ) {
             fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} COULD NOT FIT, tx: ${txid} RETRYING ",
@@ -2302,8 +2365,14 @@ producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
    } else {
       fc_tlog( _log, "Subjective bill for success ${a}: ${b} elapsed ${t}us, time ${r}us",
                ("a",first_auth)("b",sub_bill)("t",trace->elapsed)("r", end - start));
-      if ( !trx->is_read_only() )
-         _time_tracker.add_success_time(end - start);
+      // Transient trxs are dry-run or read-only.
+      // Dry-run trxs only run in write window. Read-only trxs can run in
+      // both write and read windows; time spent in read window is counted
+      // by read window summary.
+      if ( app().executor().is_write_window() ) {
+         auto dur = end - start;
+         _time_tracker.add_success_time(dur, trx->is_transient());
+      }
       log_trx_results( trx, trace, start );
       // if producing then trx is in objective cpu account billing
       if (!disable_subjective_enforcement && _pending_block_mode != pending_block_mode::producing) {
@@ -2319,12 +2388,14 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
 {
    bool exhausted = false;
    if( !_unapplied_transactions.empty() ) {
+      const chain::controller& chain = chain_plug->chain();
+      const auto pending_block_num = chain.pending_block_num();
       int num_applied = 0, num_failed = 0, num_processed = 0;
       auto unapplied_trxs_size = _unapplied_transactions.size();
       auto itr     = _unapplied_transactions.unapplied_begin();
       auto end_itr = _unapplied_transactions.unapplied_end();
       while( itr != end_itr ) {
-         if( should_interrupt_start_block( deadline ) ) {
+         if( should_interrupt_start_block( deadline, pending_block_num ) ) {
             exhausted = true;
             break;
          }
@@ -2446,7 +2517,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
          auto trace = chain.push_scheduled_transaction(trx_id, deadline, max_trx_time, 0, false);
          auto end = fc::time_point::now();
          if (trace->except) {
-            _time_tracker.add_fail_time(end - start);
+            _time_tracker.add_fail_time(end - start, false); // delayed transaction cannot be transient
             if (exception_is_exhausted(*trace->except)) {
                if( block_is_exhausted() ) {
                   exhausted = true;
@@ -2466,7 +2537,7 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p
                num_failed++;
             }
          } else {
-            _time_tracker.add_success_time(end - start);
+            _time_tracker.add_success_time(end - start, false); // delayed transaction cannot be transient
             fc_dlog(_trx_successful_trace_log,
                     "[TRX_TRACE] Block ${block_num} for producer ${prod} is ACCEPTING scheduled tx: ${txid}, time: ${r}, auth: ${a}, cpu: ${cpu}",
                     ("block_num", chain.head_block_num() + 1)("prod", get_pending_block_producer())
@@ -2499,8 +2570,10 @@ bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline
    if( itr != end ) {
       size_t processed = 0;
       fc_dlog( _log, "Processing ${n} pending transactions", ("n", _unapplied_transactions.incoming_size()) );
+      const chain::controller& chain = chain_plug->chain();
+      const auto pending_block_num = chain.pending_block_num();
       while( itr != end ) {
-         if ( should_interrupt_start_block( deadline ) ) {
+         if ( should_interrupt_start_block( deadline, pending_block_num ) ) {
             exhausted = true;
             break;
          }
@@ -2543,7 +2616,6 @@ bool producer_plugin_impl::block_is_exhausted() const {
 // -> Idle
 // --> Start block B (block time y.000) at time x.500
 void producer_plugin_impl::schedule_production_loop() {
-   _received_block = false;
    _timer.cancel();
 
    auto result = start_block();
@@ -2555,7 +2627,7 @@ void producer_plugin_impl::schedule_production_loop() {
       _timer.expires_from_now( boost::posix_time::microseconds( config::block_interval_us  / 10 ));
 
       // we failed to start a block, so try again later?
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
           [weak_this = weak_from_this(), cid = ++_timer_corelation_id]( const boost::system::error_code& ec ) {
              auto self = weak_this.lock();
              if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2608,7 +2680,7 @@ void producer_plugin_impl::schedule_maybe_produce_block( bool exhausted ) {
                ("num", chain.head_block_num() + 1)("desc", block_is_exhausted() ? "Exhausted" : "Deadline exceeded") );
    }
 
-   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
+   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
          [&chain, weak_this = weak_from_this(), cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2650,7 +2722,7 @@ void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr<
       fc_dlog(_log, "Scheduling Speculative/Production Change at ${time}", ("time", wake_up_time));
       static const boost::posix_time::ptime epoch(boost::gregorian::date(1970, 1, 1));
       _timer.expires_at(epoch + boost::posix_time::microseconds(wake_up_time->time_since_epoch().count()));
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
          [weak_this,cid=++_timer_corelation_id](const boost::system::error_code& ec) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2751,8 +2823,8 @@ void producer_plugin_impl::produce_block() {
         ("confs", new_bs->header.confirmed));
 }
 
-void producer_plugin::received_block() {
-   my->_received_block = true;
+void producer_plugin::received_block(uint32_t block_num) {
+   my->_received_block = block_num;
 }
 
 void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id, const packed_transaction_ptr& packed_trx_ptr, const char* reason) const {
@@ -2764,34 +2836,42 @@ void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id,
             ("entire_trx", packed_trx_ptr ? my->chain_plug->get_log_trx(packed_trx_ptr->get_transaction()) : fc::variant{trx_id}));
 }
 
-// Called from app thread
+// Called from only one read_only thread
 void producer_plugin_impl::switch_to_write_window() {
+   if ( _log.is_enabled( fc::log_level::debug ) ) {
+      auto now = fc::time_point::now();
+      fc_dlog( _log, "Read-only threads ${n}, read window ${r}us, total all threads ${t}us",
+               ("n", _ro_thread_pool_size)
+               ("r", now - _ro_read_window_start_time)
+               ("t", _ro_all_threads_exec_time_us.load()));
+   }
+
    // this method can be called from multiple places. it is possible
    // we are already in write window.
    if ( app().executor().is_write_window() ) {
       return;
    }
 
-   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
-   _ro_read_window_timer.cancel();
-   _ro_write_window_timer.cancel();
+   EOS_ASSERT(_ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
 
    start_write_window();
 }
 
-// Called from app thread
+// Called from app thread on plugin_startup
+// Called from only one read_only thread & called from app thread, but not concurrently
 void producer_plugin_impl::start_write_window() {
    chain::controller& chain = chain_plug->chain();
 
    app().executor().set_to_write_window();
    chain.unset_db_read_only_mode();
-   _idle_trx_time = fc::time_point::now();
+   _idle_trx_time = _ro_window_deadline = fc::time_point::now();
 
+   _ro_window_deadline += _ro_write_window_time_us; // not allowed on block producers, so no need to limit to block deadline
    auto expire_time = boost::posix_time::microseconds(_ro_write_window_time_us.count());
-   _ro_write_window_timer.expires_from_now( expire_time );
-   _ro_write_window_timer.async_wait( app().executor().wrap(  // stay on app thread
+   _ro_timer.expires_from_now( expire_time );
+   _ro_timer.async_wait( app().executor().wrap(  // stay on app thread
       priority::high,
-      exec_queue::read_only_trx_safe, // placed in read_only_trx_safe queue so it is ensured to be executed in either window
+      exec_queue::read_write, // placed in read_write so only called from main thread
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
@@ -2800,144 +2880,154 @@ void producer_plugin_impl::start_write_window() {
       }));
 }
 
-// Called from app thread
+// Called only from app thread
 void producer_plugin_impl::switch_to_read_window() {
    EOS_ASSERT(app().executor().is_write_window(),  producer_exception, "expected to be in write window");
-   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0 && _ro_trx_exec_tasks_fut.empty(),  producer_exception, "_ro_trx_exec_tasks_fut expected to be empty" );
+   EOS_ASSERT( _ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "_ro_exec_tasks_fut expected to be empty" );
 
-   _ro_write_window_timer.cancel();
-   _ro_read_window_timer.cancel();
    _time_tracker.add_idle_time( fc::time_point::now() - _idle_trx_time );
 
    // we are in write window, so no read-only trx threads are processing transactions.
-   // _ro_trx_queue is not being accessed. No need to lock.
-   if ( _ro_trx_queue.queue.empty() ) { // no read-only trxs to process. stay in write window
+   if ( app().executor().read_only_queue().empty() ) { // no read-only tasks to process. stay in write window
       start_write_window(); // restart write window timer for next round
       return;
    }
 
-   app().executor().set_to_read_window();
-   chain_plug->chain().set_db_read_only_mode();
-   _received_block = false;
 
-   // start a read-only transaction execution task in each thread in the thread pool
-   _ro_num_active_trx_exec_tasks = _ro_thread_pool_size;
-   for (auto i = 0; i < _ro_thread_pool_size; ++i ) {
-      _ro_trx_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this] () {
-         return self->read_only_trx_execution_task();
+   auto& chain = chain_plug->chain();
+   uint32_t pending_block_num = chain.head_block_num() + 1;
+   _ro_read_window_start_time = fc::time_point::now();
+   _ro_window_deadline = _ro_read_window_start_time + _ro_read_window_effective_time_us;
+   app().executor().set_to_read_window(_ro_thread_pool_size,
+      [received_block=&_received_block, pending_block_num, ro_window_deadline=_ro_window_deadline]() {
+         return fc::time_point::now() >= ro_window_deadline || (received_block->load() >= pending_block_num); // should_exit()
+      });
+   chain.set_db_read_only_mode();
+   _ro_all_threads_exec_time_us = 0;
+
+   // start a read-only execution task in each thread in the thread pool
+   _ro_num_active_exec_tasks = _ro_thread_pool_size;
+   _ro_exec_tasks_fut.resize(0);
+   for (uint32_t i = 0; i < _ro_thread_pool_size; ++i ) {
+      _ro_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this, pending_block_num] () {
+         return self->read_only_execution_task(pending_block_num);
       }) );
    }
 
    auto expire_time = boost::posix_time::microseconds(_ro_read_window_time_us.count());
-   _ro_read_window_timer.expires_from_now( expire_time );
-   _ro_read_window_timer.async_wait( app().executor().wrap(  // stay on app thread
+   _ro_timer.expires_from_now( expire_time );
+   // Needs to be on read_only because that is what is being processed until switch_to_write_window().
+   _ro_timer.async_wait( app().executor().wrap(
       priority::high,
-      exec_queue::read_only_trx_safe,
+      exec_queue::read_only,
       [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
          auto self = weak_this.lock();
          if( self && ec != boost::asio::error::operation_aborted ) {
             // use future to make sure all read-only tasks finished before switching to write window
-            for ( auto& task: self->_ro_trx_exec_tasks_fut ) {
+            for ( auto& task: self->_ro_exec_tasks_fut ) {
                task.get();
             }
-            self->_ro_trx_exec_tasks_fut.clear();
+            self->_ro_exec_tasks_fut.clear();
+            // will be executed from the main app thread because all read-only threads are idle now
             self->switch_to_write_window();
           } else if ( self ) {
-             self->_ro_trx_exec_tasks_fut.clear();
+             self->_ro_exec_tasks_fut.clear();
           }
        }));
 }
 
-// Called from a read only trx thread. Run in parallel with app and other read only trx threads
-bool producer_plugin_impl::read_only_trx_execution_task() {
-   auto start = fc::time_point::now();
-   auto read_window_deadline = start + _ro_read_window_effective_time_us;
-   // We have 4 ways to break out the while loop:
+// Called from a read only thread. Run in parallel with app and other read only threads
+bool producer_plugin_impl::read_only_execution_task(uint32_t pending_block_num) {
+   // We have 3 ways to break out the while loop:
    // 1. pass read window deadline
-   // 2. Net_plugin receives a block
-   // 3. No more transactions in the read-only trx queue
-   // 4. A transaction execution is exhaused
-   while ( fc::time_point::now() < read_window_deadline && !_received_block ) {
-      std::unique_lock<std::mutex> lck( _ro_trx_queue.mtx );
-      if ( _ro_trx_queue.queue.empty() ) {
-         break;
-      }
-      auto trx = _ro_trx_queue.queue.front();
-      _ro_trx_queue.queue.pop_front();
-      lck.unlock();
-      
-      auto retry = process_read_only_transaction( trx.trx, trx.next, start );
-      if ( retry ) {
-         lck.lock();
-         _ro_trx_queue.queue.push_front(trx);
-         // Do not schedule new execution
+   // 2. net_plugin receives a block
+   // 3. no read-only tasks to execute
+   while ( fc::time_point::now() < _ro_window_deadline && _received_block < pending_block_num ) {
+      bool more = app().executor().execute_highest_read_only(); // blocks until all read only threads are idle
+      if ( !more ) {
          break;
       }
    }
 
    // If all tasks are finished, do not wait until end of read window; switch to write window now.
-   if ( --_ro_num_active_trx_exec_tasks == 0 ) {
-      // Do switching on app thread to serialize
-      app().executor().post( priority::high, exec_queue::read_only_trx_safe, [self=this]() {
-         self->_ro_trx_exec_tasks_fut.clear();
+   if ( --_ro_num_active_exec_tasks == 0 ) {
+      // Needs to be on read_only because that is what is being processed until switch_to_write_window().
+      app().executor().post( priority::high, exec_queue::read_only, [self=this]() {
+         self->_ro_exec_tasks_fut.clear();
+         // will be executed from the main app thread because all read-only threads are idle now
          self->switch_to_write_window();
       } );
+      // last thread post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
+      ro_trx_t t;
+      while( _ro_exhausted_trx_queue.pop_front(t) ) {
+         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
+            push_read_only_transaction( std::move(trx), std::move(next) );
+         } );
+      }
    }
 
    return true;
 }
 
-// Called from a read only trx thread. Run in parallel with app and other read only trx threads
-// Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::process_read_only_transaction(const packed_transaction_ptr& trx, const next_function<transaction_trace_ptr>& next, const fc::time_point& read_window_start_time) {
-   chain::controller& chain = chain_plug->chain();
-   auto future = transaction_metadata::start_recover_keys( trx, _thread_pool.get_executor(),
-                                                           chain.get_chain_id(), fc::microseconds::maximum(),
-                                                           transaction_metadata::trx_type::read_only,
-                                                           chain.configured_subjective_signature_length_limit() );
-   auto exception_handler = [&next](fc::exception_ptr ex) {
-      next( std::move(ex) );
-      return false;
-   };
-   future.wait();
-   try {
-      auto trx_metadata = future.get();
-      return push_read_only_transaction( trx_metadata, next, read_window_start_time );
-   } CATCH_AND_CALL(exception_handler);
-   return false;
+// Called from app thread during start block.
+// Reschedule any exhausted read-only transactions from the last block
+void producer_plugin_impl::repost_exhausted_transactions(const fc::time_point& deadline) {
+   if ( !_ro_exhausted_trx_queue.empty() ) {
+      chain::controller& chain = chain_plug->chain();
+      uint32_t pending_block_num = chain.pending_block_num();
+      // post any exhausted back into read_only queue with slightly higher priority (low+1) so they are executed first
+      ro_trx_t t;
+      while( !should_interrupt_start_block( deadline, pending_block_num ) && _ro_exhausted_trx_queue.pop_front(t) ) {
+         app().executor().post(priority::low+1, exec_queue::read_only, [this, trx{std::move(t.trx)}, next{std::move(t.next)}]() mutable {
+            push_read_only_transaction( std::move(trx), std::move(next) );
+         } );
+      }
+   }
 }
 
-// Called from a read_only_trx execution thread
+// Called from a read_only_trx execution thread, or from app thread when executing exclusively
 // Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::push_read_only_transaction(
-            const transaction_metadata_ptr& trx,
-            next_function<transaction_trace_ptr> next,
-            const fc::time_point& read_window_start_time) {
+bool producer_plugin_impl::push_read_only_transaction(transaction_metadata_ptr trx, next_function<transaction_trace_ptr> next) {
    auto retry = false;
-   chain::controller& chain = chain_plug->chain();
-
-   if( !chain.is_building_block() ) {
-      // try next round
-      return true;
-   }
 
    try {
-      const auto block_deadline = calculate_block_deadline( chain.pending_block_time() );
       auto start = fc::time_point::now();
-      auto time_used_in_read_window_us = ( start - read_window_start_time );
-      if ( time_used_in_read_window_us >= _ro_read_window_time_us ) {
-         // already passed read-window deadline, try next round
+      chain::controller& chain = chain_plug->chain();
+      if ( !chain.is_building_block() ) {
+         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
          return true;
       }
 
-      auto remaining_time_in_read_window_us = _ro_read_window_time_us - time_used_in_read_window_us;
-      // Ensure the trx to finish by the end of read-window.
-      auto window_deadline = std::min( start + remaining_time_in_read_window_us, block_deadline );
+      // When executing a read-only trx on the main thread while in the write window,
+      // need to switch db mode to read only.
+      auto db_read_only_mode_guard = fc::make_scoped_exit([&]{
+         if( app().executor().is_write_window() )
+            chain.unset_db_read_only_mode();
+      });
+
+      if ( app().executor().is_write_window() ) {
+         chain.set_db_read_only_mode();
+         auto idle_time = fc::time_point::now() - _idle_trx_time;
+         _time_tracker.add_idle_time( idle_time );
+      }
+
+      // use read-window/write-window deadline if there are read/write windows, otherwise use block_deadline if only the app thead
+      auto window_deadline = (_ro_thread_pool_size != 0) ? _ro_window_deadline : calculate_block_deadline( chain.pending_block_time() );
+
+      // Ensure the trx to finish by the end of read-window or write-window or block_deadline depending on
       auto trace = chain.push_transaction( trx, window_deadline, _ro_max_trx_time_us, 0, false, 0 );
+      _ro_all_threads_exec_time_us += (fc::time_point::now() - start).count();
       auto pr = handle_push_result(trx, next, start, chain, trace, true /*return_failure_trace*/, true /*disable_subjective_enforcement*/, {} /*first_auth*/, 0 /*sub_bill*/, 0 /*prev_billed_cpu_time_us*/);
       // If a transaction was exhausted, that indicates we are close to
       // the end of read window. Retry in next round.
       retry = pr.trx_exhausted;
+      if( retry ) {
+         _ro_exhausted_trx_queue.push_front( {std::move(trx), std::move(next)} );
+      }
+
+      if ( app().executor().is_write_window() ) {
+         _idle_trx_time = fc::time_point::now();
+      }
    } catch ( const guard_exception& e ) {
       chain_plugin::handle_guard_exception(e);
    } catch ( boost::interprocess::bad_alloc& ) {

plugins/producer_plugin/test/test_read_only_trx.cpp

@@ -79,6 +79,7 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    appbase::scoped_app app;
    fc::temp_directory temp;
    auto temp_dir_str = temp.path().string();
+   producer_plugin::set_test_mode(true);
    
    std::promise<std::tuple<producer_plugin*, chain_plugin*>> plugin_promise;
    std::future<std::tuple<producer_plugin*, chain_plugin*>> plugin_fut = plugin_promise.get_future();
@@ -96,14 +97,19 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    auto chain_id = chain_plug->get_chain_id();
 
    std::atomic<size_t> next_calls = 0;
+   std::atomic<size_t> num_get_account_calls = 0;
    std::atomic<size_t> num_posts = 0;
    std::atomic<size_t> trace_with_except = 0;
    std::atomic<bool> trx_match = true;
-   const size_t num_pushes = 2558;
+   const size_t num_pushes = 4242;
 
    for( size_t i = 1; i <= num_pushes; ++i ) {
       auto ptrx = make_unique_trx( chain_id );
-      app->executor().post( priority::low, exec_queue::general, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
+      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug, &num_get_account_calls]() {
+         chain_plug->get_read_only_api(fc::seconds(90)).get_account(chain_apis::read_only::get_account_params{.account_name=config::system_account_name}, fc::time_point::now()+fc::seconds(90));
+         ++num_get_account_calls;
+      });
+      app->executor().post( priority::low, exec_queue::read_write, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
          ++num_posts;
          bool return_failure_traces = true;
          app->get_method<plugin_interface::incoming::methods::transaction_async>()(ptrx,
@@ -126,12 +132,15 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
                ++next_calls;
         });
       });
+      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug]() {
+         chain_plug->get_read_only_api(fc::seconds(90)).get_consensus_parameters(chain_apis::read_only::get_consensus_parameters_params{}, fc::time_point::now()+fc::seconds(90));
+      });
    }
 
    // Wait long enough such that all transactions are executed
    auto start = fc::time_point::now();
    auto hard_deadline = start + fc::seconds(10); // To protect against waiting forever
-   while ( next_calls < num_pushes && fc::time_point::now() < hard_deadline ){
+   while ( (next_calls < num_pushes || num_get_account_calls < num_pushes) && fc::time_point::now() < hard_deadline ){
       std::this_thread::sleep_for( 100ms );;
    }
 
@@ -141,13 +150,39 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    BOOST_CHECK_EQUAL( trace_with_except, 0 ); // should not have any traces with except in it
    BOOST_CHECK_EQUAL( num_pushes, num_posts );
    BOOST_CHECK_EQUAL( num_pushes, next_calls.load() );
-   BOOST_CHECK( trx_match.load() );  // trace should match the transaction  
+   BOOST_CHECK_EQUAL( num_pushes, num_get_account_calls.load() );
+   BOOST_CHECK( trx_match.load() );  // trace should match the transaction
 }
 
 // test read-only trxs on main thread (no --read-only-threads)
 BOOST_AUTO_TEST_CASE(no_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e" };
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e", "--abi-serializer-max-time-ms=999" };
    test_trxs_common(specific_args);
 }
 
+// test read-only trxs on 1 threads (with --read-only-threads)
+BOOST_AUTO_TEST_CASE(with_1_read_only_threads) {
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
+                                              "--read-only-threads=1",
+                                              "--max-transaction-time=10",
+                                              "--abi-serializer-max-time-ms=999",
+                                              "--read-only-write-window-time-us=100000",
+                                              "--read-only-read-window-time-us=40000",
+                                              "--disable-subjective-billing=true" };
+   test_trxs_common(specific_args);
+}
+
+// test read-only trxs on 16 separate threads (with --read-only-threads)
+BOOST_AUTO_TEST_CASE(with_16_read_only_threads) {
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
+                                              "--read-only-threads=16",
+                                              "--max-transaction-time=10",
+                                              "--abi-serializer-max-time-ms=999",
+                                              "--read-only-write-window-time-us=100000",
+                                              "--read-only-read-window-time-us=40000",
+                                              "--disable-subjective-billing=true" };
+   test_trxs_common(specific_args);
+}
+
+
 BOOST_AUTO_TEST_SUITE_END()

plugins/producer_plugin/test/test_snapshot_scheduler.cpp

@@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
             fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
             std::vector<const char*> argv =
                   {"test", "--data-dir", temp.c_str(), "--config-dir", temp.c_str(),
-                   "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true"};
+                   "-p", "eosio", "-e", "--disable-subjective-billing=true"};
             appbase::app().initialize<chain_plugin, producer_plugin>(argv.size(), (char**) &argv[0]);
             appbase::app().startup();
             plugin_promise.set_value(

plugins/producer_plugin/test/test_trx_full.cpp

@@ -112,7 +112,7 @@ BOOST_AUTO_TEST_CASE(producer) {
          fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
          std::vector<const char*> argv =
                {"test", "--data-dir", temp_dir_str.c_str(), "--config-dir", temp_dir_str.c_str(),
-                "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true" };
+                "-p", "eosio", "-e", "--disable-subjective-billing=true" };
          app->initialize<chain_plugin, producer_plugin>( argv.size(), (char**) &argv[0] );
          app->startup();
          plugin_promise.set_value(

plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp

@@ -103,7 +103,7 @@ public:
    void pop_entry(bool call_send = true) {
       send_queue.erase(send_queue.begin());
       sending = false;
-      if (call_send)
+      if (call_send || !send_queue.empty())
          send();
    }
 

plugins/test_control_api_plugin/test_control_api_plugin.cpp

@@ -51,7 +51,7 @@ void test_control_api_plugin::plugin_startup() {
 
    app().get_plugin<http_plugin>().add_api({
       TEST_CONTROL_RW_CALL(kill_node_on_producer, 202, http_params_types::params_required)
-   }, appbase::exec_queue::general);
+   }, appbase::exec_queue::read_write);
 }
 
 void test_control_api_plugin::plugin_shutdown() {}

plugins/wallet_api_plugin/wallet_api_plugin.cpp

@@ -115,7 +115,7 @@ void wallet_api_plugin::plugin_startup() {
             INVOKE_R_R_R(wallet_mgr, list_keys, std::string, std::string), 200),
        CALL_WITH_400(wallet, wallet_mgr, get_public_keys,
             INVOKE_R_V(wallet_mgr, get_public_keys), 200)
-   }, appbase::exec_queue::general);
+   }, appbase::exec_queue::read_write);
 }
 
 void wallet_api_plugin::plugin_initialize(const variables_map& options) {

programs/keosd/main.cpp

@@ -111,7 +111,7 @@ int main(int argc, char** argv)
                        [&a=app](string, string, url_response_callback cb) {
          cb(200, fc::time_point::maximum(), fc::variant(fc::variant_object()));
          a->quit();
-      }, appbase::exec_queue::general );
+      }, appbase::exec_queue::read_write );
       app->startup();
       app->exec();
    } catch (const fc::exception& e) {

scripts/install_deps.sh

@@ -1,6 +1,27 @@
-#!/usr/bin/env bash
-
+#!/bin/bash
 apt-get update
 apt-get update --fix-missing
-DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
-apt-get -y install zip unzip libncurses5 wget git build-essential cmake curl libgmp-dev libssl-dev libzstd-dev time zlib1g-dev libtinfo-dev bzip2 libbz2-dev python3 python3-numpy file
+export DEBIAN_FRONTEND='noninteractive'
+export TZ='Etc/UTC'
+apt-get install -y \
+    build-essential \
+    bzip2 \
+    cmake \
+    curl \
+    file \
+    git \
+    libbz2-dev \
+    libcurl4-openssl-dev \
+    libgmp-dev \
+    libncurses5 \
+    libssl-dev \
+    libtinfo-dev \
+    libzstd-dev \
+    python3 \
+    python3-numpy \
+    time \
+    tzdata \
+    unzip \
+    wget \
+    zip \
+    zlib1g-dev

scripts/pinned_build.sh

@@ -1,151 +1,185 @@
-#!/usr/bin/env bash
+#!/bin/bash
+set -eo pipefail
 
 echo "Leap Pinned Build"
 
 if [[ "$(uname)" == "Linux" ]]; then
-   if [[ -e /etc/os-release ]]; then
-      # obtain NAME and other information
-      . /etc/os-release
-      if [[ ${NAME} != "Ubuntu" ]]; then
-         echo "Currently only supporting Ubuntu based builds. Proceed at your own risk."
-      fi
-   else
-       echo "Currently only supporting Ubuntu based builds. /etc/os-release not found. Your Linux distribution is not supported. Proceed at your own risk."
-   fi
+    if [[ -e /etc/os-release ]]; then
+        # obtain NAME and other information
+        . /etc/os-release
+        if [[ "${NAME}" != "Ubuntu" ]]; then
+            echo "Currently only supporting Ubuntu based builds. Proceed at your own risk."
+        fi
+    else
+        echo "Currently only supporting Ubuntu based builds. /etc/os-release not found. Your Linux distribution is not supported. Proceed at your own risk."
+    fi
 else
     echo "Currently only supporting Ubuntu based builds. Your architecture is not supported. Proceed at your own risk."
 fi
 
-if [ $# -eq 0 ] || [ -z "$1" ]
-   then
-      echo "Please supply a directory for the build dependencies to be placed and a directory for leap build and a value for the number of jobs to use for building."
-      echo "The binary packages will be created and placed into the leap build directory."
-      echo "./pinned_build.sh <dependencies directory> <leap build directory> <1-100>"
-      exit -1
+if [ $# -eq 0 ] || [ -z "$1" ]; then
+    echo "Please supply a directory for the build dependencies to be placed and a directory for leap build and a value for the number of jobs to use for building."
+    echo "The binary packages will be created and placed into the leap build directory."
+    echo "./pinned_build.sh <dependencies directory> <leap build directory> <1-100>"
+    exit 255
 fi
 
-CORE_SYM=EOS
+export CORE_SYM='EOS'
 # CMAKE_C_COMPILER requires absolute path
-DEP_DIR=`realpath $1`
-LEAP_DIR=$2
-JOBS=$3
+DEP_DIR="$(realpath "$1")"
+LEAP_DIR="$2"
+JOBS="$3"
 CLANG_VER=11.0.1
 BOOST_VER=1.70.0
 LLVM_VER=7.1.0
-ARCH=`uname -m`
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
 START_DIR="$(pwd)"
 
 
 pushdir() {
-   DIR=$1
-   mkdir -p ${DIR}
-   pushd ${DIR} &> /dev/null
+    DIR="$1"
+    mkdir -p "${DIR}"
+    pushd "${DIR}" &> /dev/null
 }
 
 popdir() {
-   EXPECTED=$1
-   D=`popd`
-   popd &> /dev/null
-   echo ${D}
-   D=`eval echo $D | head -n1 | cut -d " " -f1`
+    EXPECTED="$1"
+    D="$(popd)"
+    popd &> /dev/null
+    echo "${D}"
+    D="$(eval echo "$D" | head -n1 | cut -d " " -f1)"
 
-   # -ef compares absolute paths
-   if ! [[ ${D} -ef ${EXPECTED} ]]; then
-     echo "Directory is not where expected EXPECTED=${EXPECTED} at ${D}"
-     exit 1 
-   fi
+    # -ef compares absolute paths
+    if ! [[ "${D}" -ef "${EXPECTED}" ]]; then
+        echo "Directory is not where expected EXPECTED=${EXPECTED} at ${D}"
+        exit 1
+    fi
 }
 
 try(){
-   output=$($@)
-   res=$?
-   if [[ ${res} -ne 0 ]]; then
-      exit -1
-   fi
+    "$@"
+    res=$?
+    if [[ ${res} -ne 0 ]]; then
+        exit 255
+    fi
 }
 
 install_clang() {
-   CLANG_DIR=$1
-   if [ ! -d "${CLANG_DIR}" ]; then
-      echo "Installing Clang ${CLANG_VER} @ ${CLANG_DIR}"
-      mkdir -p ${CLANG_DIR}
-      CLANG_FN=clang+llvm-${CLANG_VER}-x86_64-linux-gnu-ubuntu-16.04.tar.xz
-      try wget -O ${CLANG_FN} https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VER}/${CLANG_FN}
-      try tar -xvf ${CLANG_FN} -C ${CLANG_DIR}
-      pushdir ${CLANG_DIR}
-      mv clang+*/* .
-      popdir ${DEP_DIR}
-      rm ${CLANG_FN}
-   fi
-   export PATH=${CLANG_DIR}/bin:$PATH
-   export CLANG_DIR=${CLANG_DIR}
+    CLANG_DIR="$1"
+    if [ ! -d "${CLANG_DIR}" ]; then
+        echo "Installing Clang ${CLANG_VER} @ ${CLANG_DIR}"
+        mkdir -p "${CLANG_DIR}"
+        CLANG_FN="clang+llvm-${CLANG_VER}-x86_64-linux-gnu-ubuntu-16.04.tar.xz"
+        try wget -O "${CLANG_FN}" "https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VER}/${CLANG_FN}"
+        try tar -xvf "${CLANG_FN}" -C "${CLANG_DIR}"
+        pushdir "${CLANG_DIR}"
+        mv clang+*/* .
+        popdir "${DEP_DIR}"
+        rm "${CLANG_FN}"
+    fi
+    export PATH="${CLANG_DIR}/bin:$PATH"
+    export CLANG_DIR="${CLANG_DIR}"
 }
 
 install_llvm() {
-   LLVM_DIR=$1
-   if [ ! -d "${LLVM_DIR}" ]; then
-      echo "Installing LLVM ${LLVM_VER} @ ${LLVM_DIR}"
-      mkdir -p ${LLVM_DIR}
-      try wget -O llvm-${LLVM_VER}.src.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/llvm-${LLVM_VER}.src.tar.xz
-      try tar -xvf llvm-${LLVM_VER}.src.tar.xz
-      pushdir "${LLVM_DIR}.src"
-      pushdir build
-      try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=${LLVM_DIR} -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=Off -DLLVM_ENABLE_RTTI=On -DLLVM_ENABLE_TERMINFO=Off -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO ..
-      try make -j${JOBS} 
-      try make -j${JOBS} install
-      popdir "${LLVM_DIR}.src"
-      popdir ${DEP_DIR}
-      rm -rf ${LLVM_DIR}.src 
-      rm llvm-${LLVM_VER}.src.tar.xz
-   fi
-   export LLVM_DIR=${LLVM_DIR}
+    LLVM_DIR="$1"
+    if [ ! -d "${LLVM_DIR}" ]; then
+        echo "Installing LLVM ${LLVM_VER} @ ${LLVM_DIR}"
+        mkdir -p "${LLVM_DIR}"
+        try wget -O "llvm-${LLVM_VER}.src.tar.xz" "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/llvm-${LLVM_VER}.src.tar.xz"
+        try tar -xvf "llvm-${LLVM_VER}.src.tar.xz"
+        pushdir "${LLVM_DIR}.src"
+        pushdir build
+        try cmake -DCMAKE_TOOLCHAIN_FILE="${SCRIPT_DIR}/pinned_toolchain.cmake" -DCMAKE_INSTALL_PREFIX="${LLVM_DIR}" -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=Off -DLLVM_ENABLE_RTTI=On -DLLVM_ENABLE_TERMINFO=Off -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO ..
+        try make -j "${JOBS}"
+        try make -j "${JOBS}" install
+        popdir "${LLVM_DIR}.src"
+        popdir "${DEP_DIR}"
+        rm -rf "${LLVM_DIR}.src"
+        rm "llvm-${LLVM_VER}.src.tar.xz"
+    fi
+    export LLVM_DIR="${LLVM_DIR}"
 }
 
 install_boost() {
-   BOOST_DIR=$1
+    BOOST_DIR="$1"
 
-   if [ ! -d "${BOOST_DIR}" ]; then
-      echo "Installing Boost ${BOOST_VER} @ ${BOOST_DIR}"
-      try wget -O boost_${BOOST_VER//\./_}.tar.gz https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VER}/source/boost_${BOOST_VER//\./_}.tar.gz
-      try tar --transform="s:^boost_${BOOST_VER//\./_}:boost_${BOOST_VER//\./_}patched:" -xvzf boost_${BOOST_VER//\./_}.tar.gz -C ${DEP_DIR}
-      pushdir ${BOOST_DIR}
-      patch -p1 < "${SCRIPT_DIR}/0001-beast-fix-moved-from-executor.patch"
-      try ./bootstrap.sh -with-toolset=clang --prefix=${BOOST_DIR}/bin
-      ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I${CLANG_DIR}/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fPIE' linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j${JOBS} install
-      popdir ${DEP_DIR}
-      rm boost_${BOOST_VER//\./_}.tar.gz
-   fi
-   export BOOST_DIR=${BOOST_DIR}
+    if [ ! -d "${BOOST_DIR}" ]; then
+        echo "Installing Boost ${BOOST_VER} @ ${BOOST_DIR}"
+        try wget -O "boost_${BOOST_VER//\./_}.tar.gz" "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VER}/source/boost_${BOOST_VER//\./_}.tar.gz"
+        try tar --transform="s:^boost_${BOOST_VER//\./_}:boost_${BOOST_VER//\./_}patched:" -xvzf "boost_${BOOST_VER//\./_}.tar.gz" -C "${DEP_DIR}"
+        pushdir "${BOOST_DIR}"
+        patch -p1 < "${SCRIPT_DIR}/0001-beast-fix-moved-from-executor.patch"
+        try ./bootstrap.sh -with-toolset=clang --prefix="${BOOST_DIR}/bin"
+        ./b2 toolset=clang cxxflags="-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I\${CLANG_DIR}/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fPIE" linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j "${JOBS}" install
+        popdir "${DEP_DIR}"
+        rm "boost_${BOOST_VER//\./_}.tar.gz"
+    fi
+    export BOOST_DIR="${BOOST_DIR}"
 }
 
-pushdir ${DEP_DIR}
+pushdir "${DEP_DIR}"
 
-install_clang ${DEP_DIR}/clang-${CLANG_VER}
-install_llvm ${DEP_DIR}/llvm-${LLVM_VER}
-install_boost ${DEP_DIR}/boost_${BOOST_VER//\./_}patched
+install_clang "${DEP_DIR}/clang-${CLANG_VER}"
+install_llvm "${DEP_DIR}/llvm-${LLVM_VER}"
+install_boost "${DEP_DIR}/boost_${BOOST_VER//\./_}patched"
 
 # go back to the directory where the script starts
-popdir ${START_DIR}
+popdir "${START_DIR}"
 
-pushdir ${LEAP_DIR}
+pushdir "${LEAP_DIR}"
 
 # build Leap
 echo "Building Leap ${SCRIPT_DIR}"
-try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=${LLVM_DIR}/lib/cmake -DCMAKE_PREFIX_PATH=${BOOST_DIR}/bin ${SCRIPT_DIR}/..
+try cmake -DCMAKE_TOOLCHAIN_FILE="${SCRIPT_DIR}/pinned_toolchain.cmake" -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="${LLVM_DIR}/lib/cmake" -DCMAKE_PREFIX_PATH="${BOOST_DIR}/bin" "${SCRIPT_DIR}/.."
 
-try make -j${JOBS}
+try make -j "${JOBS}"
 try cpack
 
-echo " .----------------.  .----------------.  .----------------.  .----------------. ";
-echo "| .--------------. || .--------------. || .--------------. || .--------------. |";
-echo "| |   _____      | || |  _________   | || |      __      | || |   ______     | |";
-echo "| |  |_   _|     | || | |_   ___  |  | || |     /  \     | || |  |_   __ \   | |";
-echo "| |    | |       | || |   | |_  \_|  | || |    / /\ \    | || |    | |__) |  | |";
-echo "| |    | |   _   | || |   |  _|  _   | || |   / ____ \   | || |    |  ___/   | |";
-echo "| |   _| |__/ |  | || |  _| |___/ |  | || | _/ /    \ \_ | || |   _| |_      | |";
-echo "| |  |________|  | || | |_________|  | || ||____|  |____|| || |  |_____|     | |";
-echo "| |              | || |              | || |              | || |              | |";
-echo "| '--------------' || '--------------' || '--------------' || '--------------' |";
-echo " '----------------'  '----------------'  '----------------'  '----------------' ";
-echo "Leap has successfully built and constructed its packages.  You should be able to find the packages at ${LEAP_DIR}.  Enjoy!!!"
+# art generated with DALL-E (https://openai.com/blog/dall-e), then fed through ASCIIart.club (https://asciiart.club) with permission
+cat <<'TXT' # BASH interpolation must remain disabled for the ASCII art to print correctly
+
+                                ,▄▄A`
+                             _╓▄██`
+                          ╓▄▓▓▀▀`
+                        ╓▓█▀╓▄▓
+                        ▓▌▓▓▓▀
+                      ,▄▓███▓H
+                    _╨╫▀╚▀╠▌`╙¥,
+                  ╓«    _╟▄▄   `½,                              ╓▄▄╦▄≥_
+                 ╙▓╫╬▒R▀▀╙▀▀▓φ_ «_╙Y╥▄mmMM#╦▄,_          ,╓╦mM╩╨╙╙╙\`║═
+                   ``       `▀▄__╫▓▓╨`    _```"""*ⁿⁿ^`````Ω,        `╟∩
+                              ╙▌▓▓"`    ,«ñ`            ╔╬▓▌⌂        ╔▌
+                               ╙█▌,,╔╗M╨,░             `  "╫▓m_      ╟H      _
+                        _,,,,__,╠█▓▒`  .╣▌µ _       _.╓╔▄▄▓█▓▓N_     ╙▀╩KKM╙╟▓N
+                      ,▄▓█▓████▀▀▀╙▀╓╔φ»█▓▓Ñ╦«, :»»µ╦▓▓█▀└╙▀███▓╥__   _,╓▄▓▓▓M▓`,
+                  __╓Φ▓█╫▓▓▓▓▓▓▓▓▓▓▓▓▓▀K▀▀███▓▓▓▓▓▓▀▀╙       `▀▀▀▓▄▄K╨╙└   `▀▌╙█▄*.
+                ,╓Φ▓▓▀▄▓▀`         ╙▀╙                                       ╙▓╙╙▓▄*
+              .▄▓╫▀╦▄▀`                                                       ╙▓╙µ╙▀
+             ▄▓▀╨▓▓╨_                                                          `▀▄▄M
+            _█▌▄▌╙`                                                               `╙
+             ╙└`
+
+
+                Ñ▓▓▓▓          ¢▄▄▄▄▄▄▄▄▄▄         ,           ,,,,,,,,,,,_
+               _╫▓▓█▌          ╟▓████████▀       ╓╣▓▌_         ╠╫▓█▓▓▓▓█▓▓▓▓▄
+               _▓▓▓█▌          ╟╫██▄,,,,_       æ▄███▓▄        ▐║████▀╨╨▀▓███▌
+               :╫▓▓█▌          ╟╢████████▓    ,╬███████▓,      ▐║████▓▄▄▓▓███▀
+               :╫▓▓█▌_        _╟║███▀▀▀▀▀▀   ╓╫███╣╫╬███▓N_    j▐██▀▀▀▀▀▀▀▀▀`
+    ___________]╫▓▓█▓φ╓╓╓╓╓╓,__╟╣█▌▌,,,,,_ _╬▓████████████▓▄_  ▐M█▌
+    _ _________]╫▌▓█████████▓▓▄╟╣████████▓▄╣██▓▀^    _ ╙▓██▓▓╗__M█▓
+      ___   _ _ ▀╣▀╣╩╩╩╩╩╩╩▀▀▀▀╙▀▀▀▀▀▀▀▀▀▀▀▀▀▀`          ╙▀▀▀▀╩═╩▀▀
+     _ __ __ _____ ___  ____ _  __  _  ____ ___
+    ____ ____ __  _ _  _ _ _ _ __   _ __ __
+    __ _ ________  ___ ________   ___ _ _ _      _
+       _ __  __     _ __ ____ _     _ ____ _   _ _       _
+     _  _ _     ____ ____ _ _    _ __  _ _ _ __
+
+---
+Leap has successfully built and constructed its packages. You should be able to
+find the packages at:
+TXT
+echo "${LEAP_DIR}"
+echo
+echo 'Thank you, fam!!'
+echo

tests/CMakeLists.txt

@@ -125,8 +125,12 @@ add_test(NAME nodeos_protocol_feature_test COMMAND tests/nodeos_protocol_feature
 set_property(TEST nodeos_protocol_feature_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME compute_transaction_test COMMAND tests/compute_transaction_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST compute_transaction_test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read_only_trx_test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST read_only_trx_test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 0 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST read-only-trx-basic-test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST read-only-trx-parallel-test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST read-only-trx-parallel-eos-vm-oc-test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME subjective_billing_test COMMAND tests/subjective_billing_test.py -v -p 2 -n 4 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST subjective_billing_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME get_account_test COMMAND tests/get_account_test.py -v -p 2 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})

tests/TestHarness/Cluster.py

@@ -22,8 +22,11 @@ from .Node import BlockType
 from .Node import Node
 from .WalletMgr import WalletMgr
 from .launch_transaction_generators import TransactionGeneratorsLauncher, TpsTrxGensConfig
-from .libc import unshare, CLONE_NEWNET
-from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
+try:
+    from .libc import unshare, CLONE_NEWNET
+    from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
+except:
+    pass
 
 # Protocol Feature Setup Policy
 class PFSetupPolicy:
@@ -1447,7 +1450,7 @@ class Cluster(object):
     def killall(self, kill=True, silent=True, allInstances=False):
         """Kill cluster nodeos instances. allInstances will kill all nodeos instances running on the system."""
         signalNum=9 if kill else 15
-        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"python3 {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
+        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"{sys.executable} {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         if 0 != subprocess.call(cmd.split(), stdout=Utils.FNull):
             if not silent: Utils.Print("Launcher failed to shut down eos cluster.")
@@ -1693,7 +1696,6 @@ class Cluster(object):
                             waitToComplete:bool=False, abiFile=None, actionsData=None, actionsAuths=None):
         Utils.Print("Configure txn generators")
         node=self.getNode(nodeId)
-        p2pListenPort = self.getNodeP2pPort(nodeId)
         info = node.getInfo()
         chainId = info['chain_id']
         lib_id = info['last_irreversible_block_id']
@@ -1702,13 +1704,12 @@ class Cluster(object):
         tpsLimitPerGenerator=tpsPerGenerator
 
         self.preExistingFirstTrxFiles = glob.glob(f"{Utils.DataDir}/first_trx_*.txt")
-
-        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=targetTps, tpsLimitPerGenerator=tpsLimitPerGenerator)
+        connectionPairList = [f"{self.host}:{self.getNodeP2pPort(nodeId)}"]
+        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=targetTps, tpsLimitPerGenerator=tpsLimitPerGenerator, connectionPairList=connectionPairList)
         self.trxGenLauncher = TransactionGeneratorsLauncher(chainId=chainId, lastIrreversibleBlockId=lib_id,
                                                     contractOwnerAccount=contractOwnerAcctName, accts=','.join(map(str, acctNamesList)),
                                                     privateKeys=','.join(map(str, acctPrivKeysList)), trxGenDurationSec=durationSec, logDir=Utils.DataDir,
-                                                    abiFile=abiFile, actionsData=actionsData, actionsAuths=actionsAuths,
-                                                    peerEndpoint=self.host, port=p2pListenPort, tpsTrxGensConfig=tpsTrxGensConfig)
+                                                    abiFile=abiFile, actionsData=actionsData, actionsAuths=actionsAuths, tpsTrxGensConfig=tpsTrxGensConfig)
 
         Utils.Print("Launch txn generators and start generating/sending transactions")
         self.trxGenLauncher.launch(waitToComplete=waitToComplete)
@@ -1729,4 +1730,7 @@ class Cluster(object):
                 for line in f:
                     firstTrxs.append(line.rstrip('\n'))
         Utils.Print(f"first transactions: {firstTrxs}")
-        node.waitForTransactionsInBlock(firstTrxs)
+        status = node.waitForTransactionsInBlock(firstTrxs)
+        if status is None:
+            Utils.Print('ERROR: Failed to spin up transaction generators: never received first transactions')
+        return status

tests/TestHarness/Node.py

@@ -61,7 +61,7 @@ class Node(Transactions):
             self.fetchBlock = lambda blockNum: self.processUrllibRequest("trace_api", "get_block", {"block_num":blockNum}, silentErrors=False, exitOnError=True)
             self.fetchKeyCommand = lambda: "[transaction][transaction_header][ref_block_num]"
             self.fetchRefBlock = lambda trans: trans["transaction_header"]["ref_block_num"]
-            self.cleosLimit = "--time-limit 99999"
+            self.cleosLimit = "--time-limit 999"
             self.fetchHeadBlock = lambda node, headBlock: node.processUrllibRequest("chain", "get_block_info", {"block_num":headBlock}, silentErrors=False, exitOnError=True)
 
     def __str__(self):
@@ -484,6 +484,10 @@ class Node(Transactions):
 
     def scheduleSnapshot(self):
         return self.processUrllibRequest("producer", "schedule_snapshot")
+    
+    def scheduleSnapshotAt(self, sbn):
+        param = { "start_block_num": sbn, "end_block_num": sbn }
+        return self.processUrllibRequest("producer", "schedule_snapshot", param)
 
     # kill all existing nodeos in case lingering from previous test
     @staticmethod

tests/TestHarness/WalletMgr.py

@@ -12,9 +12,9 @@ from .testUtils import Utils
 Wallet=namedtuple("Wallet", "name password host port")
 # pylint: disable=too-many-instance-attributes
 class WalletMgr(object):
-    __walletLogOutFile=f"{Utils.TestLogRoot}/test_keosd_out.log"
-    __walletLogErrFile=f"{Utils.TestLogRoot}/test_keosd_err.log"
     __walletDataDir=f"{Utils.TestLogRoot}/test_wallet_0"
+    __walletLogOutFile=f"{__walletDataDir}/test_keosd_out.log"
+    __walletLogErrFile=f"{__walletDataDir}/test_keosd_err.log"
     __MaxPort=9999
 
     # pylint: disable=too-many-arguments
@@ -83,6 +83,9 @@ class WalletMgr(object):
         cmd="%s --data-dir %s --config-dir %s --unlock-timeout=999999 --http-server-address=%s:%d --http-max-response-time-ms 99999 --verbose-http-errors" % (
             Utils.EosWalletPath, WalletMgr.__walletDataDir, WalletMgr.__walletDataDir, self.host, self.port)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+        if not os.path.isdir(WalletMgr.__walletDataDir):
+            if Utils.Debug: Utils.Print(f"Creating dir {WalletMgr.__walletDataDir} in dir: {os.getcwd()}")
+            os.mkdir(WalletMgr.__walletDataDir)
         with open(WalletMgr.__walletLogOutFile, 'w') as sout, open(WalletMgr.__walletLogErrFile, 'w') as serr:
             popen=subprocess.Popen(cmd.split(), stdout=sout, stderr=serr)
             self.__walletPid=popen.pid
@@ -299,6 +302,5 @@ class WalletMgr(object):
 
     @staticmethod
     def cleanup():
-        dataDir=WalletMgr.__walletDataDir
-        if os.path.isdir(dataDir) and os.path.exists(dataDir):
+        if os.path.isdir(WalletMgr.__walletDataDir) and os.path.exists(WalletMgr.__walletDataDir):
             shutil.rmtree(WalletMgr.__walletDataDir)

tests/TestHarness/launch_transaction_generators.py

@@ -10,16 +10,19 @@ sys.path.append(harnessPath)
 
 from .testUtils import Utils
 from pathlib import Path
+from re import sub
 
 Print = Utils.Print
 
 class TpsTrxGensConfig:
 
-    def __init__(self, targetTps: int, tpsLimitPerGenerator: int):
+    def __init__(self, targetTps: int, tpsLimitPerGenerator: int, connectionPairList: list):
         self.targetTps: int = targetTps
         self.tpsLimitPerGenerator: int = tpsLimitPerGenerator
-
-        self.numGenerators = math.ceil(self.targetTps / self.tpsLimitPerGenerator)
+        self.connectionPairList = connectionPairList
+        self.numConnectionPairs = len(self.connectionPairList)
+        round_to_multiple = lambda num, multiple: math.ceil(num/multiple) * multiple
+        self.numGenerators = round_to_multiple(math.ceil(self.targetTps / self.tpsLimitPerGenerator), self.numConnectionPairs)
         self.initialTpsPerGenerator = math.floor(self.targetTps / self.numGenerators)
         self.modTps = self.targetTps % self.numGenerators
         self.cleanlyDivisible = self.modTps == 0
@@ -35,8 +38,7 @@ class TpsTrxGensConfig:
 class TransactionGeneratorsLauncher:
 
     def __init__(self, chainId: int, lastIrreversibleBlockId: int, contractOwnerAccount: str, accts: str, privateKeys: str, trxGenDurationSec: int, logDir: str,
-                 abiFile: Path, actionsData, actionsAuths,
-                 peerEndpoint: str, port: int, tpsTrxGensConfig: TpsTrxGensConfig):
+                 abiFile: Path, actionsData, actionsAuths, tpsTrxGensConfig: TpsTrxGensConfig):
         self.chainId = chainId
         self.lastIrreversibleBlockId = lastIrreversibleBlockId
         self.contractOwnerAccount  = contractOwnerAccount
@@ -46,14 +48,14 @@ class TransactionGeneratorsLauncher:
         self.tpsTrxGensConfig = tpsTrxGensConfig
         self.logDir = logDir
         self.abiFile = abiFile
-        self.actionsData=actionsData
-        self.actionsAuths=actionsAuths
-        self.peerEndpoint = peerEndpoint
-        self.port = port
+        self.actionsData = actionsData
+        self.actionsAuths = actionsAuths
 
     def launch(self, waitToComplete=True):
         self.subprocess_ret_codes = []
+        connectionPairIter = 0
         for id, targetTps in enumerate(self.tpsTrxGensConfig.targetTpsPerGenList):
+            connectionPair = self.tpsTrxGensConfig.connectionPairList[connectionPairIter].rsplit(":")
             popenStringList = [
                                 './tests/trx_generator/trx_generator',
                                 '--generator-id', f'{id}',
@@ -65,8 +67,8 @@ class TransactionGeneratorsLauncher:
                                 '--trx-gen-duration', f'{self.trxGenDurationSec}',
                                 '--target-tps', f'{targetTps}',
                                 '--log-dir', f'{self.logDir}',
-                                '--peer-endpoint', f'{self.peerEndpoint}',
-                                '--port', f'{self.port}']
+                                '--peer-endpoint', f'{connectionPair[0]}',
+                                '--port', f'{connectionPair[1]}']
             if self.abiFile is not None and self.actionsData is not None and self.actionsAuths is not None:
                 popenStringList.extend(['--abi-file', f'{self.abiFile}',
                                         '--actions-data', f'{self.actionsData}',
@@ -74,6 +76,7 @@ class TransactionGeneratorsLauncher:
             if Utils.Debug:
                 Print(f"Running trx_generator: {' '.join(popenStringList)}")
             self.subprocess_ret_codes.append(subprocess.Popen(popenStringList))
+            connectionPairIter = (connectionPairIter + 1) % len(self.tpsTrxGensConfig.connectionPairList)
         exitCodes=None
         if waitToComplete:
             exitCodes = [ret_code.wait() for ret_code in self.subprocess_ret_codes]
@@ -100,20 +103,20 @@ def parseArgs():
     parser.add_argument("abi_file", type=str, help="The path to the contract abi file to use for the supplied transaction action data")
     parser.add_argument("actions_data", type=str, help="The json actions data file or json actions data description string to use")
     parser.add_argument("actions_auths", type=str, help="The json actions auth file or json actions auths description string to use, containting authAcctName to activePrivateKey pairs.")
-    parser.add_argument("peer_endpoint", type=str, help="set the peer endpoint to send transactions to", default="127.0.0.1")
-    parser.add_argument("port", type=int, help="set the peer endpoint port to send transactions to", default=9876)
+    parser.add_argument("connection_pair_list", type=str, help="Comma separated list of endpoint:port combinations to send transactions to", default="localhost:9876")
     args = parser.parse_args()
     return args
 
 def main():
     args = parseArgs()
+    connectionPairList = sub('[\s+]', '', args.connection_pair_list)
+    connectionPairList = connectionPairList.rsplit(',')
 
     trxGenLauncher = TransactionGeneratorsLauncher(chainId=args.chain_id, lastIrreversibleBlockId=args.last_irreversible_block_id,
                                                    contractOwnerAccount=args.contract_owner_account, accts=args.accounts,
                                                    privateKeys=args.priv_keys, trxGenDurationSec=args.trx_gen_duration, logDir=args.log_dir,
                                                    abiFile=args.abi_file, actionsData=args.actions_data, actionsAuths=args.actions_auths,
-                                                   peerEndpoint=args.peer_endpoint, port=args.port,
-                                                   tpsTrxGensConfig=TpsTrxGensConfig(targetTps=args.target_tps, tpsLimitPerGenerator=args.tps_limit_per_generator))
+                                                   tpsTrxGensConfig=TpsTrxGensConfig(targetTps=args.target_tps, tpsLimitPerGenerator=args.tps_limit_per_generator, connectionPairList=connectionPairList))
 
 
     exit_codes = trxGenLauncher.launch()

tests/TestHarness/queries.py

@@ -118,10 +118,10 @@ class NodeosQueries:
         # could be a transaction response
         if cntxt.hasKey("processed"):
             cntxt.add("processed")
-            cntxt.add("action_traces")
-            cntxt.index(0)
             if not cntxt.isSectionNull("except"):
                 return "no_block"
+            cntxt.add("action_traces")
+            cntxt.index(0)
             return cntxt.add("block_num")
 
         # or what the trace api plugin returns
@@ -242,7 +242,7 @@ class NodeosQueries:
         assert(isinstance(transId, str))
         exitOnErrorForDelayed=not delayedRetry and exitOnError
         timeout=3
-        cmdDesc="get transaction_trace"
+        cmdDesc=self.fetchTransactionCommand()
         cmd="%s %s" % (cmdDesc, transId)
         msg="(transaction id=%s)" % (transId);
         for i in range(0,(int(60/timeout) - 1)):
@@ -295,8 +295,8 @@ class NodeosQueries:
         refBlockNum=None
         key=""
         try:
-            key="[transaction][transaction_header][ref_block_num]"
-            refBlockNum=trans["transaction_header"]["ref_block_num"]
+            key = self.fetchKeyCommand()
+            refBlockNum = self.fetchRefBlock(trans)
             refBlockNum=int(refBlockNum)+1
         except (TypeError, ValueError, KeyError) as _:
             Utils.Print("transaction%s not found. Transaction: %s" % (key, trans))
@@ -347,8 +347,8 @@ class NodeosQueries:
 
     def getTable(self, contract, scope, table, exitOnError=False):
         cmdDesc = "get table"
-        cmd="%s %s %s %s" % (cmdDesc, contract, scope, table)
-        msg="contract=%s, scope=%s, table=%s" % (contract, scope, table);
+        cmd=f"{cmdDesc} {self.cleosLimit} {contract} {scope} {table}"
+        msg=f"contract={contract}, scope={scope}, table={table}"
         return self.processCleosCmd(cmd, cmdDesc, exitOnError=exitOnError, exitMsg=msg)
 
     def getTableAccountBalance(self, contract, scope):
@@ -529,7 +529,7 @@ class NodeosQueries:
             return m.group(1)
         except subprocess.CalledProcessError as ex:
             end=time.perf_counter()
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             Utils.Print("ERROR: Exception during code hash retrieval.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
             return None
 
@@ -580,8 +580,9 @@ class NodeosQueries:
         except subprocess.CalledProcessError as ex:
             if not silentErrors:
                 end=time.perf_counter()
-                msg=ex.output.decode("utf-8")
-                errorMsg="Exception during \"%s\". Exception message: %s.  cmd Duration=%.3f sec. %s" % (cmdDesc, msg, end-start, exitMsg)
+                out=ex.output.decode("utf-8")
+                msg=ex.stderr.decode("utf-8")
+                errorMsg="Exception during \"%s\". Exception message: %s.  stdout: %s.  cmd Duration=%.3f sec. %s" % (cmdDesc, msg, out, end-start, exitMsg)
                 if exitOnError:
                     Utils.cmdError(errorMsg)
                     Utils.errorExit(errorMsg)

tests/TestHarness/transactions.py

@@ -107,7 +107,7 @@ class Transactions(NodeosQueries):
                 self.trackCmdTransaction(trans, reportStatus=reportStatus)
         except subprocess.CalledProcessError as ex:
             end=time.perf_counter()
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             Utils.Print("ERROR: Exception during funds transfer.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
             if exitOnError:
                 Utils.cmdError("could not transfer \"%s\" from %s to %s" % (amountStr, source, destination))
@@ -131,7 +131,7 @@ class Transactions(NodeosQueries):
                 Utils.Print("cmd Duration: %.3f sec" % (end-start))
         except subprocess.CalledProcessError as ex:
             end=time.perf_counter()
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             Utils.Print("ERROR: Exception during spawn of funds transfer.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
             if exitOnError:
                 Utils.cmdError("could not transfer \"%s\" from %s to %s" % (amountStr, source, destination))
@@ -158,15 +158,15 @@ class Transactions(NodeosQueries):
         except subprocess.CalledProcessError as ex:
             if not shouldFail:
                 end=time.perf_counter()
-                msg=ex.output.decode("utf-8")
-                Utils.Print("ERROR: Exception during set contract.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
+                out=ex.output.decode("utf-8")
+                msg=ex.stderr.decode("utf-8")
+                Utils.Print("ERROR: Exception during set contract. stderr: %s.  stdout: %s.  cmd Duration: %.3f sec." % (msg, out, end-start))
                 return None
             else:
                 retMap={}
                 retMap["returncode"]=ex.returncode
                 retMap["cmd"]=ex.cmd
                 retMap["output"]=ex.output
-                retMap["stdout"]=ex.stdout
                 retMap["stderr"]=ex.stderr
                 return retMap
 
@@ -213,7 +213,7 @@ class Transactions(NodeosQueries):
                 Utils.Print("cmd Duration: %.3f sec" % (end-start))
             return (NodeosQueries.getTransStatus(retTrans) == 'executed', retTrans)
         except subprocess.CalledProcessError as ex:
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             if not silentErrors:
                 end=time.perf_counter()
                 Utils.Print("ERROR: Exception during push transaction.  cmd Duration=%.3f sec.  %s" % (end - start, msg))
@@ -245,7 +245,6 @@ class Transactions(NodeosQueries):
         except subprocess.CalledProcessError as ex:
             msg=ex.stderr.decode("utf-8")
             output=ex.output.decode("utf-8")
-            msg=ex.output.decode("utf-8")
             if not silentErrors:
                 end=time.perf_counter()
                 Utils.Print("ERROR: Exception during push message. stderr: %s. stdout: %s.  cmd Duration=%.3f sec." % (msg, output, end - start))

tests/auto_bp_peering_test.py

@@ -35,8 +35,7 @@ args = TestHelper.parse_args({
 Utils.Debug = args.v
 killAll = args.clean_run
 dumpErrorDetails = args.dump_error_details
-# dontKill = args.leave_running
-dontKill = True
+dontKill = args.leave_running
 killEosInstances = not dontKill
 killWallet = not dontKill
 keepLogs = args.keep_logs

tests/cli_test.py

@@ -11,7 +11,7 @@ import time
 import shutil
 import signal
 
-from TestHarness import Account, Cluster, Node, ReturnType, Utils, WalletMgr
+from TestHarness import Account, Node, ReturnType, Utils, WalletMgr
 
 testSuccessful=False
 
@@ -352,7 +352,7 @@ def abi_file_with_nodeos_test():
         os.makedirs(data_dir, exist_ok=True)
         walletMgr = WalletMgr(True)
         walletMgr.launch()
-        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
+        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --max-transaction-time=-1 --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
         node.verifyAlive() # Setting node state to not alive
         node.relaunch(newChain=True, cachePopen=True)
         node.waitForBlock(1)
@@ -403,7 +403,7 @@ def abi_file_with_nodeos_test():
                     os.kill(node.pid, signal.SIGKILL)
         if testSuccessful:
             Utils.Print("Cleanup nodeos data.")
-            shutil.rmtree(data_dir)
+            shutil.rmtree(Utils.DataPath)
 
         if malicious_token_abi_path:
             if os.path.exists(malicious_token_abi_path):

tests/launcher.py

@@ -174,7 +174,7 @@ class launcher(object):
         parser.add_argument('-i', '--timestamp', help='set the timestamp for the first block.  Use "now" to indicate the current time')
         parser.add_argument('-l', '--launch', choices=['all', 'none', 'local'], help='select a subset of nodes to launch.  If not set, the default is to launch all unless an output file is named, in which case none are started.', default='all')
         parser.add_argument('-o', '--output', help='save a copy of the generated topology in this file and exit without launching', dest='topology_filename')
-        parser.add_argument('-k', '--kill', help='retrieve the list of previously started process ids and issue a kill to each')
+        parser.add_argument('-k', '--kill', type=int, help='kill a network as specified in arguments with given signal')
         parser.add_argument('--down', type=comma_separated, help='comma-separated list of node numbers that will be shut down', default=[])
         parser.add_argument('--bounce', type=comma_separated, help='comma-separated list of node numbers that will be restarted', default=[])
         parser.add_argument('--roll', type=comma_separated, help='comma-separated list of host names where the nodes will be rolled to a new version')
@@ -613,6 +613,17 @@ plugin = eosio::chain_api_plugin
                     if relaunch:
                         self.launch(node)
 
+    def kill(self, signum):
+        errorCode = 0
+        for node in self.network.nodes.values():
+            try:
+                with open(node.data_dir_name / f'{Utils.EosServerName}.pid', 'r') as f:
+                    pid = int(f.readline())
+                    self.terminate_wait_pid(pid, signum, raise_if_missing=False)
+            except FileNotFoundError as err:
+                errorCode = 1
+        return errorCode
+
     def start_all(self):
         if self.args.launch.lower() != 'none':
             for instance in self.network.nodes.values():
@@ -628,8 +639,9 @@ plugin = eosio::chain_api_plugin
                     f.write(f'"{node.dot_label}"->"{pname}" [dir="forward"];\n')
             f.write('}')
 
-    def terminate_wait_pid(self, pid, raise_if_missing=True):
-        '''Terminate a non-child process with SIGTERM and wait for it to exit.'''
+    def terminate_wait_pid(self, pid, signum = signal.SIGTERM, raise_if_missing=True):
+        '''Terminate a non-child process with given signal number or with SIGTERM if not
+           provided and wait for it to exit.'''
         if sys.version_info >= (3, 9) and platform.system() == 'Linux': # on our supported platforms, Python 3.9 accompanies a kernel > 5.3
             try:
                 fd = os.pidfd_open(pid)
@@ -640,7 +652,7 @@ plugin = eosio::chain_api_plugin
                 po = select.poll()
                 po.register(fd, select.POLLIN)
                 try:
-                    os.kill(pid, signal.SIGTERM)
+                    os.kill(pid, signum)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -663,7 +675,7 @@ plugin = eosio::chain_api_plugin
                     return min(delay * 2, 0.04)
                 delay = 0.0001
                 try:
-                    os.kill(pid, signal.SIGTERM)
+                    os.kill(pid, signum)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -676,12 +688,16 @@ plugin = eosio::chain_api_plugin
                         return
 
 if __name__ == '__main__':
+    errorCode = 0
     l = launcher(sys.argv[1:])
     if len(l.args.down):
         l.down(l.args.down)
     elif len(l.args.bounce):
         l.bounce(l.args.bounce)
+    elif l.args.kill:
+        errorCode = l.kill(l.args.kill)
     elif l.args.launch == 'all' or l.args.launch == 'local':
         l.start_all()
     for f in glob.glob(Utils.DataPath):
         shutil.rmtree(f)
+    sys.exit(errorCode)

tests/nodeos_contrl_c_test.py

@@ -115,7 +115,7 @@ try:
         errorExit("Failed to kill the seed node")
 
 finally:
-    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=True, cleanRun=True, dumpErrorDetails=True)
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=False, cleanRun=True, dumpErrorDetails=True)
 
 errorCode = 0 if testSuccessful else 1
 exit(errorCode)

tests/nodeos_snapshot_diff_test.py

@@ -126,13 +126,15 @@ try:
     waitForBlock(node0, blockNum, blockType=BlockType.lib)
 
     Print("Configure and launch txn generators")
+    
     targetTpsPerGenerator = 10
     testTrxGenDurationSec=60*30
     cluster.launchTrxGenerators(contractOwnerAcctName=cluster.eosioAccount.name, acctNamesList=[account1Name, account2Name],
                                 acctPrivKeysList=[account1PrivKey,account2PrivKey], nodeId=snapshotNodeId, tpsPerGenerator=targetTpsPerGenerator,
                                 numGenerators=trxGeneratorCnt, durationSec=testTrxGenDurationSec, waitToComplete=False)
 
-    cluster.waitForTrxGeneratorsSpinup(nodeId=snapshotNodeId, numGenerators=trxGeneratorCnt)
+    status = cluster.waitForTrxGeneratorsSpinup(nodeId=snapshotNodeId, numGenerators=trxGeneratorCnt)
+    assert status is not None, "ERROR: Failed to spinup Transaction Generators"
 
     blockNum=node0.getBlockNum(BlockType.head)
     timePerBlock=500
@@ -156,11 +158,8 @@ try:
     minReqPctLeeway=0.60
     minRequiredTransactions=minReqPctLeeway*transactionsPerBlock
     assert steadyStateAvg>=minRequiredTransactions, "Expected to at least receive %s transactions per block, but only getting %s" % (minRequiredTransactions, steadyStateAvg)
-
-    Print("Create snapshot")
-    ret = nodeProg.scheduleSnapshot()
-    assert ret is not None, "Snapshot creation failed"
     
+    Print("Create snapshot (node 0)")
     ret = nodeSnap.createSnapshot()
     assert ret is not None, "Snapshot creation failed"
     ret_head_block_num = ret["payload"]["head_block_num"]
@@ -177,6 +176,29 @@ try:
     Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(snapshotFile), "snapshot to-json", silentErrors=False)
     snapshotFile = snapshotFile + ".json"
 
+    Print("Trim programmable blocklog to snapshot head block num and relaunch programmable node")
+    nodeProg.kill(signal.SIGTERM)
+    output=cluster.getBlockLog(progNodeId, blockLogAction=BlockLogAction.trim, first=0, last=ret_head_block_num, throwException=True)
+    removeState(progNodeId)
+    Utils.rmFromFile(Utils.getNodeConfigDir(progNodeId, "config.ini"), "p2p-peer-address")
+    isRelaunchSuccess = nodeProg.relaunch(chainArg="--replay", addSwapFlags={}, timeout=relaunchTimeout, cachePopen=True)
+    assert isRelaunchSuccess, "Failed to relaunch programmable node"
+
+    Print("Schedule snapshot (node 2)")
+    ret = nodeProg.scheduleSnapshotAt(ret_head_block_num)
+    assert ret is not None, "Snapshot scheduling failed"
+
+    Print("Wait for programmable node lib to advance")
+    waitForBlock(nodeProg, ret_head_block_num+1, blockType=BlockType.lib)
+
+    Print("Kill programmable node")
+    nodeProg.kill(signal.SIGTERM)
+
+    Print("Convert snapshot to JSON")
+    progSnapshotFile = getLatestSnapshot(progNodeId)
+    Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(progSnapshotFile), "snapshot to-json", silentErrors=False)
+    progSnapshotFile = progSnapshotFile + ".json"
+
     Print("Trim irreversible blocklog to snapshot head block num")
     nodeIrr.kill(signal.SIGTERM)
     output=cluster.getBlockLog(irrNodeId, blockLogAction=BlockLogAction.trim, first=0, last=ret_head_block_num, throwException=True)
@@ -204,15 +226,6 @@ try:
     irrSnapshotFile = irrSnapshotFile + ".json"
 
     assert Utils.compareFiles(snapshotFile, irrSnapshotFile), f"Snapshot files differ {snapshotFile} != {irrSnapshotFile}"
-  
-    Print("Kill programmable node")
-    nodeProg.kill(signal.SIGTERM)
-
-    Print("Convert snapshot to JSON")
-    progSnapshotFile = getLatestSnapshot(progNodeId)
-    Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(progSnapshotFile), "snapshot to-json", silentErrors=False)
-    progSnapshotFile = progSnapshotFile + ".json"
-
     assert Utils.compareFiles(progSnapshotFile, irrSnapshotFile), f"Snapshot files differ {progSnapshotFile} != {irrSnapshotFile}"
 
     testSuccessful=True

tests/nodeos_snapshot_forked_test.py

@@ -171,7 +171,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(), \
+    assert prodC.waitForLibToAdvance(60), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
  
     for prodNode in prodNodes:

tests/nodeos_startup_catchup.py

@@ -118,7 +118,8 @@ try:
                                 tpsPerGenerator=targetTpsPerGenerator, numGenerators=trxGeneratorCnt, durationSec=testTrxGenDurationSec,
                                 waitToComplete=False)
 
-    cluster.waitForTrxGeneratorsSpinup(nodeId=node0.nodeId, numGenerators=trxGeneratorCnt)
+    status = cluster.waitForTrxGeneratorsSpinup(nodeId=node0.nodeId, numGenerators=trxGeneratorCnt)
+    assert status is not None, "ERROR: Failed to spinup Transaction Generators"
 
     blockNum=head(node0)
     timePerBlock=500

tests/performance_tests/README.md

@@ -488,8 +488,7 @@ Performance Test Basic Single Test:
 * `abi_file`                    The path to the contract abi file to use for the supplied transaction action data
 * `actions_data`                The json actions data file or json actions data description string to use
 * `actions_auths`               The json actions auth file or json actions auths description string to use, containting authAcctName to activePrivateKey pairs.
-* `peer_endpoint`               set the peer endpoint to send transactions to, default="127.0.0.1"
-* `port`                        set the peer endpoint port to send transactions to, default=9876
+* `connection_pair_list`        Comma separated list of endpoint:port combinations to send transactions to
 </details>
 
 #### Transaction Generator

tests/performance_tests/performance_test_basic.py

@@ -327,7 +327,9 @@ class PerformanceTestBasic:
     def runTpsTest(self) -> PtbTpsTestResult:
         completedRun = False
         self.producerNode = self.cluster.getNode(self.producerNodeId)
-        self.producerP2pPort = self.cluster.getNodeP2pPort(self.producerNodeId)
+        self.connectionPairList = []
+        for producer in range(0, self.clusterConfig.pnodes):
+            self.connectionPairList.append(f"{self.cluster.getNode(producer).host}:{self.cluster.getNodeP2pPort(producer)}")
         self.validationNode = self.cluster.getNode(self.validationNodeId)
         self.wallet = self.walletMgr.create('default')
         self.setupContract()
@@ -371,13 +373,13 @@ class PerformanceTestBasic:
         self.cluster.biosNode.kill(signal.SIGTERM)
 
         self.data.startBlock = self.waitForEmptyBlocks(self.validationNode, self.emptyBlockGoal)
-        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=self.ptbConfig.targetTps, tpsLimitPerGenerator=self.ptbConfig.tpsLimitPerGenerator)
+        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=self.ptbConfig.targetTps, tpsLimitPerGenerator=self.ptbConfig.tpsLimitPerGenerator, connectionPairList=self.connectionPairList)
 
         self.cluster.trxGenLauncher = TransactionGeneratorsLauncher(chainId=chainId, lastIrreversibleBlockId=lib_id, contractOwnerAccount=self.clusterConfig.specifiedContract.account.name,
                                                        accts=','.join(map(str, self.accountNames)), privateKeys=','.join(map(str, self.accountPrivKeys)),
                                                        trxGenDurationSec=self.ptbConfig.testTrxGenDurationSec, logDir=self.trxGenLogDirPath,
                                                        abiFile=abiFile, actionsData=actionsDataJson, actionsAuths=actionsAuthsJson,
-                                                       peerEndpoint=self.producerNode.host, port=self.producerP2pPort, tpsTrxGensConfig=tpsTrxGensConfig)
+                                                       tpsTrxGensConfig=tpsTrxGensConfig)
 
         trxGenExitCodes = self.cluster.trxGenLauncher.launch()
         print(f"Transaction Generator exit codes: {trxGenExitCodes}")
@@ -495,15 +497,17 @@ class PerformanceTestBasic:
             traceback.print_exc()
 
         finally:
+            # Despite keepLogs being hardcoded to False, logs will still appear on test failure in TestLogs
+            # due to testSuccessful being False
             TestHelper.shutdown(
-                self.cluster,
-                self.walletMgr,
-                testSuccessful,
-                self.testHelperConfig._killEosInstances,
-                self.testHelperConfig._killWallet,
-                self.testHelperConfig.keepLogs,
-                self.testHelperConfig.killAll,
-                self.testHelperConfig.dumpErrorDetails
+                cluster=self.cluster,
+                walletMgr=self.walletMgr,
+                testSuccessful=testSuccessful,
+                killEosInstances=self.testHelperConfig._killEosInstances,
+                killWallet=self.testHelperConfig._killWallet,
+                keepLogs=False,
+                cleanRun=self.testHelperConfig.killAll,
+                dumpErrorDetails=self.testHelperConfig.dumpErrorDetails
                 )
 
             if self.ptbConfig.delPerfLogs:

tests/plugin_http_api_test.py

@@ -40,8 +40,8 @@ class PluginHttpTest(unittest.TestCase):
         self.keosd.killall(True)
         WalletMgr.cleanup()
         Node.killAllNodeos()
-        if os.path.exists(self.data_dir):
-            shutil.rmtree(self.data_dir)
+        if os.path.exists(Utils.DataPath):
+            shutil.rmtree(Utils.DataPath)
         if os.path.exists(self.config_dir):
             shutil.rmtree(self.config_dir)
         time.sleep(self.sleep_s)
@@ -56,7 +56,7 @@ class PluginHttpTest(unittest.TestCase):
                         "http_plugin", "db_size_api_plugin", "prometheus_plugin"]
         nodeos_plugins = "--plugin eosio::" +  " --plugin eosio::".join(plugin_names)
         nodeos_flags = (" --data-dir=%s --config-dir=%s --trace-dir=%s --trace-no-abis --access-control-allow-origin=%s "
-                        "--contracts-console --http-validate-host=%s --verbose-http-errors --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
+                        "--contracts-console --http-validate-host=%s --verbose-http-errors --max-transaction-time -1 --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
                         "--p2p-peer-address localhost:9011 --resource-monitor-not-shutdown-on-threshold-exceeded ") % (self.data_dir, self.config_dir, self.data_dir, "\'*\'", "false")
         start_nodeos_cmd = ("%s -e -p eosio %s %s ") % (Utils.EosServerPath, nodeos_plugins, nodeos_flags)
         self.nodeos.launchCmd(start_nodeos_cmd, self.node_id)

tests/read_only_trx_test.py

@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
 
 import random
+import time
+import signal
+import threading
 
-from TestHarness import Account, Cluster, TestHelper, Utils, WalletMgr
+from TestHarness import Account, Cluster, ReturnType, TestHelper, Utils, WalletMgr
 from TestHarness.TestHelper import AppArgs
 
 ###############################################################
@@ -18,6 +21,7 @@ errorExit=Utils.errorExit
 
 appArgs=AppArgs()
 appArgs.add(flag="--read-only-threads", type=int, help="number of read-only threads", default=0)
+appArgs.add(flag="--num-test-runs", type=int, help="number of times to run the tests", default=1)
 appArgs.add_bool(flag="--eos-vm-oc-enable", help="enable eos-vm-oc")
 appArgs.add(flag="--wasm-runtime", type=str, help="if set to eos-vm-oc, must compile with EOSIO_EOS_VM_OC_DEVELOPER", default="eos-vm-jit")
 
@@ -29,6 +33,10 @@ pnodes=args.p
 topo=args.s
 delay=args.d
 total_nodes = pnodes if args.n < pnodes else args.n
+# For this test, we need at least 1 non-producer
+if total_nodes <= pnodes:
+    Print ("non-producing nodes %d must be greater than 0. Force it to %d. producing nodes: %d," % (total_nodes - pnodes, pnodes + 1, pnodes))
+    total_nodes = pnodes + 1
 debug=args.v
 nodesFile=args.nodes_file
 dontLaunch=nodesFile is not None
@@ -37,6 +45,7 @@ dontKill=args.leave_running
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
+numTestRuns=args.num_test_runs
 
 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -45,6 +54,7 @@ if nodesFile is not None:
 
 Utils.Debug=debug
 testSuccessful=False
+errorInThread=False
 
 random.seed(seed) # Use a fixed seed for repeatability.
 cluster=Cluster(walletd=True,unshared=args.unshared)
@@ -54,6 +64,9 @@ EOSIO_ACCT_PRIVATE_DEFAULT_KEY = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79z
 EOSIO_ACCT_PUBLIC_DEFAULT_KEY = "EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV"
 
 try:
+    TestHelper.printSystemInfo("BEGIN")
+    cluster.setWalletMgr(walletMgr)
+
     if dontLaunch: # run test against remote cluster
         jsonStr=None
         with open(nodesFile, "r") as f:
@@ -71,11 +84,24 @@ try:
             cluster.killall(allInstances=killAll)
             cluster.cleanup()
 
-    Print ("producing nodes: %s, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" % (pnodes, total_nodes-pnodes, topo, delay))
+    Print ("producing nodes: %d, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" % (pnodes, total_nodes-pnodes, topo, delay))
 
+    cluster.killall(allInstances=killAll)
+    cluster.cleanup()
     Print("Stand up cluster")
+    # set up read-only options for API node
+    specificExtraNodeosArgs={}
+    # producer nodes will be mapped to 0 through pnodes-1, so the number pnodes is the no-producing API node
+    specificExtraNodeosArgs[pnodes]=" --plugin eosio::net_api_plugin"
+    specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
+    specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
+    if args.eos_vm_oc_enable:
+        specificExtraNodeosArgs[pnodes]+=" --eos-vm-oc-enable"
+    if args.wasm_runtime:
+        specificExtraNodeosArgs[pnodes]+=" --wasm-runtime "
+        specificExtraNodeosArgs[pnodes]+=args.wasm_runtime
     extraNodeosArgs=" --http-max-response-time-ms 990000 --disable-subjective-api-billing false "
-    if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay,extraNodeosArgs=extraNodeosArgs ) is False:
+    if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, specificExtraNodeosArgs=specificExtraNodeosArgs, extraNodeosArgs=extraNodeosArgs ) is False:
         errorExit("Failed to stand up eos cluster.")
 
     Print ("Wait for Cluster stabilization")
@@ -97,7 +123,7 @@ try:
     userAccount = Account(userAccountName)
     userAccount.ownerPublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
     userAccount.activePublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
-    cluster.createAccountAndVerify(userAccount, cluster.eosioAccount)
+    cluster.createAccountAndVerify(userAccount, cluster.eosioAccount, stakeCPU=2000)
 
     noAuthTableContractDir="unittests/test-contracts/no_auth_table"
     noAuthTableWasmFile="no_auth_table.wasm"
@@ -116,28 +142,161 @@ try:
         }
         return apiNode.pushTransaction(trx, opts)
 
-    Print("Insert a user")
-    results = sendTransaction('insert', {"user": userAccountName, "id": 1, "age": 10}, auth=[{"actor": userAccountName, "permission":"active"}])
-    assert(results[0])
-    apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
+    def sendReadOnlyTrxOnThread(startId, numTrxs):
+        Print("start sendReadOnlyTrxOnThread")
 
-    # verify the return value (age) from read-only is the same as created.
-    Print("Send a read-only Get transaction to verify previous Insert")
-    results = sendTransaction('getage', {"user": userAccountName}, opts='--read')
-    assert(results[0])
-    assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 10)
+        global errorInThread
+        errorInThread = False
+        try:
+           for i in range(numTrxs):
+               results = sendTransaction('age', {"user": userAccountName, "id": startId + i}, opts='--read')
+               assert(results[0])
+               assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 25)
+        except Exception as e:
+            Print("Exception in sendReadOnlyTrxOnThread: ", e)
+            errorInThread = True
 
-    # verify non-read-only modification works
-    Print("Send a non-read-only Modify transaction")
-    results = sendTransaction('modify', {"user": userAccountName, "age": 25},        auth=[{"actor": userAccountName, "permission": "active"}])
-    assert(results[0])
-    apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
+    def sendTrxsOnThread(startId, numTrxs, opts=None):
+        Print("sendTrxsOnThread: ", startId, numTrxs, opts)
 
-    # verify 'cleos push action getage "user": "user" --read' works
-    Print("Send a read-only Get action")
-    results = apiNode.pushMessage(testAccountName, 'getage', "{{\"user\": \"{}\"}}".format(userAccountName), opts='--read');
-    assert(results[0])
-    assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 25)
+        global errorInThread
+        errorInThread = False
+        try:
+            for i in range(numTrxs):
+                results = sendTransaction('age', {"user": userAccountName, "id": startId + i}, auth=[{"actor": userAccountName, "permission":"active"}], opts=opts)
+                assert(results[0])
+        except Exception as e:
+            Print("Exception in sendTrxsOnThread: ", e)
+            errorInThread = True
+
+    def doRpc(resource, command, numRuns, fieldIn, expectedValue, code, payload={}):
+        global errorInThread
+        errorInThread = False
+        try:
+            for i in range(numRuns):
+                ret_json = apiNode.processUrllibRequest(resource, command, payload)
+                if code is None:
+                    assert(ret_json["payload"][fieldIn] is not None)
+                    if expectedValue is not None:
+                        assert(ret_json["payload"][fieldIn] == expectedValue)
+                else:
+                    assert(ret_json["code"] == code)
+        except Exception as e:
+            Print("Exception in doRpc: ", e)
+            errorInThread = True
+
+    def runReadOnlyTrxAndRpcInParallel(resource, command, fieldIn=None, expectedValue=None, code=None, payload={}):
+        Print("runReadOnlyTrxAndRpcInParallel: ", command)
+
+        numRuns = 10
+        trxThread = threading.Thread(target = sendReadOnlyTrxOnThread, args = (0, numRuns ))
+        rpcThread = threading.Thread(target = doRpc, args = (resource, command, numRuns, fieldIn, expectedValue, code, payload))
+        trxThread.start()
+        rpcThread.start()
+
+        trxThread.join()
+        rpcThread.join()
+        assert(not errorInThread)
+
+    def mixedOpsTest(opt=None):
+        Print("mixedOpsTest -- opt = ", opt)
+
+        numRuns = 300
+        readOnlyThread = threading.Thread(target = sendReadOnlyTrxOnThread, args = (0, numRuns ))
+        readOnlyThread.start()
+        sendTrxThread = threading.Thread(target = sendTrxsOnThread, args = (numRuns, numRuns, opt))
+        sendTrxThread.start()
+        pushBlockThread = threading.Thread(target = doRpc, args = ("chain", "push_block", numRuns, None, None, 202, {"block":"signed_block"}))
+        pushBlockThread.start()
+
+        readOnlyThread.join()
+        sendTrxThread.join()
+        pushBlockThread.join()
+        assert(not errorInThread)
+
+    def sendMulReadOnlyTrx(numThreads):
+        threadList = []
+        num_trxs_per_thread = 500
+        for i in range(numThreads):
+            thr = threading.Thread(target = sendReadOnlyTrxOnThread, args = (i * num_trxs_per_thread, num_trxs_per_thread ))
+            thr.start()
+            threadList.append(thr)
+        for thr in threadList:
+            thr.join()
+
+    def basicTests():
+        Print("Insert a user")
+        results = sendTransaction('insert', {"user": userAccountName, "id": 1, "age": 10}, auth=[{"actor": userAccountName, "permission":"active"}])
+        assert(results[0])
+        apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
+
+        # verify the return value (age) from read-only is the same as created.
+        Print("Send a read-only Get transaction to verify previous Insert")
+        results = sendTransaction('getage', {"user": userAccountName}, opts='--read')
+        assert(results[0])
+        assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 10)
+
+        # verify non-read-only modification works
+        Print("Send a non-read-only Modify transaction")
+        results = sendTransaction('modify', {"user": userAccountName, "age": 25}, auth=[{"actor": userAccountName, "permission": "active"}])
+        assert(results[0])
+        apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
+
+    def multiReadOnlyTests():
+        Print("Verify multiple read-only Get actions work after Modify")
+        sendMulReadOnlyTrx(numThreads=5)
+
+    def chainApiTests():
+        # verify chain APIs can run in parallel with read-ony transactions
+        runReadOnlyTrxAndRpcInParallel("chain", "get_info", "server_version")
+        runReadOnlyTrxAndRpcInParallel("chain", "get_consensus_parameters", "chain_config")
+        runReadOnlyTrxAndRpcInParallel("chain", "get_activated_protocol_features", "activated_protocol_features")
+        runReadOnlyTrxAndRpcInParallel("chain", "get_block", "block_num", expectedValue=1, payload={"block_num_or_id":1})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_block_info", "block_num", expectedValue=1, payload={"block_num":1})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_account", "account_name", expectedValue=userAccountName, payload = {"account_name":userAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_code", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_code_hash", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_raw_code_and_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_raw_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_producers", "rows", payload = {"json":"true","lower_bound":""})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", "rows", payload = {"json":"true","code":"eosio","scope":"eosio","table":"global"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_table_by_scope", fieldIn="rows", payload = {"json":"true","table":"noauth"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_currency_balance", code=200, payload = {"code":"eosio.token", "account":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_currency_stats", fieldIn="SYS", payload = {"code":"eosio.token", "symbol":"SYS"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_required_keys", code=400)
+        runReadOnlyTrxAndRpcInParallel("chain", "get_transaction_id", code=200, payload = {"ref_block_num":"1"})
+        runReadOnlyTrxAndRpcInParallel("chain", "push_block", code=202, payload = {"block":"signed_block"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_producer_schedule", "active")
+        runReadOnlyTrxAndRpcInParallel("chain", "get_scheduled_transactions", "transactions", payload = {"json":"true","lower_bound":""})
+
+    def netApiTests():
+        # NET APIs
+        runReadOnlyTrxAndRpcInParallel("net", "status", code=201, payload = "localhost")
+        runReadOnlyTrxAndRpcInParallel("net", "connections", code=201)
+        runReadOnlyTrxAndRpcInParallel("net", "connect", code=201, payload = "localhost")
+        runReadOnlyTrxAndRpcInParallel("net", "disconnect", code=201, payload = "localhost")
+
+    def runEverythingParallel():
+        threadList = []
+        threadList.append(threading.Thread(target = multiReadOnlyTests))
+        threadList.append(threading.Thread(target = chainApiTests))
+        threadList.append(threading.Thread(target = netApiTests))
+        threadList.append(threading.Thread(target = mixedOpsTest))
+        for thr in threadList:
+            thr.start()
+        for thr in threadList:
+            thr.join()
+
+    basicTests()
+
+    if args.read_only_threads > 0: # Save test time. No need to run other tests if multi-threaded is not enabled
+        for i in range(numTestRuns):
+            multiReadOnlyTests()
+            chainApiTests()
+            netApiTests()
+            mixedOpsTest()
+            runEverythingParallel()
 
     testSuccessful = True
 finally:

tests/trace_plugin_test.py

@@ -8,6 +8,8 @@ import os
 
 from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr, CORE_SYMBOL
 
+testSuccessful = True
+
 class TraceApiPluginTest(unittest.TestCase):
     sleep_s = 1
     cluster=Cluster(walletd=True, defproduceraPrvtKey=None)
@@ -101,6 +103,8 @@ class TraceApiPluginTest(unittest.TestCase):
                     self.assertIn('memo', prms)
                 break
         self.assertTrue(isTrxInBlockFromTraceApi)
+        global testSuccessful
+        testSuccessful = True
 
     @classmethod
     def setUpClass(self):
@@ -109,7 +113,7 @@ class TraceApiPluginTest(unittest.TestCase):
 
     @classmethod
     def tearDownClass(self):
-        self.cleanEnv(self, shouldCleanup=False)   # not cleanup to save log in case for further investigation
+        self.cleanEnv(self, shouldCleanup=testSuccessful)
 
 if __name__ == "__main__":
     unittest.main()

tests/trx_finality_status_forked_test.py

@@ -184,7 +184,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(), \
+    assert prodC.waitForLibToAdvance(60), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
 
     retStatus = prodC.getTransactionStatus(transId)

unittests/restart_chain_tests.cpp

@@ -237,19 +237,4 @@ BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) {
    BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->digest(), other_trace->action_traces.at(1).receipt->digest());
 }
 
-namespace{
-   struct scoped_temp_path {
-      boost::filesystem::path path;
-      scoped_temp_path() {
-         path = boost::filesystem::unique_path();
-         if (boost::unit_test::framework::master_test_suite().argc >= 2) {
-            path += boost::unit_test::framework::master_test_suite().argv[1];
-         }
-      }
-      ~scoped_temp_path() {
-         boost::filesystem::remove_all(path);
-      }
-   };
-}
-
 BOOST_AUTO_TEST_SUITE_END()

unittests/snapshot_tests.cpp

@@ -412,8 +412,8 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_compatible_versions, SNAPSHOT_SUITE, snapshot
       bfs::copy_file(source_i, source_log_dir / "blocks.index", bfs::copy_option::overwrite_if_exists);
       chain.close();
    }
-
-   auto config = tester::default_config(fc::temp_directory(), legacy_default_max_inline_action_size).first;
+   fc::temp_directory temp_dir;
+   auto config = tester::default_config(temp_dir, legacy_default_max_inline_action_size).first;
    auto genesis = eosio::chain::block_log::extract_genesis_state(source_log_dir);
    bfs::create_directories(config.blocks_dir);
    bfs::copy(source_log_dir / "blocks.log", config.blocks_dir / "blocks.log");
@@ -477,7 +477,8 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_pending_schedule_snapshot, SNAPSHOT_SUITE, sn
    source_log_dir_str += "prod_sched";
    const auto source_log_dir = bfs::path(source_log_dir_str.c_str());
    const uint32_t legacy_default_max_inline_action_size = 4 * 1024;
-   auto config = tester::default_config(fc::temp_directory(), legacy_default_max_inline_action_size).first;
+   fc::temp_directory temp_dir;
+   auto config = tester::default_config(temp_dir, legacy_default_max_inline_action_size).first;
    auto genesis = eosio::chain::block_log::extract_genesis_state(source_log_dir);
    bfs::create_directories(config.blocks_dir);
    bfs::copy(source_log_dir / "blocks.log", config.blocks_dir / "blocks.log");