simplify read-only threads execution

2023-03-17 12:46:26 -04:00
30 changed files with 489 additions and 911 deletions
@@ -14,7 +14,7 @@ set( CMAKE_CXX_EXTENSIONS ON )
 set( CXX_STANDARD_REQUIRED ON)

 set(VERSION_MAJOR 4)
-set(VERSION_MINOR 1)
+set(VERSION_MINOR 0)
 set(VERSION_PATCH 0)
 set(VERSION_SUFFIX dev)

@@ -77,9 +77,29 @@ if(ENABLE_OC AND CMAKE_SIZEOF_VOID_P EQUAL 8 AND NOT WIN32)
      list(APPEND EOSIO_WASM_RUNTIMES eos-vm-oc)
      # EOS VM OC requires LLVM, but move the check up here to a central location so that the EosioTester.cmakes
      # can be created with the exact version found
-      find_package(LLVM REQUIRED CONFIG)
+      # find_package VERSION range is only available in 3.19 and newer.
+      if(${CMAKE_VERSION} VERSION_LESS "3.19")
+         find_package(LLVM REQUIRED CONFIG)
+      else()
+         # Preferring the latest LLVM version, search through major 11..7 and minor 9..0 in reverse order.
+         # Minor versions may break API. This manifests in the LLVM config files such that a range of `11...<12`
+         # will NOT find LLVM `11.1`. We loop over minor as well as major to resolve this.
+         set(max_minor 9)
+         set(major 11)
+         set(minor ${max_minor})
+         while(NOT LLVM_FOUND AND major GREATER_EQUAL 7)
+            math(EXPR end "${major} + 1")
+            find_package(LLVM ${major}.${minor}...<${end} CONFIG QUIET)
+            if(minor GREATER 0)
+               math(EXPR minor "${minor} - 1")
+            else()
+               math(EXPR major "${major} - 1")
+               set(minor ${max_minor})
+            endif()
+         endwhile()
+      endif()
      if(LLVM_VERSION_MAJOR VERSION_LESS 7 OR LLVM_VERSION_MAJOR VERSION_GREATER_EQUAL 12)
-	      message(FATAL_ERROR "Leap requires an LLVM version 7 through 11")
+         message(FATAL_ERROR "Leap requires an LLVM version 7 through 11")
      endif()
   endif()
 endif()
@@ -72,7 +72,7 @@ git clone --recursive https://github.com/AntelopeIO/leap.git
 git clone --recursive git@github.com:AntelopeIO/leap.git
 ```

-> ℹ️ **HTTPS vs. SSH Clone** ℹ️  
+> ℹ️ **HTTPS vs. SSH Clone** ℹ️
 Both an HTTPS or SSH git clone will yield the same result - a folder named `leap` containing our source code. It doesn't matter which type you use.

 Navigate into that folder:
@@ -96,13 +96,13 @@ git submodule update --init --recursive
 ### Step 3 - Build
 Select build instructions below for a [pinned build](#pinned-build) (preferred) or an [unpinned build](#unpinned-build).

-> ℹ️ **Pinned vs. Unpinned Build** ℹ️  
+> ℹ️ **Pinned vs. Unpinned Build** ℹ️
 We have two types of builds for Leap: "pinned" and "unpinned." The only difference is that pinned builds use specific versions for some dependencies hand-picked by the Leap engineers - they are "pinned" to those versions. In contrast, unpinned builds use the default dependency versions available on the build system at the time. We recommend performing a "pinned" build to ensure the compiler and boost versions remain the same between builds of different Leap versions. Leap requires these versions to remain the same, otherwise its state might need to be recovered from a portable snapshot or the chain needs to be replayed.

-> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️  
+> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️
 When building C/C++ software, often the build is performed in parallel via a command such as `make -j "$(nproc)"` which uses all available CPU threads. However, be aware that some compilation units (`*.cpp` files) in Leap will consume nearly 4GB of memory. Failures due to memory exhaustion will typically, but not always, manifest as compiler crashes. Using all available CPU threads may also prevent you from doing other things on your computer during compilation. For these reasons, consider reducing this value.

-> 🐋 **Docker and `sudo`** 🐋  
+> 🐋 **Docker and `sudo`** 🐋
 If you are in an Ubuntu docker container, omit `sudo` from all commands because you run as `root` by default. Most other docker containers also exclude `sudo`, especially Debian-family containers. If your shell prompt is a hash tag (`#`), omit `sudo`.

 #### Pinned Build
@@ -111,14 +111,12 @@ Make sure you are in the root of the `leap` repo, then run the `install_depts.sh
 sudo scripts/install_deps.sh
 ```

-Next, run the pinned build script. You have to give it three arguments in the following order:
-1. A temporary folder, for all dependencies that need to be built from source.
-1. A build folder, where the binaries you need to install will be built to.
-1. The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
+Next, run the pinned build script. You have to give it three arguments, in the following order:
+  - A temporary folder, for all dependencies that need to be built from source.
+  - A build folder, where the binaries you need to install will be built to.
+  - The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).

-> 🔒 You do not need to run this script with `sudo` or as root.
-
-For example, the following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads:
+The following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads (Note: you don't need `sudo` for this command):
 ```bash
 scripts/pinned_build.sh deps build "$(nproc)"
 ```
@@ -148,9 +146,11 @@ To build, make sure you are in the root of the `leap` repo, then run the followi
 ```bash
 mkdir -p build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/usr/lib/llvm-11 ..
+cmake -DCMAKE_BUILD_TYPE=Release ..
 make -j "$(nproc)" package
 ```
+
+If CMake fails to find a valid llvm configure file, it may be necessary to add `-DCMAKE_PREFIX_PATH=/usr/lib/llvm-11` to the cmake command.
 </details>

 <details> <summary>Ubuntu 18.04 Bionic</summary>
@@ -1,33 +1,30 @@
 #pragma once

 #include <appbase/application_base.hpp>
-#include <eosio/chain/exec_pri_queue.hpp>
-#include <mutex>
+#include <appbase/execution_priority_queue.hpp>

 /*
- * Customize appbase to support two-queue executor.
+ * Custmomize appbase to support two-queue exector.
 */
 namespace appbase { 

 enum class exec_window {
-   read,  // the window during which operations from parallel queue
-          // can be executed in parallel in the read-only thread pool
-          // as well as in the app thread.
-   write, // the window during which operations from both read_write and
-          // parallel queues can be executed in app thread,
-          // while read-only operations are not executed in read-only
-          // thread pool. The read-only thread pool is not active; only
-          // the main app thread is active.
+   read,  // the window during which operations from read_only_trx_safe queue
+          // can be executed in app thread in parallel with read-only transactions
+          // in read-only transaction excuting threads.
+   write, // the window during which operations from both general and
+          // read_only_trx_safe queues can be executed in app thread,
+          // while read-only transactions are not executed in read-only
+          // transaction excuting threads.
 };

 enum class exec_queue {
-   read_only,          // the queue storing tasks which are safe to execute
-                       // in parallel with other read-only tasks in the read-only
-                       // thread pool as well as on the main app thread.
-                       // Multi-thread safe as long as nothing is executed from the read_write queue.
-   read_write          // the queue storing tasks which can be only executed
-                       // on the app thread while read-only tasks are
-                       // not being executed in read-only threads. Single threaded.
+   read_only_trx_safe, // the queue storing operations which are safe to execute
+                       // on app thread in parallel with read-only transactions
+                       // in read-only transaction excuting threads.
+   general             // the queue storing operations which can be only executed
+                       // on the app thread while read-only transactions are
+                       // not being executed in read-only threads
 };

 class two_queue_executor {
@@ -35,64 +32,56 @@ public:

   template <typename Func>
   auto post( int priority, exec_queue q, Func&& func ) {
-      if ( q == exec_queue::read_write )
-         return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      if ( q == exec_queue::general )
+         return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
      else
-         return boost::asio::post( io_serv_, read_only_queue_.wrap( priority, --order_, std::forward<Func>( func)));
+         return boost::asio::post(io_serv_, read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Func>(func)));
   }

   // Legacy and deprecated. To be removed after cleaning up its uses in base appbase
   template <typename Func>
   auto post( int priority, Func&& func ) {
-      // safer to use read_write queue for unknown type of operation since operations
-      // from read_write queue are not executed in parallel with read-only operations
-      return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      // safer to use general queue for unknown type of operation since operations
+      // from general queue are not executed in parallel with read-only transactions
+      return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
   }

   boost::asio::io_service& get_io_service() { return io_serv_; }

   bool execute_highest() {
      if ( exec_window_ == exec_window::write ) {
-         // During write window only main thread is accessing anything in two_queue_executor, no locking required
-         if( !read_write_queue_.empty() && (read_only_queue_.empty() || *read_only_queue_.top() < *read_write_queue_.top()) )  {
-            // read_write_queue_'s top function's priority greater than read_only_queue_'s top function's, or read_only_queue_ empty
-            read_write_queue_.execute_highest();
-         } else if( !read_only_queue_.empty() ) {
-            read_only_queue_.execute_highest();
+         if( !general_queue_.empty() && ( read_only_trx_safe_queue_.empty() || *read_only_trx_safe_queue_.top() < *general_queue_.top()) )  {
+            // general_queue_'s top function's priority greater than read_only_trx_safe_queue_'s top function's, or general_queue_ empty
+            general_queue_.execute_highest();
+         } else if( !read_only_trx_safe_queue_.empty() ) {
+            read_only_trx_safe_queue_.execute_highest();
         }
-         return !read_only_queue_.empty() || !read_write_queue_.empty();
+         return !read_only_trx_safe_queue_.empty() || !general_queue_.empty();
      } else {
-         // When in read window, multiple threads including main app thread are accessing two_queue_executor, locking required
-         return read_only_queue_.execute_highest_locked();
+         return read_only_trx_safe_queue_.execute_highest();
      }
   }

-   bool execute_highest_read_only() {
-      return read_only_queue_.execute_highest_locked();
-   }
-
   template <typename Function>
-   boost::asio::executor_binder<Function, appbase::exec_pri_queue::executor>
+   boost::asio::executor_binder<Function, appbase::execution_priority_queue::executor>
   wrap(int priority, exec_queue q, Function&& func ) {
-      if ( q == exec_queue::read_write )
-         return read_write_queue_.wrap(priority, --order_, std::forward<Function>(func));
+      if ( q == exec_queue::general )
+         return general_queue_.wrap(priority, --order_, std::forward<Function>(func));
      else
-         return read_only_queue_.wrap( priority, --order_, std::forward<Function>( func));
+         return read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Function>(func));
   }
     
   void clear() {
-      read_only_queue_.clear();
-      read_write_queue_.clear();
+      read_only_trx_safe_queue_.clear();
+      general_queue_.clear();
   }

   void set_to_read_window() {
      exec_window_ = exec_window::read;
-      read_only_queue_.enable_locking();
   }

   void set_to_write_window() {
      exec_window_ = exec_window::write;
-      read_only_queue_.disable_locking();
   }

   bool is_read_window() const {
@@ -103,15 +92,15 @@ public:
      return exec_window_ == exec_window::write;
   }

-   auto& read_only_queue() { return read_only_queue_; }
+   auto& read_only_trx_safe_queue() { return read_only_trx_safe_queue_; }

-   auto& read_write_queue() { return read_write_queue_; }
+   auto& general_queue() { return general_queue_; }

   // members are ordered taking into account that the last one is destructed first
 private:
   boost::asio::io_service            io_serv_;
-   appbase::exec_pri_queue            read_only_queue_;
-   appbase::exec_pri_queue            read_write_queue_;
+   appbase::execution_priority_queue  read_only_trx_safe_queue_;
+   appbase::execution_priority_queue  general_queue_;
   std::atomic<std::size_t>           order_ { std::numeric_limits<size_t>::max() }; // to maintain FIFO ordering in both queues within priority
   exec_window                        exec_window_ { exec_window::write };
 };
@@ -1,203 +0,0 @@
-#pragma once
-#include <boost/asio.hpp>
-
-#include <mutex>
-#include <queue>
-
-namespace appbase {
-// adapted from: https://www.boost.org/doc/libs/1_69_0/doc/html/boost_asio/example/cpp11/invocation/prioritised_handlers.cpp
-
-// Locking has to be coordinated by caller, use with care.
-class exec_pri_queue : public boost::asio::execution_context
-{
-public:
-
-   void enable_locking() {
-      lock_enabled_ = true;
-   }
-
-   void disable_locking() {
-      lock_enabled_ = false;
-   }
-
-   // called from appbase::application_base::exec poll_one() or run_one()
-   template <typename Function>
-   void add(int priority, size_t order, Function function)
-   {
-      std::unique_ptr<queued_handler_base> handler(new queued_handler<Function>(priority, order, std::move(function)));
-      if (lock_enabled_) {
-         std::lock_guard g( mtx_ );
-         handlers_.push( std::move( handler ) );
-      } else {
-         handlers_.push( std::move( handler ) );
-      }
-   }
-
-   // only call when no lock required
-   void clear()
-   {
-      handlers_ = prio_queue();
-   }
-
-   // only call when no lock required
-   bool execute_highest()
-   {
-      if( !handlers_.empty() ) {
-         handlers_.top()->execute();
-         handlers_.pop();
-      }
-
-      return !handlers_.empty();
-   }
-
-private:
-   // has to be defined before use, auto return type
-   auto pop() {
-      auto t = std::move(const_cast<std::unique_ptr<queued_handler_base>&>(handlers_.top()));
-      handlers_.pop();
-      return t;
-   }
-
-public:
-
-   bool execute_highest_locked() {
-      std::unique_lock g(mtx_);
-      if( handlers_.empty() )
-         return false;
-      auto t = pop();
-      g.unlock();
-      t->execute();
-      g.lock();
-      return !handlers_.empty();
-   }
-
-   // Only call when locking disabled
-   size_t size() const { return handlers_.size(); }
-
-   // Only call when locking disabled
-   bool empty() const { return handlers_.empty(); }
-
-   // Only call when locking disabled
-   const auto& top() const { return handlers_.top(); }
-
-   class executor
-   {
-   public:
-      executor(exec_pri_queue& q, int p, size_t o)
-            : context_(q), priority_(p), order_(o)
-      {
-      }
-
-      exec_pri_queue& context() const noexcept
-      {
-         return context_;
-      }
-
-      template <typename Function, typename Allocator>
-      void dispatch(Function f, const Allocator&) const
-      {
-         context_.add(priority_, order_, std::move(f));
-      }
-
-      template <typename Function, typename Allocator>
-      void post(Function f, const Allocator&) const
-      {
-         context_.add(priority_, order_, std::move(f));
-      }
-
-      template <typename Function, typename Allocator>
-      void defer(Function f, const Allocator&) const
-      {
-         context_.add(priority_, order_, std::move(f));
-      }
-
-      void on_work_started() const noexcept {}
-      void on_work_finished() const noexcept {}
-
-      bool operator==(const executor& other) const noexcept
-      {
-         return order_ == other.order_ && &context_ == &other.context_ && priority_ == other.priority_;
-      }
-
-      bool operator!=(const executor& other) const noexcept
-      {
-         return !operator==(other);
-      }
-
-   private:
-      exec_pri_queue& context_;
-      int priority_;
-      size_t order_;
-   };
-
-   template <typename Function>
-   boost::asio::executor_binder<Function, executor>
-   wrap(int priority, size_t order, Function&& func)
-   {
-      return boost::asio::bind_executor( executor(*this, priority, order), std::forward<Function>(func) );
-   }
-
-private:
-   class queued_handler_base
-   {
-   public:
-      queued_handler_base( int p, size_t order )
-            : priority_( p )
-            , order_( order )
-      {
-      }
-
-      virtual ~queued_handler_base() = default;
-
-      virtual void execute() = 0;
-
-      int priority() const { return priority_; }
-      // C++20
-      // friend std::weak_ordering operator<=>(const queued_handler_base&,
-      //                                       const queued_handler_base&) noexcept = default;
-      friend bool operator<(const queued_handler_base& a,
-                            const queued_handler_base& b) noexcept
-      {
-         return std::tie( a.priority_, a.order_ ) < std::tie( b.priority_, b.order_ );
-      }
-
-   private:
-      int priority_;
-      size_t order_;
-   };
-
-   template <typename Function>
-   class queued_handler : public queued_handler_base
-   {
-   public:
-      queued_handler(int p, size_t order, Function f)
-            : queued_handler_base( p, order )
-            , function_( std::move(f) )
-      {
-      }
-
-      void execute() override
-      {
-         function_();
-      }
-
-   private:
-      Function function_;
-   };
-
-   struct deref_less
-   {
-      template<typename Pointer>
-      bool operator()(const Pointer& a, const Pointer& b) noexcept(noexcept(*a < *b))
-      {
-         return *a < *b;
-      }
-   };
-
-   bool lock_enabled_ = false;
-   std::mutex mtx_;
-   using prio_queue = std::priority_queue<std::unique_ptr<queued_handler_base>, std::deque<std::unique_ptr<queued_handler_base>>, deref_less>;
-   prio_queue handlers_;
-};
-
-} // appbase
@@ -27,28 +27,28 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
   // post functions
   std::map<int, int> rslts {};
   int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );

   // Stop app. Use the lowest priority to make sure this function to execute the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
-      // read_only_queue should only contain the current lambda function,
-      // and read_write_queue should have executed all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_only_trx_safe_queue should only contain the current lambda function,
+      // and general_queue should have execute all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
      app->quit();
      } );
   app_thread.join();

   // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);

   // exactly number of both queues' functions processed
   BOOST_REQUIRE_EQUAL( rslts.size(), 8 );
@@ -64,40 +64,40 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
   BOOST_CHECK_LT( rslts[6], rslts[7] );
 }

-// verify functions only from read_only queue are processed during read window
+// verify functions only from read_only_trx_safe queue are processed during read window
 BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
   appbase::scoped_app app;
   auto app_thread = start_app_thread(app);
   
-   // set to run functions from read_only queue only
+   // set to run functions from read_only_trx_safe queue only
   app->executor().set_to_read_window();

   // post functions
   std::map<int, int> rslts {};
   int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );

   // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
-      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 4 );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_queue should have current function and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 4 );
      app->quit();
      } );
   app_thread.join();

   // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);

   // exactly number of posts processed
   BOOST_REQUIRE_EQUAL( rslts.size(), 6 );
@@ -109,40 +109,40 @@ BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
   BOOST_CHECK_LT( rslts[3], rslts[4] );
 }

-// verify no functions are executed during  read window if read_only queue is empty
+// verify no functions are executed during  read window if read_only_trx_safe queue is empty
 BOOST_AUTO_TEST_CASE( execute_from_empty_read_queue ) {
   appbase::scoped_app app;
   auto app_thread = start_app_thread(app);
   
-   // set to run functions from read_only queue only
+   // set to run functions from read_only_trx_safe queue only
   app->executor().set_to_read_window();

   // post functions
   std::map<int, int> rslts {};
   int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );

   // Stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
-      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 10 );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_queue should have current function and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 10 );
      app->quit();
      } );
   app_thread.join();

   // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);

   // no results
   BOOST_REQUIRE_EQUAL( rslts.size(), 0 );
@@ -159,32 +159,32 @@ BOOST_AUTO_TEST_CASE( execute_from_both_queues ) {
   // post functions
   std::map<int, int> rslts {};
   int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[9]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[10]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[11]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[10]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[11]=seq_num; ++seq_num; } );

   // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
      // read_queue should have current function and write_queue's functions are all executed 
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
-      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
      app->quit();
      } );

   app_thread.join();

   // queues are emptied after quit
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);

   // exactly number of posts processed
   BOOST_REQUIRE_EQUAL( rslts.size(), 12 );
@@ -75,7 +75,7 @@ void chain_api_plugin::plugin_startup() {
   ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() );

   _http_plugin.add_api( {
-      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only, appbase::priority::medium_high);
+      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
   _http_plugin.add_api({
      CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params),
      CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required),
@@ -103,12 +103,12 @@ void chain_api_plugin::plugin_startup() {
      CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required),
      CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required),
      CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_only);
+   }, appbase::exec_queue::read_only_trx_safe);

   // Not safe to run in parallel with read-only transactions
   _http_plugin.add_api({
      CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_write, appbase::priority::medium_low );
+   }, appbase::exec_queue::general, appbase::priority::medium_low );

   if (chain.account_queries_enabled()) {
      _http_plugin.add_async_api({
@@ -124,7 +124,7 @@ void chain_api_plugin::plugin_startup() {
   if (chain.transaction_finality_status_enabled()) {
      _http_plugin.add_api({
         CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required),
-      }, appbase::exec_queue::read_only);
+      }, appbase::exec_queue::read_only_trx_safe);
   }

   _http_plugin.add_api({
@@ -160,7 +160,7 @@ void chain_api_plugin::plugin_startup() {
           }
        }
      }
-   }, appbase::exec_queue::read_only);
+   }, appbase::exec_queue::read_only_trx_safe);
 }

 void chain_api_plugin::plugin_shutdown() {}
@@ -28,7 +28,7 @@ using namespace eosio;
 void db_size_api_plugin::plugin_startup() {
   app().get_plugin<http_plugin>().add_api({
       CALL_WITH_400(db_size, this, get,  INVOKE_R_V(this, get), 200),
-   }, appbase::exec_queue::read_only);
+   }, appbase::exec_queue::read_only_trx_safe);
 }

 db_size_stats db_size_api_plugin::get() {
@@ -334,7 +334,7 @@ namespace eosio {
                     handle_exception("node", "get_supported_apis", body.empty() ? "{}" : body, cb);
                  }
               }
-            }}, appbase::exec_queue::read_only);
+            }}, appbase::exec_queue::read_only_trx_safe);
            
         } catch (...) {
            fc_elog(logger(), "http_plugin startup fails, shutting down");
@@ -87,12 +87,8 @@ namespace eosio {
        void plugin_shutdown();
        void handle_sighup() override;

-        void add_handler(const string& url, const url_handler&, appbase::exec_queue q,
-                         int priority = appbase::priority::medium_low,
-                         http_content_type content_type = http_content_type::json);
-        void add_api(const api_description& api, appbase::exec_queue q,
-                     int priority = appbase::priority::medium_low,
-                     http_content_type content_type = http_content_type::json) {
+        void add_handler(const string& url, const url_handler&, appbase::exec_queue q, int priority = appbase::priority::medium_low, http_content_type content_type = http_content_type::json);
+        void add_api(const api_description& api, appbase::exec_queue q, int priority = appbase::priority::medium_low, http_content_type content_type = http_content_type::json) {
           for (const auto& call : api)
              add_handler(call.first, call.second, q, priority, content_type);
        }
@@ -52,7 +52,7 @@ public:
                  cb(200, fc::time_point::maximum(), fc::variant(ok ? string("yes") : string("no")));
               }
            },
-         }, appbase::exec_queue::read_write);
+         }, appbase::exec_queue::general);
   }

 private:
@@ -53,7 +53,11 @@ void net_api_plugin::plugin_startup() {
   ilog("starting net_api_plugin");
   // lifetime of plugin is lifetime of application
   auto& net_mgr = app().get_plugin<net_plugin>();
-   app().get_plugin<http_plugin>().add_async_api({
+   app().get_plugin<http_plugin>().add_api({
+    //   CALL(net, net_mgr, set_timeout,
+    //        INVOKE_V_R(net_mgr, set_timeout, int64_t), 200),
+    //   CALL(net, net_mgr, sign_transaction,
+    //        INVOKE_R_R_R_R(net_mgr, sign_transaction, chain::signed_transaction, flat_set<public_key_type>, chain::chain_id_type), 201),
       CALL_WITH_400(net, net_mgr, connect,
            INVOKE_R_R(net_mgr, connect, std::string), 201),
       CALL_WITH_400(net, net_mgr, disconnect,
@@ -62,7 +66,9 @@ void net_api_plugin::plugin_startup() {
            INVOKE_R_R(net_mgr, status, std::string), 201),
       CALL_WITH_400(net, net_mgr, connections,
            INVOKE_R_V(net_mgr, connections), 201),
-   } );
+    //   CALL(net, net_mgr, open,
+    //        INVOKE_V_R(net_mgr, open, std::string), 200),
+   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
 }

 void net_api_plugin::plugin_initialize(const variables_map& options) {
@@ -3234,7 +3234,7 @@ namespace eosio {
            my_impl->dispatcher->bcast_block( bsp->block, bsp->id );
         }

-         app().executor().post(priority::medium, exec_queue::read_write, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
+         app().executor().post(priority::medium, exec_queue::general, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
            c->process_signed_block( id, std::move(ptr), std::move(bsp) );
         });

@@ -118,11 +118,9 @@ void producer_api_plugin::plugin_startup() {
            INVOKE_R_R(producer, get_account_ram_corrections, producer_plugin::get_account_ram_corrections_params), 201),
       CALL_WITH_400(producer, producer, get_unapplied_transactions,
                     INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200),
-       CALL_WITH_400(producer, producer, get_snapshot_requests,
-                     INVOKE_R_V(producer, get_snapshot_requests), 201),
-   }, appbase::exec_queue::read_only, appbase::priority::medium_high);
+   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);

-   // Not safe to run in parallel
+   // Not safe to run in parallel with read-only transactions
   app().get_plugin<http_plugin>().add_api({
       CALL_WITH_400(producer, producer, pause,
            INVOKE_V_V(producer, pause), 201),
@@ -140,13 +138,15 @@ void producer_api_plugin::plugin_startup() {
            INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
       CALL_WITH_400(producer, producer, schedule_snapshot,
            INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+            INVOKE_R_V(producer, get_snapshot_requests), 201),
       CALL_WITH_400(producer, producer, unschedule_snapshot,
            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
       CALL_WITH_400(producer, producer, get_integrity_hash,
            INVOKE_R_V(producer, get_integrity_hash), 201),
       CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
            INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201),
-   }, appbase::exec_queue::read_write, appbase::priority::medium_high);
+   }, appbase::exec_queue::general, appbase::priority::medium_high);
 }

 void producer_api_plugin::plugin_initialize(const variables_map& options) {
@@ -14,7 +14,6 @@
 #include <fc/io/json.hpp>
 #include <fc/log/logger_config.hpp>
 #include <fc/scoped_exit.hpp>
-#include <fc/time.hpp>

 #include <boost/asio.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -22,6 +21,7 @@
 #include <iostream>
 #include <algorithm>
 #include <mutex>
+#include <chrono>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/range/adaptor/map.hpp>
 #include <boost/multi_index_container.hpp>
@@ -325,9 +325,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
      push_result push_transaction( const fc::time_point& block_deadline,
                                    const transaction_metadata_ptr& trx,
                                    bool api_trx, bool return_failure_trace,
-                                    const next_function<transaction_trace_ptr>& next );
+                                    next_function<transaction_trace_ptr> next );
      push_result handle_push_result( const transaction_metadata_ptr& trx,
-                                      const next_function<transaction_trace_ptr>& next,
+                                      next_function<transaction_trace_ptr> next,
                                      const fc::time_point& start,
                                      const chain::controller& chain,
                                      const transaction_trace_ptr& trace,
@@ -418,7 +418,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
         packed_transaction_ptr  trx;
         next_func_t             next;
      };
-      // The queue storing previously exhausted read-only transactions to be re-executed by read-only threads
+      // The queue storing read-only transactions to be executed by read-only threads
      struct ro_trx_queue_t {
         std::mutex              mtx;
         std::deque<ro_trx_t>    queue;
@@ -430,26 +430,28 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
      fc::microseconds                _ro_read_window_time_us{ 60000 };
      static constexpr fc::microseconds _ro_read_window_minimum_time_us{ 10000 };
      fc::microseconds                _ro_read_window_effective_time_us{ 0 }; // calculated during option initialization
-      fc::time_point                  _ro_window_deadline; // only modified on app thread
      boost::asio::deadline_timer     _ro_write_window_timer;
      boost::asio::deadline_timer     _ro_read_window_timer;
      fc::microseconds                _ro_max_trx_time_us{ 0 }; // calculated during option initialization
-      ro_trx_queue_t                  _ro_exhausted_trx_queue;
-      std::atomic<uint32_t>           _ro_num_active_exec_tasks{ 0 };
-      std::atomic<bool>               _ro_exiting_read_window{ false };
-      bool                            _ro_in_read_only_mode{false}; // only modified on app thread
-      std::vector<std::future<bool>>  _ro_exec_tasks_fut;
+      ro_trx_queue_t                  _ro_trx_queue;
+      std::atomic<uint32_t>            _ro_num_active_trx_exec_tasks{ 0 };
+      std::condition_variable         _switch_window_cond;
+      std::mutex                      _switch_window_mtx;
+      uint32_t                        _ro_idle_threads{ 0 };
+      std::condition_variable         _ro_more_work_cond;

+      void start_read_only_thread();
      void start_write_window();
      void switch_to_write_window();
      void switch_to_read_window();
-      bool read_only_execution_task();
-      void process_read_only_transactions(const fc::time_point& deadline);
+      bool read_only_trx_execution_task();
      bool process_read_only_transaction(const packed_transaction_ptr& trx,
-                                         const next_function<transaction_trace_ptr>& next);
+                                         const next_function<transaction_trace_ptr>& next,
+                                         const fc::time_point& read_window_start_time);
      bool push_read_only_transaction(const transaction_metadata_ptr& trx,
-                                      const next_function<transaction_trace_ptr>& next);
-
+                                      next_function<transaction_trace_ptr> next,
+                                      const fc::time_point& read_window_start_time);
+      
      void consider_new_watermark( account_name producer, uint32_t block_num, block_timestamp_type timestamp) {
         auto itr = _producer_watermarks.find( producer );
         if( itr != _producer_watermarks.end() ) {
@@ -642,11 +644,15 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                         transaction_metadata::trx_type trx_type,
                                         bool return_failure_traces,
                                         next_function<transaction_trace_ptr> next) {
-         if ( trx_type == transaction_metadata::trx_type::read_only ) {
-            // Post all read only trxs to read_only queue for execution.
-            app().executor().post(priority::low, exec_queue::read_only, [this, trx=trx, next{std::move(next)}]() mutable {
-               process_read_only_transaction( trx, next );
-            } );
+         if ( trx_type == transaction_metadata::trx_type::read_only && _ro_thread_pool_size > 0 ) {
+            // Parallel read-only trx execution enabled.
+            // Store the transaction in read-only-trx queue so that it is
+            // executed in read window
+            std::lock_guard<std::mutex> g( _ro_trx_queue.mtx );
+            _ro_trx_queue.queue.push_back({trx, std::move(next)});
+            if ( _ro_idle_threads > 0 ) {
+               _ro_more_work_cond.notify_all();
+            }
            return;
         }

@@ -679,7 +685,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                                                          next{std::move(next)}, trx=trx]() mutable {
            if( future.valid() ) {
               future.wait();
-               app().executor().post( priority::low, exec_queue::read_write, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
+               app().executor().post( priority::low, exec_queue::general, [self, future{std::move(future)}, api_trx, is_transient, next{std::move( next )}, trx{std::move(trx)}, return_failure_traces]() mutable {
                  auto start = fc::time_point::now();
                  auto idle_time = start - self->_idle_trx_time;
                  self->_time_tracker.add_idle_time( idle_time );
@@ -711,7 +717,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
      bool process_incoming_transaction_async(const transaction_metadata_ptr& trx,
                                              bool api_trx,
                                              bool return_failure_trace,
-                                              const next_function<transaction_trace_ptr>& next) {
+                                              next_function<transaction_trace_ptr> next) {
         bool exhausted = false;
         chain::controller& chain = chain_plug->chain();
         try {
@@ -894,7 +900,7 @@ void producer_plugin::set_program_options(
          "Number of worker threads in producer thread pool")
         ("snapshots-dir", bpo::value<bfs::path>()->default_value("snapshots"),
          "the location of the snapshots directory (absolute path or relative to application data dir)")
-         ("read-only-threads", bpo::value<uint16_t>(),
+         ("read-only-threads", bpo::value<uint16_t>()->default_value(my->_ro_thread_pool_size),
          "Number of worker threads in read-only transaction execution thread pool")
         ("read-only-write-window-time-us", bpo::value<uint32_t>()->default_value(my->_ro_write_window_time_us.count()),
          "time in microseconds the write window lasts")
@@ -1075,15 +1081,11 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
      }
   }

-   if ( options.count( "read-only-threads" ) ) {
-      my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint16_t>();
-      EOS_ASSERT( my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
-   } else if ( my->_producers.empty() ) {
-      // default to 3 threads for non producer nodes if not specified
-      my->_ro_thread_pool_size = 3;
-   }
+   my->_ro_thread_pool_size = options.at( "read-only-threads" ).as<uint16_t>();
   // only initialize other read-only options when read-only thread pool is enabled
   if ( my->_ro_thread_pool_size > 0 ) {
+      EOS_ASSERT( my->_producers.empty(), plugin_config_exception, "--read-only-threads not allowed on producer node" );
+
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
      if (chain.is_eos_vm_oc_enabled()) {
         // EOS VM OC requires 4.2TB Virtual for each executing thread. Make sure the memory
@@ -1115,26 +1117,25 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
         }
      }
 #endif
-   }

-   my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
-   my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
-   EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
-   my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;
+      my->_ro_write_window_time_us = fc::microseconds( options.at( "read-only-write-window-time-us" ).as<uint32_t>() );
+      my->_ro_read_window_time_us = fc::microseconds( options.at( "read-only-read-window-time-us" ).as<uint32_t>() );
+      EOS_ASSERT( my->_ro_read_window_time_us > my->_ro_read_window_minimum_time_us, plugin_config_exception, "minimum of --read-only-read-window-time-us (${read}) must be ${min} microseconds", ("read", my->_ro_read_window_time_us) ("min", my->_ro_read_window_minimum_time_us) );
+      my->_ro_read_window_effective_time_us = my->_ro_read_window_time_us - my->_ro_read_window_minimum_time_us;

-   // Make sure a read-only transaction can finish within the read
-   // window if scheduled at the very beginning of the window.
-   // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
-   // for safety marging
-   if ( my->_max_transaction_time_ms.load() > 0 ) {
-      EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
-      my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
-   } else {
-      // _max_transaction_time_ms can be set to negative in testing (for unlimited)
-      my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
+      // Make sure a read-only transaction can finish within the read
+      // window if scheduled at the very beginning of the window.
+      // Use _ro_read_window_effective_time_us instead of _ro_read_window_time_us
+      // for safety marging
+      if ( my->_max_transaction_time_ms.load() > 0 ) {
+         EOS_ASSERT( my->_ro_read_window_effective_time_us > fc::milliseconds(my->_max_transaction_time_ms.load()), plugin_config_exception, "--read-only-read-window-time-us (${read}) must be greater than --max-transaction-time ${trx_time} ms plus a margin of ${min} us", ("read", my->_ro_read_window_time_us) ("trx_time", my->_max_transaction_time_ms.load()) ("min", my->_ro_read_window_minimum_time_us) );
+         my->_ro_max_trx_time_us = fc::milliseconds(my->_max_transaction_time_ms.load());
+      } else {
+         // _max_transaction_time_ms can be set to negative in testing (for unlimited)
+         my->_ro_max_trx_time_us = my->_ro_read_window_effective_time_us;
+      }
+      ilog("_ro_thread_pool_size ${s},  _ro_write_window_time_us ${ww}, _ro_read_window_time_us ${rw}, _ro_max_trx_time_us ${t}, _ro_read_window_effective_time_us ${w}", ("s", my->_ro_thread_pool_size) ("ww", my->_ro_write_window_time_us) ("rw", my->_ro_read_window_time_us) ("t", my->_ro_max_trx_time_us) ("w", my->_ro_read_window_effective_time_us));
   }
-   ilog("ro_thread_pool_size ${s}, ro_write_window_time_us ${ww}, ro_read_window_time_us ${rw}, ro_max_trx_time_us ${t}, ro_read_window_effective_time_us ${w}",
-        ("s", my->_ro_thread_pool_size)("ww", my->_ro_write_window_time_us)("rw", my->_ro_read_window_time_us)("t", my->_ro_max_trx_time_us)("w", my->_ro_read_window_effective_time_us));

   my->_incoming_block_sync_provider = app().get_method<incoming::methods::block_sync>().register_provider(
         [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
@@ -1221,8 +1222,8 @@ void producer_plugin::plugin_startup()
            fc_elog( _log, "Exception in read-only transaction thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
            app().quit();
         },
-         [&]() {
-            chain.init_thread_local_data();
+         [this]() {
+            this->my->start_read_only_thread();
         });

      my->start_write_window();
@@ -2001,8 +2002,6 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
            process_scheduled_and_incoming_trxs( scheduled_trx_deadline, incoming_itr );
         }

-         process_read_only_transactions( preprocess_deadline );
-
         if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit()
            return start_block_result::failed;
         if ( should_interrupt_start_block( preprocess_deadline ) || block_is_exhausted() ) {
@@ -2204,10 +2203,9 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
                                        const transaction_metadata_ptr& trx,
                                        bool api_trx,
                                        bool return_failure_trace,
-                                        const next_function<transaction_trace_ptr>& next )
+                                        next_function<transaction_trace_ptr> next )
 {
   auto start = fc::time_point::now();
-   EOS_ASSERT(!trx->is_read_only(), producer_exception, "Unexpected read-only trx");

   bool disable_subjective_enforcement = (api_trx && _disable_subjective_api_billing)
                                         || (!api_trx && _disable_subjective_p2p_billing)
@@ -2244,6 +2242,14 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,
      }
   }

+   // Make sure db_read_only_mode always to be unset
+   auto db_read_only_mode_guard = fc::make_scoped_exit([trx, &chain]{
+      if( trx->is_read_only() )
+         chain.unset_db_read_only_mode();
+   });
+   if( trx->is_read_only() )
+      chain.set_db_read_only_mode();
+
   auto trace = chain.push_transaction( trx, block_deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );

   return handle_push_result(trx, next, start, chain, trace, return_failure_trace, disable_subjective_enforcement, first_auth, sub_bill, prev_billed_cpu_time_us);
@@ -2251,7 +2257,7 @@ producer_plugin_impl::push_transaction( const fc::time_point& block_deadline,

 producer_plugin_impl::push_result
 producer_plugin_impl::handle_push_result( const transaction_metadata_ptr& trx,
-                                          const next_function<transaction_trace_ptr>& next,
+                                          next_function<transaction_trace_ptr> next,
                                          const fc::time_point& start,
                                          const chain::controller& chain,
                                          const transaction_trace_ptr& trace,
@@ -2557,7 +2563,7 @@ void producer_plugin_impl::schedule_production_loop() {
      _timer.expires_from_now( boost::posix_time::microseconds( config::block_interval_us  / 10 ));

      // we failed to start a block, so try again later?
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
          [weak_this = weak_from_this(), cid = ++_timer_corelation_id]( const boost::system::error_code& ec ) {
             auto self = weak_this.lock();
             if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2610,7 +2616,7 @@ void producer_plugin_impl::schedule_maybe_produce_block( bool exhausted ) {
               ("num", chain.head_block_num() + 1)("desc", block_is_exhausted() ? "Exhausted" : "Deadline exceeded") );
   }

-   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
+   _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
         [&chain, weak_this = weak_from_this(), cid=++_timer_corelation_id](const boost::system::error_code& ec) {
            auto self = weak_this.lock();
            if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2652,7 +2658,7 @@ void producer_plugin_impl::schedule_delayed_production_loop(const std::weak_ptr<
      fc_dlog(_log, "Scheduling Speculative/Production Change at ${time}", ("time", wake_up_time));
      static const boost::posix_time::ptime epoch(boost::gregorian::date(1970, 1, 1));
      _timer.expires_at(epoch + boost::posix_time::microseconds(wake_up_time->time_since_epoch().count()));
-      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::read_write,
+      _timer.async_wait( app().executor().wrap( priority::high, exec_queue::general,
         [weak_this,cid=++_timer_corelation_id](const boost::system::error_code& ec) {
            auto self = weak_this.lock();
            if( self && ec != boost::asio::error::operation_aborted && cid == self->_timer_corelation_id ) {
@@ -2766,7 +2772,7 @@ void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id,
            ("entire_trx", packed_trx_ptr ? my->chain_plug->get_log_trx(packed_trx_ptr->get_transaction()) : fc::variant{trx_id}));
 }

-// Called from only one read_only thread
+// Called from app thread
 void producer_plugin_impl::switch_to_write_window() {
   // this method can be called from multiple places. it is possible
   // we are already in write window.
@@ -2774,29 +2780,30 @@ void producer_plugin_impl::switch_to_write_window() {
      return;
   }

-   EOS_ASSERT( _ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "no read-only tasks should be running before switching to write window");
+   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0, producer_exception, "no read-only tasks should be running before switching to write window");
   _ro_read_window_timer.cancel();
   _ro_write_window_timer.cancel();

   start_write_window();
 }

-// Called from app thread on plugin_startup
-// Called from only one read_only thread & called from app thread, but not concurrently
+// Called from app thread
 void producer_plugin_impl::start_write_window() {
   chain::controller& chain = chain_plug->chain();

   app().executor().set_to_write_window();
   chain.unset_db_read_only_mode();
-   _ro_in_read_only_mode = false;
   _idle_trx_time = fc::time_point::now();
+   {
+      std::lock_guard<std::mutex> g(_switch_window_mtx);
+      _switch_window_cond.notify_all();
+   }

-   _ro_window_deadline = _idle_trx_time + _ro_write_window_time_us; // not allowed on block producers, so no need to limit to block deadline
   auto expire_time = boost::posix_time::microseconds(_ro_write_window_time_us.count());
   _ro_write_window_timer.expires_from_now( expire_time );
-   _ro_write_window_timer.async_wait( app().executor().wrap( // run from app thread
+   _ro_write_window_timer.async_wait( app().executor().wrap(  // stay on app thread
      priority::high,
-      exec_queue::read_write, // placed in read_write so only called from main thread
+      exec_queue::read_only_trx_safe, // placed in read_only_trx_safe queue so it is ensured to be executed in either window
      [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
         auto self = weak_this.lock();
         if( self && ec != boost::asio::error::operation_aborted ) {
@@ -2805,127 +2812,111 @@ void producer_plugin_impl::start_write_window() {
      }));
 }

-// Called only from app thread
+// Called from app thread
 void producer_plugin_impl::switch_to_read_window() {
   EOS_ASSERT(app().executor().is_write_window(),  producer_exception, "expected to be in write window");
-   EOS_ASSERT( _ro_num_active_exec_tasks.load() == 0 && _ro_exec_tasks_fut.empty(), producer_exception, "_ro_exec_tasks_fut expected to be empty" );
+   EOS_ASSERT(_ro_num_active_trx_exec_tasks.load() == 0,  producer_exception, "_ro_num_active_trx_exec_tasks expected to be 0" );

   _ro_write_window_timer.cancel();
   _ro_read_window_timer.cancel();
   _time_tracker.add_idle_time( fc::time_point::now() - _idle_trx_time );

   // we are in write window, so no read-only trx threads are processing transactions.
-   // _ro_exhausted_trx_queue is not being accessed by read-only threads, no need to lock.
-   if ( _ro_exhausted_trx_queue.queue.empty() && app().executor().read_only_queue().empty() ) { // no read-only trxs to process. stay in write window
+   // _ro_trx_queue is not being accessed. No need to lock.
+   if ( _ro_trx_queue.queue.empty() ) { // no read-only trxs to process. stay in write window
      start_write_window(); // restart write window timer for next round
      return;
   }

   app().executor().set_to_read_window();
   chain_plug->chain().set_db_read_only_mode();
-   _ro_in_read_only_mode = true;
   _received_block = false;
-   _ro_exiting_read_window = false;
-
-   // start a read-only transaction execution task in each thread in the thread pool
-   _ro_num_active_exec_tasks = _ro_thread_pool_size;
-   _ro_window_deadline = fc::time_point::now() + _ro_read_window_effective_time_us;
-   for (auto i = 0; i < _ro_thread_pool_size; ++i ) {
-      _ro_exec_tasks_fut.emplace_back( post_async_task( _ro_thread_pool.get_executor(), [self = this] () {
-         return self->read_only_execution_task();
-      }) );
+   {
+      std::lock_guard<std::mutex> g(_switch_window_mtx);
+      _switch_window_cond.notify_all();
   }
-
-   auto expire_time = boost::posix_time::microseconds(_ro_read_window_time_us.count());
-   _ro_read_window_timer.expires_from_now( expire_time );
-   // Needs to be on read_only because that is what is being processed until switch_to_write_window().
-   _ro_read_window_timer.async_wait( app().executor().wrap(
-      priority::high,
-      exec_queue::read_only,
-      [weak_this = weak_from_this()]( const boost::system::error_code& ec ) {
-         auto self = weak_this.lock();
-         if( self && ec != boost::asio::error::operation_aborted ) {
-            // use future to make sure all read-only tasks finished before switching to write window
-            for ( auto& task: self->_ro_exec_tasks_fut ) {
-               task.get();
-            }
-            self->_ro_exec_tasks_fut.clear();
-            // will be executed from the main app thread because all read-only threads are idle now
-            self->switch_to_write_window();
-          } else if ( self ) {
-             self->_ro_exec_tasks_fut.clear();
-          }
-       }));
+   // read-only threads decide when to switch back to write window
 }

-// Called from a read only thread. Run in parallel with app and other read only threads
-bool producer_plugin_impl::read_only_execution_task() {
-   // We have 4 ways to break out the while loop:
-   // 1. pass read window deadline
-   // 2. net_plugin receives a block
-   // 3. No more transactions in the exhausted read-only queue and no read-only tasks to execute
-   // 4. A transaction execution is exhausted (means end of read window)
-   bool exhausted_trx_queue_empty = false;
-   while ( fc::time_point::now() < _ro_window_deadline && !_received_block && !_ro_exiting_read_window ) {
-      if (!exhausted_trx_queue_empty) {
-         std::unique_lock<std::mutex> lck( _ro_exhausted_trx_queue.mtx );
-         auto size = _ro_exhausted_trx_queue.queue.size();
-         if ( size > 0 ) { // execute exhausted read-only first
-            auto trx = _ro_exhausted_trx_queue.queue.front();
-            _ro_exhausted_trx_queue.queue.pop_front();
-            lck.unlock();
+// Called from a read only trx thread. Run in parallel with main and other read only trx threads
+void producer_plugin_impl::start_read_only_thread() {
+   chain::controller& chain = chain_plug->chain();
+   chain.init_thread_local_data();

-            auto retry = process_read_only_transaction( trx.trx, trx.next );
-            if( retry ) {
-               _ro_exiting_read_window = true;
-               break;
-            }
-         }
-         exhausted_trx_queue_empty = (size <= 1); // one was executed above
-      } else {
-         bool more = app().executor().execute_highest_read_only();
-         if ( !more ) { // read_only queue empty, anything queued after this will be processed in the next window
-            _ro_exiting_read_window = true;
-            break;
-         }
+   while ( 1 ) {
+      {
+         // wait until we are in read window
+         std::unique_lock<std::mutex> lck(_switch_window_mtx);
+         _switch_window_cond.wait(lck, []{ return app().executor().is_read_window(); });
      }
-   }

-   // If all tasks are finished, do not wait until end of read window; switch to write window now.
-   if ( --_ro_num_active_exec_tasks == 0 ) {
-      // Needs to be on read_only because that is what is being processed until switch_to_write_window().
-      app().executor().post( priority::high, exec_queue::read_only, [self=this]() {
-         self->_ro_exec_tasks_fut.clear();
-         // will be executed from the main app thread because all read-only threads are idle now
-         self->switch_to_write_window();
-      } );
-   }
+      // executue trxs until deadline or no more trxs
+      read_only_trx_execution_task();

-   return true;
-}
-
-// Called from app thread during start block.
-// Used for executing exhausted read-only trxs when no read only thread pool enabled.
-void producer_plugin_impl::process_read_only_transactions(const fc::time_point& deadline) {
-   // if the read only thread pool is enabled then exhausted read only transactions will be executed
-   // by read_only_execution_task, so don't execute them here.
-   if ( _ro_thread_pool_size == 0 ) {
-      _ro_window_deadline = std::min(deadline, fc::time_point::now() + _ro_read_window_effective_time_us);
-      // only app thread accessing _ro_exhausted_trx_queue since _ro_thread_pool_size == 0, no lock needed
-      while ( !_ro_exhausted_trx_queue.queue.empty() && fc::time_point::now() < _ro_window_deadline && !_received_block ) {
-         auto trx = _ro_exhausted_trx_queue.queue.front();
-         _ro_exhausted_trx_queue.queue.pop_front();
-
-         auto retry = process_read_only_transaction( trx.trx, trx.next );
-         if( retry )
-            break;
+      if ( --_ro_num_active_trx_exec_tasks == 0 ) {
+         // last active thread switches the window
+         switch_to_write_window();
+      } else {
+         // other threads wait
+         std::unique_lock<std::mutex> lck(_switch_window_mtx);
+         _switch_window_cond.wait(lck, []{ return app().executor().is_write_window(); });
      }
   }
 }

+// Called from a read only trx thread. Run in parallel with app and other read only trx threads
+bool producer_plugin_impl::read_only_trx_execution_task() {
+   auto start = fc::time_point::now();
+   auto read_window_deadline = start + _ro_read_window_effective_time_us;
+   // We have 4 ways to break out the while loop:
+   // 1. pass read window deadline
+   // 2. Net_plugin receives a block
+   // 3. No more transactions in the read-only trx queue
+   // 4. A transaction execution is exhaused
+   while ( fc::time_point::now() < read_window_deadline && !_received_block ) {
+      std::unique_lock<std::mutex> lck( _ro_trx_queue.mtx );
+      if ( _ro_trx_queue.queue.empty() ) {
+         // double check so read_window_deadline - now is valid
+         auto now = fc::time_point::now();
+         if ( now >= read_window_deadline || _received_block ) {
+            break;
+         }
+         auto wait_time = std::min(read_window_deadline - now, _ro_max_trx_time_us).count();
+
+         ++_ro_idle_threads;
+         _ro_more_work_cond.wait_for(lck, std::chrono::microseconds(wait_time));
+         --_ro_idle_threads;
+         if ( fc::time_point::now() >= read_window_deadline || !_received_block ) {
+            // honor read-window deadline and _received_block
+            break;
+         }
+         if ( !_ro_trx_queue.queue.empty() || _ro_idle_threads > 0 ) {
+            // may have more work
+            continue;
+         } else {
+            // no need to wait
+            break;
+         }
+      }
+      auto trx = _ro_trx_queue.queue.front();
+      _ro_trx_queue.queue.pop_front();
+      lck.unlock();
+      
+      auto retry = process_read_only_transaction( trx.trx, trx.next, start );
+      if ( retry ) {
+         lck.lock();
+         _ro_trx_queue.queue.push_front(trx);
+         // This happens only when a trx exhausted its time. Do not schedule new executions
+         break;
+      }
+   }
+
+   return true;
+}
+
 // Called from a read only trx thread. Run in parallel with app and other read only trx threads
 // Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::process_read_only_transaction(const packed_transaction_ptr& trx, const next_function<transaction_trace_ptr>& next) {
+bool producer_plugin_impl::process_read_only_transaction(const packed_transaction_ptr& trx, const next_function<transaction_trace_ptr>& next, const fc::time_point& read_window_start_time) {
   chain::controller& chain = chain_plug->chain();
   auto future = transaction_metadata::start_recover_keys( trx, _thread_pool.get_executor(),
                                                           chain.get_chain_id(), fc::microseconds::maximum(),
@@ -2938,19 +2929,17 @@ bool producer_plugin_impl::process_read_only_transaction(const packed_transactio
   future.wait();
   try {
      auto trx_metadata = future.get();
-      bool retry = push_read_only_transaction( trx_metadata, next );
-      if( retry ) {
-         std::lock_guard g( _ro_exhausted_trx_queue.mtx );
-         _ro_exhausted_trx_queue.queue.push_front( {trx, next} );
-      }
-      return retry;
+      return push_read_only_transaction( trx_metadata, next, read_window_start_time );
   } CATCH_AND_CALL(exception_handler);
   return false;
 }

 // Called from a read_only_trx execution thread
 // Return whether the trx needs to be retried in next read window
-bool producer_plugin_impl::push_read_only_transaction(const transaction_metadata_ptr& trx, const next_function<transaction_trace_ptr>& next) {
+bool producer_plugin_impl::push_read_only_transaction(
+            const transaction_metadata_ptr& trx,
+            next_function<transaction_trace_ptr> next,
+            const fc::time_point& read_window_start_time) {
   auto retry = false;
   chain::controller& chain = chain_plug->chain();

@@ -2960,24 +2949,18 @@ bool producer_plugin_impl::push_read_only_transaction(const transaction_metadata
   }

   try {
+      const auto block_deadline = calculate_block_deadline( chain.pending_block_time() );
      auto start = fc::time_point::now();
-      if ( start >= _ro_window_deadline ) {
+      auto time_used_in_read_window_us = ( start - read_window_start_time );
+      if ( time_used_in_read_window_us >= _ro_read_window_time_us ) {
         // already passed read-window deadline, try next round
         return true;
      }

-      // when executing on the main thread while in the write window, need to switch db mode to read only
-      // _ro_in_read_only_mode can only be false if running on main thread as it is only modified from the main thread
-      const bool switch_read_mode = !_ro_in_read_only_mode;
-      auto db_read_only_mode_guard = fc::make_scoped_exit([&]{
-         if( switch_read_mode )
-            chain.unset_db_read_only_mode();
-      });
-      if( switch_read_mode )
-         chain.set_db_read_only_mode();
-
+      auto remaining_time_in_read_window_us = _ro_read_window_time_us - time_used_in_read_window_us;
      // Ensure the trx to finish by the end of read-window.
-      auto trace = chain.push_transaction( trx, _ro_window_deadline, _ro_max_trx_time_us, 0, false, 0 );
+      auto window_deadline = std::min( start + remaining_time_in_read_window_us, block_deadline );
+      auto trace = chain.push_transaction( trx, window_deadline, _ro_max_trx_time_us, 0, false, 0 );
      auto pr = handle_push_result(trx, next, start, chain, trace, true /*return_failure_trace*/, true /*disable_subjective_enforcement*/, {} /*first_auth*/, 0 /*sub_bill*/, 0 /*prev_billed_cpu_time_us*/);
      // If a transaction was exhausted, that indicates we are close to
      // the end of read window. Retry in next round.
@@ -103,7 +103,7 @@ void test_trxs_common(std::vector<const char*>& specific_args) {

   for( size_t i = 1; i <= num_pushes; ++i ) {
      auto ptrx = make_unique_trx( chain_id );
-      app->executor().post( priority::low, exec_queue::read_write, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
+      app->executor().post( priority::low, exec_queue::general, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
         ++num_posts;
         bool return_failure_traces = true;
         app->get_method<plugin_interface::incoming::methods::transaction_async>()(ptrx,
@@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
            fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
            std::vector<const char*> argv =
                  {"test", "--data-dir", temp.c_str(), "--config-dir", temp.c_str(),
-                   "-p", "eosio", "-e", "--disable-subjective-billing=true"};
+                   "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true"};
            appbase::app().initialize<chain_plugin, producer_plugin>(argv.size(), (char**) &argv[0]);
            appbase::app().startup();
            plugin_promise.set_value(
@@ -112,7 +112,7 @@ BOOST_AUTO_TEST_CASE(producer) {
         fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
         std::vector<const char*> argv =
               {"test", "--data-dir", temp_dir_str.c_str(), "--config-dir", temp_dir_str.c_str(),
-                "-p", "eosio", "-e", "--disable-subjective-billing=true" };
+                "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true" };
         app->initialize<chain_plugin, producer_plugin>( argv.size(), (char**) &argv[0] );
         app->startup();
         plugin_promise.set_value(
@@ -51,7 +51,7 @@ void test_control_api_plugin::plugin_startup() {

   app().get_plugin<http_plugin>().add_api({
      TEST_CONTROL_RW_CALL(kill_node_on_producer, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_write);
+   }, appbase::exec_queue::general);
 }

 void test_control_api_plugin::plugin_shutdown() {}
@@ -115,7 +115,7 @@ void wallet_api_plugin::plugin_startup() {
            INVOKE_R_R_R(wallet_mgr, list_keys, std::string, std::string), 200),
       CALL_WITH_400(wallet, wallet_mgr, get_public_keys,
            INVOKE_R_V(wallet_mgr, get_public_keys), 200)
-   }, appbase::exec_queue::read_write);
+   }, appbase::exec_queue::general);
 }

 void wallet_api_plugin::plugin_initialize(const variables_map& options) {
@@ -111,7 +111,7 @@ int main(int argc, char** argv)
                       [&a=app](string, string, url_response_callback cb) {
         cb(200, fc::time_point::maximum(), fc::variant(fc::variant_object()));
         a->quit();
-      }, appbase::exec_queue::read_write );
+      }, appbase::exec_queue::general );
      app->startup();
      app->exec();
   } catch (const fc::exception& e) {
@@ -1,27 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
 apt-get update
 apt-get update --fix-missing
-export DEBIAN_FRONTEND='noninteractive'
-export TZ='Etc/UTC'
-apt-get install -y \
-    build-essential \
-    bzip2 \
-    cmake \
-    curl \
-    file \
-    git \
-    libbz2-dev \
-    libcurl4-openssl-dev \
-    libgmp-dev \
-    libncurses5 \
-    libssl-dev \
-    libtinfo-dev \
-    libzstd-dev \
-    python3 \
-    python3-numpy \
-    time \
-    tzdata \
-    unzip \
-    wget \
-    zip \
-    zlib1g-dev
+DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
+apt-get -y install zip unzip libncurses5 wget git build-essential cmake curl libgmp-dev libssl-dev libzstd-dev time zlib1g-dev libtinfo-dev bzip2 libbz2-dev python3 python3-numpy file
@@ -1,185 +1,151 @@
-#!/bin/bash
-set -eo pipefail
+#!/usr/bin/env bash

 echo "Leap Pinned Build"

 if [[ "$(uname)" == "Linux" ]]; then
-    if [[ -e /etc/os-release ]]; then
-        # obtain NAME and other information
-        . /etc/os-release
-        if [[ "${NAME}" != "Ubuntu" ]]; then
-            echo "Currently only supporting Ubuntu based builds. Proceed at your own risk."
-        fi
-    else
-        echo "Currently only supporting Ubuntu based builds. /etc/os-release not found. Your Linux distribution is not supported. Proceed at your own risk."
-    fi
+   if [[ -e /etc/os-release ]]; then
+      # obtain NAME and other information
+      . /etc/os-release
+      if [[ ${NAME} != "Ubuntu" ]]; then
+         echo "Currently only supporting Ubuntu based builds. Proceed at your own risk."
+      fi
+   else
+       echo "Currently only supporting Ubuntu based builds. /etc/os-release not found. Your Linux distribution is not supported. Proceed at your own risk."
+   fi
 else
    echo "Currently only supporting Ubuntu based builds. Your architecture is not supported. Proceed at your own risk."
 fi

-if [ $# -eq 0 ] || [ -z "$1" ]; then
-    echo "Please supply a directory for the build dependencies to be placed and a directory for leap build and a value for the number of jobs to use for building."
-    echo "The binary packages will be created and placed into the leap build directory."
-    echo "./pinned_build.sh <dependencies directory> <leap build directory> <1-100>"
-    exit 255
+if [ $# -eq 0 ] || [ -z "$1" ]
+   then
+      echo "Please supply a directory for the build dependencies to be placed and a directory for leap build and a value for the number of jobs to use for building."
+      echo "The binary packages will be created and placed into the leap build directory."
+      echo "./pinned_build.sh <dependencies directory> <leap build directory> <1-100>"
+      exit -1
 fi

-export CORE_SYM='EOS'
+CORE_SYM=EOS
 # CMAKE_C_COMPILER requires absolute path
-DEP_DIR="$(realpath "$1")"
-LEAP_DIR="$2"
-JOBS="$3"
+DEP_DIR=`realpath $1`
+LEAP_DIR=$2
+JOBS=$3
 CLANG_VER=11.0.1
 BOOST_VER=1.70.0
 LLVM_VER=7.1.0
+ARCH=`uname -m`
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
 START_DIR="$(pwd)"


 pushdir() {
-    DIR="$1"
-    mkdir -p "${DIR}"
-    pushd "${DIR}" &> /dev/null
+   DIR=$1
+   mkdir -p ${DIR}
+   pushd ${DIR} &> /dev/null
 }

 popdir() {
-    EXPECTED="$1"
-    D="$(popd)"
-    popd &> /dev/null
-    echo "${D}"
-    D="$(eval echo "$D" | head -n1 | cut -d " " -f1)"
+   EXPECTED=$1
+   D=`popd`
+   popd &> /dev/null
+   echo ${D}
+   D=`eval echo $D | head -n1 | cut -d " " -f1`

-    # -ef compares absolute paths
-    if ! [[ "${D}" -ef "${EXPECTED}" ]]; then
-        echo "Directory is not where expected EXPECTED=${EXPECTED} at ${D}"
-        exit 1
-    fi
+   # -ef compares absolute paths
+   if ! [[ ${D} -ef ${EXPECTED} ]]; then
+     echo "Directory is not where expected EXPECTED=${EXPECTED} at ${D}"
+     exit 1 
+   fi
 }

 try(){
-    "$@"
-    res=$?
-    if [[ ${res} -ne 0 ]]; then
-        exit 255
-    fi
+   output=$($@)
+   res=$?
+   if [[ ${res} -ne 0 ]]; then
+      exit -1
+   fi
 }

 install_clang() {
-    CLANG_DIR="$1"
-    if [ ! -d "${CLANG_DIR}" ]; then
-        echo "Installing Clang ${CLANG_VER} @ ${CLANG_DIR}"
-        mkdir -p "${CLANG_DIR}"
-        CLANG_FN="clang+llvm-${CLANG_VER}-x86_64-linux-gnu-ubuntu-16.04.tar.xz"
-        try wget -O "${CLANG_FN}" "https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VER}/${CLANG_FN}"
-        try tar -xvf "${CLANG_FN}" -C "${CLANG_DIR}"
-        pushdir "${CLANG_DIR}"
-        mv clang+*/* .
-        popdir "${DEP_DIR}"
-        rm "${CLANG_FN}"
-    fi
-    export PATH="${CLANG_DIR}/bin:$PATH"
-    export CLANG_DIR="${CLANG_DIR}"
+   CLANG_DIR=$1
+   if [ ! -d "${CLANG_DIR}" ]; then
+      echo "Installing Clang ${CLANG_VER} @ ${CLANG_DIR}"
+      mkdir -p ${CLANG_DIR}
+      CLANG_FN=clang+llvm-${CLANG_VER}-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+      try wget -O ${CLANG_FN} https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VER}/${CLANG_FN}
+      try tar -xvf ${CLANG_FN} -C ${CLANG_DIR}
+      pushdir ${CLANG_DIR}
+      mv clang+*/* .
+      popdir ${DEP_DIR}
+      rm ${CLANG_FN}
+   fi
+   export PATH=${CLANG_DIR}/bin:$PATH
+   export CLANG_DIR=${CLANG_DIR}
 }

 install_llvm() {
-    LLVM_DIR="$1"
-    if [ ! -d "${LLVM_DIR}" ]; then
-        echo "Installing LLVM ${LLVM_VER} @ ${LLVM_DIR}"
-        mkdir -p "${LLVM_DIR}"
-        try wget -O "llvm-${LLVM_VER}.src.tar.xz" "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/llvm-${LLVM_VER}.src.tar.xz"
-        try tar -xvf "llvm-${LLVM_VER}.src.tar.xz"
-        pushdir "${LLVM_DIR}.src"
-        pushdir build
-        try cmake -DCMAKE_TOOLCHAIN_FILE="${SCRIPT_DIR}/pinned_toolchain.cmake" -DCMAKE_INSTALL_PREFIX="${LLVM_DIR}" -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=Off -DLLVM_ENABLE_RTTI=On -DLLVM_ENABLE_TERMINFO=Off -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO ..
-        try make -j "${JOBS}"
-        try make -j "${JOBS}" install
-        popdir "${LLVM_DIR}.src"
-        popdir "${DEP_DIR}"
-        rm -rf "${LLVM_DIR}.src"
-        rm "llvm-${LLVM_VER}.src.tar.xz"
-    fi
-    export LLVM_DIR="${LLVM_DIR}"
+   LLVM_DIR=$1
+   if [ ! -d "${LLVM_DIR}" ]; then
+      echo "Installing LLVM ${LLVM_VER} @ ${LLVM_DIR}"
+      mkdir -p ${LLVM_DIR}
+      try wget -O llvm-${LLVM_VER}.src.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/llvm-${LLVM_VER}.src.tar.xz
+      try tar -xvf llvm-${LLVM_VER}.src.tar.xz
+      pushdir "${LLVM_DIR}.src"
+      pushdir build
+      try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=${LLVM_DIR} -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=Off -DLLVM_ENABLE_RTTI=On -DLLVM_ENABLE_TERMINFO=Off -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO ..
+      try make -j${JOBS} 
+      try make -j${JOBS} install
+      popdir "${LLVM_DIR}.src"
+      popdir ${DEP_DIR}
+      rm -rf ${LLVM_DIR}.src 
+      rm llvm-${LLVM_VER}.src.tar.xz
+   fi
+   export LLVM_DIR=${LLVM_DIR}
 }

 install_boost() {
-    BOOST_DIR="$1"
+   BOOST_DIR=$1

-    if [ ! -d "${BOOST_DIR}" ]; then
-        echo "Installing Boost ${BOOST_VER} @ ${BOOST_DIR}"
-        try wget -O "boost_${BOOST_VER//\./_}.tar.gz" "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VER}/source/boost_${BOOST_VER//\./_}.tar.gz"
-        try tar --transform="s:^boost_${BOOST_VER//\./_}:boost_${BOOST_VER//\./_}patched:" -xvzf "boost_${BOOST_VER//\./_}.tar.gz" -C "${DEP_DIR}"
-        pushdir "${BOOST_DIR}"
-        patch -p1 < "${SCRIPT_DIR}/0001-beast-fix-moved-from-executor.patch"
-        try ./bootstrap.sh -with-toolset=clang --prefix="${BOOST_DIR}/bin"
-        ./b2 toolset=clang cxxflags="-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I\${CLANG_DIR}/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fPIE" linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j "${JOBS}" install
-        popdir "${DEP_DIR}"
-        rm "boost_${BOOST_VER//\./_}.tar.gz"
-    fi
-    export BOOST_DIR="${BOOST_DIR}"
+   if [ ! -d "${BOOST_DIR}" ]; then
+      echo "Installing Boost ${BOOST_VER} @ ${BOOST_DIR}"
+      try wget -O boost_${BOOST_VER//\./_}.tar.gz https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VER}/source/boost_${BOOST_VER//\./_}.tar.gz
+      try tar --transform="s:^boost_${BOOST_VER//\./_}:boost_${BOOST_VER//\./_}patched:" -xvzf boost_${BOOST_VER//\./_}.tar.gz -C ${DEP_DIR}
+      pushdir ${BOOST_DIR}
+      patch -p1 < "${SCRIPT_DIR}/0001-beast-fix-moved-from-executor.patch"
+      try ./bootstrap.sh -with-toolset=clang --prefix=${BOOST_DIR}/bin
+      ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I${CLANG_DIR}/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fPIE' linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j${JOBS} install
+      popdir ${DEP_DIR}
+      rm boost_${BOOST_VER//\./_}.tar.gz
+   fi
+   export BOOST_DIR=${BOOST_DIR}
 }

-pushdir "${DEP_DIR}"
+pushdir ${DEP_DIR}

-install_clang "${DEP_DIR}/clang-${CLANG_VER}"
-install_llvm "${DEP_DIR}/llvm-${LLVM_VER}"
-install_boost "${DEP_DIR}/boost_${BOOST_VER//\./_}patched"
+install_clang ${DEP_DIR}/clang-${CLANG_VER}
+install_llvm ${DEP_DIR}/llvm-${LLVM_VER}
+install_boost ${DEP_DIR}/boost_${BOOST_VER//\./_}patched

 # go back to the directory where the script starts
-popdir "${START_DIR}"
+popdir ${START_DIR}

-pushdir "${LEAP_DIR}"
+pushdir ${LEAP_DIR}

 # build Leap
 echo "Building Leap ${SCRIPT_DIR}"
-try cmake -DCMAKE_TOOLCHAIN_FILE="${SCRIPT_DIR}/pinned_toolchain.cmake" -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="${LLVM_DIR}/lib/cmake" -DCMAKE_PREFIX_PATH="${BOOST_DIR}/bin" "${SCRIPT_DIR}/.."
+try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=${LLVM_DIR}/lib/cmake -DCMAKE_PREFIX_PATH=${BOOST_DIR}/bin ${SCRIPT_DIR}/..

-try make -j "${JOBS}"
+try make -j${JOBS}
 try cpack

-# art generated with DALL-E (https://openai.com/blog/dall-e), then fed through ASCIIart.club (https://asciiart.club) with permission
-cat <<'TXT' # BASH interpolation must remain disabled for the ASCII art to print correctly
-
-                                ,▄▄A`
-                             _╓▄██`
-                          ╓▄▓▓▀▀`
-                        ╓▓█▀╓▄▓
-                        ▓▌▓▓▓▀
-                      ,▄▓███▓H
-                    _╨╫▀╚▀╠▌`╙¥,
-                  ╓«    _╟▄▄   `½,                              ╓▄▄╦▄≥_
-                 ╙▓╫╬▒R▀▀╙▀▀▓φ_ «_╙Y╥▄mmMM#╦▄,_          ,╓╦mM╩╨╙╙╙\`║═
-                   ``       `▀▄__╫▓▓╨`    _```"""*ⁿⁿ^`````Ω,        `╟∩
-                              ╙▌▓▓"`    ,«ñ`            ╔╬▓▌⌂        ╔▌
-                               ╙█▌,,╔╗M╨,░             `  "╫▓m_      ╟H      _
-                        _,,,,__,╠█▓▒`  .╣▌µ _       _.╓╔▄▄▓█▓▓N_     ╙▀╩KKM╙╟▓N
-                      ,▄▓█▓████▀▀▀╙▀╓╔φ»█▓▓Ñ╦«, :»»µ╦▓▓█▀└╙▀███▓╥__   _,╓▄▓▓▓M▓`,
-                  __╓Φ▓█╫▓▓▓▓▓▓▓▓▓▓▓▓▓▀K▀▀███▓▓▓▓▓▓▀▀╙       `▀▀▀▓▄▄K╨╙└   `▀▌╙█▄*.
-                ,╓Φ▓▓▀▄▓▀`         ╙▀╙                                       ╙▓╙╙▓▄*
-              .▄▓╫▀╦▄▀`                                                       ╙▓╙µ╙▀
-             ▄▓▀╨▓▓╨_                                                          `▀▄▄M
-            _█▌▄▌╙`                                                               `╙
-             ╙└`
-
-
-                Ñ▓▓▓▓          ¢▄▄▄▄▄▄▄▄▄▄         ,           ,,,,,,,,,,,_
-               _╫▓▓█▌          ╟▓████████▀       ╓╣▓▌_         ╠╫▓█▓▓▓▓█▓▓▓▓▄
-               _▓▓▓█▌          ╟╫██▄,,,,_       æ▄███▓▄        ▐║████▀╨╨▀▓███▌
-               :╫▓▓█▌          ╟╢████████▓    ,╬███████▓,      ▐║████▓▄▄▓▓███▀
-               :╫▓▓█▌_        _╟║███▀▀▀▀▀▀   ╓╫███╣╫╬███▓N_    j▐██▀▀▀▀▀▀▀▀▀`
-    ___________]╫▓▓█▓φ╓╓╓╓╓╓,__╟╣█▌▌,,,,,_ _╬▓████████████▓▄_  ▐M█▌
-    _ _________]╫▌▓█████████▓▓▄╟╣████████▓▄╣██▓▀^    _ ╙▓██▓▓╗__M█▓
-      ___   _ _ ▀╣▀╣╩╩╩╩╩╩╩▀▀▀▀╙▀▀▀▀▀▀▀▀▀▀▀▀▀▀`          ╙▀▀▀▀╩═╩▀▀
-     _ __ __ _____ ___  ____ _  __  _  ____ ___
-    ____ ____ __  _ _  _ _ _ _ __   _ __ __
-    __ _ ________  ___ ________   ___ _ _ _      _
-       _ __  __     _ __ ____ _     _ ____ _   _ _       _
-     _  _ _     ____ ____ _ _    _ __  _ _ _ __
-
---
-Leap has successfully built and constructed its packages. You should be able to
-find the packages at:
-TXT
-echo "${LEAP_DIR}"
-echo
-echo 'Thank you, fam!!'
-echo
+echo " .----------------.  .----------------.  .----------------.  .----------------. ";
+echo "| .--------------. || .--------------. || .--------------. || .--------------. |";
+echo "| |   _____      | || |  _________   | || |      __      | || |   ______     | |";
+echo "| |  |_   _|     | || | |_   ___  |  | || |     /  \     | || |  |_   __ \   | |";
+echo "| |    | |       | || |   | |_  \_|  | || |    / /\ \    | || |    | |__) |  | |";
+echo "| |    | |   _   | || |   |  _|  _   | || |   / ____ \   | || |    |  ___/   | |";
+echo "| |   _| |__/ |  | || |  _| |___/ |  | || | _/ /    \ \_ | || |   _| |_      | |";
+echo "| |  |________|  | || | |_________|  | || ||____|  |____|| || |  |_____|     | |";
+echo "| |              | || |              | || |              | || |              | |";
+echo "| '--------------' || '--------------' || '--------------' || '--------------' |";
+echo " '----------------'  '----------------'  '----------------'  '----------------' ";
+echo "Leap has successfully built and constructed its packages.  You should be able to find the packages at ${LEAP_DIR}.  Enjoy!!!"
@@ -125,12 +125,8 @@ add_test(NAME nodeos_protocol_feature_test COMMAND tests/nodeos_protocol_feature
 set_property(TEST nodeos_protocol_feature_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME compute_transaction_test COMMAND tests/compute_transaction_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST compute_transaction_test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 0 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST read-only-trx-basic-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST read-only-trx-parallel-test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST read-only-trx-parallel-eos-vm-oc-test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME read_only_trx_test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST read_only_trx_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME subjective_billing_test COMMAND tests/subjective_billing_test.py -v -p 2 -n 4 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST subjective_billing_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME get_account_test COMMAND tests/get_account_test.py -v -p 2 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
@@ -1693,6 +1693,7 @@ class Cluster(object):
                            waitToComplete:bool=False, abiFile=None, actionsData=None, actionsAuths=None):
        Utils.Print("Configure txn generators")
        node=self.getNode(nodeId)
+        p2pListenPort = self.getNodeP2pPort(nodeId)
        info = node.getInfo()
        chainId = info['chain_id']
        lib_id = info['last_irreversible_block_id']
@@ -1701,12 +1702,13 @@ class Cluster(object):
        tpsLimitPerGenerator=tpsPerGenerator

        self.preExistingFirstTrxFiles = glob.glob(f"{Utils.DataDir}/first_trx_*.txt")
-        connectionPairList = [f"{self.host}:{self.getNodeP2pPort(nodeId)}"]
-        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=targetTps, tpsLimitPerGenerator=tpsLimitPerGenerator, connectionPairList=connectionPairList)
+
+        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=targetTps, tpsLimitPerGenerator=tpsLimitPerGenerator)
        self.trxGenLauncher = TransactionGeneratorsLauncher(chainId=chainId, lastIrreversibleBlockId=lib_id,
                                                    contractOwnerAccount=contractOwnerAcctName, accts=','.join(map(str, acctNamesList)),
                                                    privateKeys=','.join(map(str, acctPrivKeysList)), trxGenDurationSec=durationSec, logDir=Utils.DataDir,
-                                                    abiFile=abiFile, actionsData=actionsData, actionsAuths=actionsAuths, tpsTrxGensConfig=tpsTrxGensConfig)
+                                                    abiFile=abiFile, actionsData=actionsData, actionsAuths=actionsAuths,
+                                                    peerEndpoint=self.host, port=p2pListenPort, tpsTrxGensConfig=tpsTrxGensConfig)

        Utils.Print("Launch txn generators and start generating/sending transactions")
        self.trxGenLauncher.launch(waitToComplete=waitToComplete)
@@ -484,10 +484,6 @@ class Node(Transactions):

    def scheduleSnapshot(self):
        return self.processUrllibRequest("producer", "schedule_snapshot")
-    
-    def scheduleSnapshotAt(self, sbn):
-        param = { "start_block_num": sbn, "end_block_num": sbn }
-        return self.processUrllibRequest("producer", "schedule_snapshot", param)

    # kill all existing nodeos in case lingering from previous test
    @staticmethod
@@ -10,19 +10,16 @@ sys.path.append(harnessPath)

 from .testUtils import Utils
 from pathlib import Path
-from re import sub

 Print = Utils.Print

 class TpsTrxGensConfig:

-    def __init__(self, targetTps: int, tpsLimitPerGenerator: int, connectionPairList: list):
+    def __init__(self, targetTps: int, tpsLimitPerGenerator: int):
        self.targetTps: int = targetTps
        self.tpsLimitPerGenerator: int = tpsLimitPerGenerator
-        self.connectionPairList = connectionPairList
-        self.numConnectionPairs = len(self.connectionPairList)
-        round_to_multiple = lambda num, multiple: math.ceil(num/multiple) * multiple
-        self.numGenerators = round_to_multiple(math.ceil(self.targetTps / self.tpsLimitPerGenerator), self.numConnectionPairs)
+
+        self.numGenerators = math.ceil(self.targetTps / self.tpsLimitPerGenerator)
        self.initialTpsPerGenerator = math.floor(self.targetTps / self.numGenerators)
        self.modTps = self.targetTps % self.numGenerators
        self.cleanlyDivisible = self.modTps == 0
@@ -38,7 +35,8 @@ class TpsTrxGensConfig:
 class TransactionGeneratorsLauncher:

    def __init__(self, chainId: int, lastIrreversibleBlockId: int, contractOwnerAccount: str, accts: str, privateKeys: str, trxGenDurationSec: int, logDir: str,
-                 abiFile: Path, actionsData, actionsAuths, tpsTrxGensConfig: TpsTrxGensConfig):
+                 abiFile: Path, actionsData, actionsAuths,
+                 peerEndpoint: str, port: int, tpsTrxGensConfig: TpsTrxGensConfig):
        self.chainId = chainId
        self.lastIrreversibleBlockId = lastIrreversibleBlockId
        self.contractOwnerAccount  = contractOwnerAccount
@@ -48,14 +46,14 @@ class TransactionGeneratorsLauncher:
        self.tpsTrxGensConfig = tpsTrxGensConfig
        self.logDir = logDir
        self.abiFile = abiFile
-        self.actionsData = actionsData
-        self.actionsAuths = actionsAuths
+        self.actionsData=actionsData
+        self.actionsAuths=actionsAuths
+        self.peerEndpoint = peerEndpoint
+        self.port = port

    def launch(self, waitToComplete=True):
        self.subprocess_ret_codes = []
-        connectionPairIter = 0
        for id, targetTps in enumerate(self.tpsTrxGensConfig.targetTpsPerGenList):
-            connectionPair = self.tpsTrxGensConfig.connectionPairList[connectionPairIter].rsplit(":")
            popenStringList = [
                                './tests/trx_generator/trx_generator',
                                '--generator-id', f'{id}',
@@ -67,8 +65,8 @@ class TransactionGeneratorsLauncher:
                                '--trx-gen-duration', f'{self.trxGenDurationSec}',
                                '--target-tps', f'{targetTps}',
                                '--log-dir', f'{self.logDir}',
-                                '--peer-endpoint', f'{connectionPair[0]}',
-                                '--port', f'{connectionPair[1]}']
+                                '--peer-endpoint', f'{self.peerEndpoint}',
+                                '--port', f'{self.port}']
            if self.abiFile is not None and self.actionsData is not None and self.actionsAuths is not None:
                popenStringList.extend(['--abi-file', f'{self.abiFile}',
                                        '--actions-data', f'{self.actionsData}',
@@ -76,7 +74,6 @@ class TransactionGeneratorsLauncher:
            if Utils.Debug:
                Print(f"Running trx_generator: {' '.join(popenStringList)}")
            self.subprocess_ret_codes.append(subprocess.Popen(popenStringList))
-            connectionPairIter = (connectionPairIter + 1) % len(self.tpsTrxGensConfig.connectionPairList)
        exitCodes=None
        if waitToComplete:
            exitCodes = [ret_code.wait() for ret_code in self.subprocess_ret_codes]
@@ -103,20 +100,20 @@ def parseArgs():
    parser.add_argument("abi_file", type=str, help="The path to the contract abi file to use for the supplied transaction action data")
    parser.add_argument("actions_data", type=str, help="The json actions data file or json actions data description string to use")
    parser.add_argument("actions_auths", type=str, help="The json actions auth file or json actions auths description string to use, containting authAcctName to activePrivateKey pairs.")
-    parser.add_argument("connection_pair_list", type=str, help="Comma separated list of endpoint:port combinations to send transactions to", default="localhost:9876")
+    parser.add_argument("peer_endpoint", type=str, help="set the peer endpoint to send transactions to", default="127.0.0.1")
+    parser.add_argument("port", type=int, help="set the peer endpoint port to send transactions to", default=9876)
    args = parser.parse_args()
    return args

 def main():
    args = parseArgs()
-    connectionPairList = sub('[\s+]', '', args.connection_pair_list)
-    connectionPairList = connectionPairList.rsplit(',')

    trxGenLauncher = TransactionGeneratorsLauncher(chainId=args.chain_id, lastIrreversibleBlockId=args.last_irreversible_block_id,
                                                   contractOwnerAccount=args.contract_owner_account, accts=args.accounts,
                                                   privateKeys=args.priv_keys, trxGenDurationSec=args.trx_gen_duration, logDir=args.log_dir,
                                                   abiFile=args.abi_file, actionsData=args.actions_data, actionsAuths=args.actions_auths,
-                                                   tpsTrxGensConfig=TpsTrxGensConfig(targetTps=args.target_tps, tpsLimitPerGenerator=args.tps_limit_per_generator, connectionPairList=connectionPairList))
+                                                   peerEndpoint=args.peer_endpoint, port=args.port,
+                                                   tpsTrxGensConfig=TpsTrxGensConfig(targetTps=args.target_tps, tpsLimitPerGenerator=args.tps_limit_per_generator))


    exit_codes = trxGenLauncher.launch()
@@ -158,8 +158,7 @@ try:
    assert steadyStateAvg>=minRequiredTransactions, "Expected to at least receive %s transactions per block, but only getting %s" % (minRequiredTransactions, steadyStateAvg)

    Print("Create snapshot")
-    blockNumSnap=nodeSnap.getBlockNum(BlockType.head)
-    ret = nodeProg.scheduleSnapshotAt(blockNumSnap)
+    ret = nodeProg.scheduleSnapshot()
    assert ret is not None, "Snapshot creation failed"
    
    ret = nodeSnap.createSnapshot()
@@ -488,7 +488,8 @@ Performance Test Basic Single Test:
 * `abi_file`                    The path to the contract abi file to use for the supplied transaction action data
 * `actions_data`                The json actions data file or json actions data description string to use
 * `actions_auths`               The json actions auth file or json actions auths description string to use, containting authAcctName to activePrivateKey pairs.
-* `connection_pair_list`        Comma separated list of endpoint:port combinations to send transactions to
+* `peer_endpoint`               set the peer endpoint to send transactions to, default="127.0.0.1"
+* `port`                        set the peer endpoint port to send transactions to, default=9876
 </details>

 #### Transaction Generator
@@ -327,9 +327,7 @@ class PerformanceTestBasic:
    def runTpsTest(self) -> PtbTpsTestResult:
        completedRun = False
        self.producerNode = self.cluster.getNode(self.producerNodeId)
-        self.connectionPairList = []
-        for producer in range(0, self.clusterConfig.pnodes):
-            self.connectionPairList.append(f"{self.cluster.getNode(producer).host}:{self.cluster.getNodeP2pPort(producer)}")
+        self.producerP2pPort = self.cluster.getNodeP2pPort(self.producerNodeId)
        self.validationNode = self.cluster.getNode(self.validationNodeId)
        self.wallet = self.walletMgr.create('default')
        self.setupContract()
@@ -373,13 +371,13 @@ class PerformanceTestBasic:
        self.cluster.biosNode.kill(signal.SIGTERM)

        self.data.startBlock = self.waitForEmptyBlocks(self.validationNode, self.emptyBlockGoal)
-        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=self.ptbConfig.targetTps, tpsLimitPerGenerator=self.ptbConfig.tpsLimitPerGenerator, connectionPairList=self.connectionPairList)
+        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=self.ptbConfig.targetTps, tpsLimitPerGenerator=self.ptbConfig.tpsLimitPerGenerator)

        self.cluster.trxGenLauncher = TransactionGeneratorsLauncher(chainId=chainId, lastIrreversibleBlockId=lib_id, contractOwnerAccount=self.clusterConfig.specifiedContract.account.name,
                                                       accts=','.join(map(str, self.accountNames)), privateKeys=','.join(map(str, self.accountPrivKeys)),
                                                       trxGenDurationSec=self.ptbConfig.testTrxGenDurationSec, logDir=self.trxGenLogDirPath,
                                                       abiFile=abiFile, actionsData=actionsDataJson, actionsAuths=actionsAuthsJson,
-                                                       tpsTrxGensConfig=tpsTrxGensConfig)
+                                                       peerEndpoint=self.producerNode.host, port=self.producerP2pPort, tpsTrxGensConfig=tpsTrxGensConfig)

        trxGenExitCodes = self.cluster.trxGenLauncher.launch()
        print(f"Transaction Generator exit codes: {trxGenExitCodes}")
@@ -1,11 +1,8 @@
 #!/usr/bin/env python3

 import random
-import time
-import signal
-import threading

-from TestHarness import Account, Cluster, ReturnType, TestHelper, Utils, WalletMgr
+from TestHarness import Account, Cluster, TestHelper, Utils, WalletMgr
 from TestHarness.TestHelper import AppArgs

 ###############################################################
@@ -21,7 +18,6 @@ errorExit=Utils.errorExit

 appArgs=AppArgs()
 appArgs.add(flag="--read-only-threads", type=int, help="number of read-only threads", default=0)
-appArgs.add(flag="--num-test-runs", type=int, help="number of times to run the tests", default=1)
 appArgs.add_bool(flag="--eos-vm-oc-enable", help="enable eos-vm-oc")
 appArgs.add(flag="--wasm-runtime", type=str, help="if set to eos-vm-oc, must compile with EOSIO_EOS_VM_OC_DEVELOPER", default="eos-vm-jit")

@@ -33,10 +29,6 @@ pnodes=args.p
 topo=args.s
 delay=args.d
 total_nodes = pnodes if args.n < pnodes else args.n
-# For this test, we need at least 1 non-producer
-if total_nodes <= pnodes:
-    Print ("non-producing nodes %d must be greater than 0. Force it to %d. producing nodes: %d," % (total_nodes - pnodes, pnodes + 1, pnodes))
-    total_nodes = pnodes + 1
 debug=args.v
 nodesFile=args.nodes_file
 dontLaunch=nodesFile is not None
@@ -45,7 +37,6 @@ dontKill=args.leave_running
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
-numTestRuns=args.num_test_runs

 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -54,7 +45,6 @@ if nodesFile is not None:

 Utils.Debug=debug
 testSuccessful=False
-errorInThread=False

 random.seed(seed) # Use a fixed seed for repeatability.
 cluster=Cluster(walletd=True,unshared=args.unshared)
@@ -64,9 +54,6 @@ EOSIO_ACCT_PRIVATE_DEFAULT_KEY = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79z
 EOSIO_ACCT_PUBLIC_DEFAULT_KEY = "EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV"

 try:
-    TestHelper.printSystemInfo("BEGIN")
-    cluster.setWalletMgr(walletMgr)
-
    if dontLaunch: # run test against remote cluster
        jsonStr=None
        with open(nodesFile, "r") as f:
@@ -84,24 +71,11 @@ try:
            cluster.killall(allInstances=killAll)
            cluster.cleanup()

-    Print ("producing nodes: %d, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" % (pnodes, total_nodes-pnodes, topo, delay))
+    Print ("producing nodes: %s, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" % (pnodes, total_nodes-pnodes, topo, delay))

-    cluster.killall(allInstances=killAll)
-    cluster.cleanup()
    Print("Stand up cluster")
-    # set up read-only options for API node
-    specificExtraNodeosArgs={}
-    # producer nodes will be mapped to 0 through pnodes-1, so the number pnodes is the no-producing API node
-    specificExtraNodeosArgs[pnodes]=" --plugin eosio::net_api_plugin"
-    specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
-    specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
-    if args.eos_vm_oc_enable:
-        specificExtraNodeosArgs[pnodes]+=" --eos-vm-oc-enable"
-    if args.wasm_runtime:
-        specificExtraNodeosArgs[pnodes]+=" --wasm-runtime "
-        specificExtraNodeosArgs[pnodes]+=args.wasm_runtime
    extraNodeosArgs=" --http-max-response-time-ms 990000 --disable-subjective-api-billing false "
-    if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, specificExtraNodeosArgs=specificExtraNodeosArgs, extraNodeosArgs=extraNodeosArgs ) is False:
+    if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay,extraNodeosArgs=extraNodeosArgs ) is False:
        errorExit("Failed to stand up eos cluster.")

    Print ("Wait for Cluster stabilization")
@@ -123,7 +97,7 @@ try:
    userAccount = Account(userAccountName)
    userAccount.ownerPublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
    userAccount.activePublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
-    cluster.createAccountAndVerify(userAccount, cluster.eosioAccount, stakeCPU=2000)
+    cluster.createAccountAndVerify(userAccount, cluster.eosioAccount)

    noAuthTableContractDir="unittests/test-contracts/no_auth_table"
    noAuthTableWasmFile="no_auth_table.wasm"
@@ -142,149 +116,28 @@ try:
        }
        return apiNode.pushTransaction(trx, opts)

-    def sendReadOnlyTrxOnThread(startId, numTrxs):
-        Print("start sendReadOnlyTrxOnThread")
+    Print("Insert a user")
+    results = sendTransaction('insert', {"user": userAccountName, "id": 1, "age": 10}, auth=[{"actor": userAccountName, "permission":"active"}])
+    assert(results[0])
+    apiNode.waitForTransactionInBlock(results[1]['transaction_id'])

-        global errorInThread
-        errorInThread = False
-        try:
-           for i in range(numTrxs):
-               results = sendTransaction('age', {"user": userAccountName, "id": startId + i}, opts='--read')
-               assert(results[0])
-               assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 25)
-        except Exception as e:
-            Print("Exception in sendReadOnlyTrxOnThread: ", e)
-            errorInThread = True
+    # verify the return value (age) from read-only is the same as created.
+    Print("Send a read-only Get transaction to verify previous Insert")
+    results = sendTransaction('getage', {"user": userAccountName}, opts='--read')
+    assert(results[0])
+    assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 10)

-    def sendTrxsOnThread(startId, numTrxs, opts=None):
-        Print("sendTrxsOnThread: ", startId, numTrxs, opts)
+    # verify non-read-only modification works
+    Print("Send a non-read-only Modify transaction")
+    results = sendTransaction('modify', {"user": userAccountName, "age": 25},        auth=[{"actor": userAccountName, "permission": "active"}])
+    assert(results[0])
+    apiNode.waitForTransactionInBlock(results[1]['transaction_id'])

-        global errorInThread
-        errorInThread = False
-        try:
-            for i in range(numTrxs):
-                results = sendTransaction('age', {"user": userAccountName, "id": startId + i}, auth=[{"actor": userAccountName, "permission":"active"}], opts=opts)
-                assert(results[0])
-        except Exception as e:
-            Print("Exception in sendTrxsOnThread: ", e)
-            errorInThread = True
-
-    def doRpc(resource, command, numRuns, fieldIn, expectedValue, code, payload={}):
-        global errorInThread
-        errorInThread = False
-        try:
-            for i in range(numRuns):
-                ret_json = apiNode.processUrllibRequest(resource, command, payload)
-                if code is None:
-                    assert(ret_json["payload"][fieldIn] is not None)
-                    if expectedValue is not None:
-                        assert(ret_json["payload"][fieldIn] == expectedValue)
-                else:
-                    assert(ret_json["code"] == code)
-        except Exception as e:
-            Print("Exception in doRpc: ", e)
-            errorInThread = True
-
-    def runReadOnlyTrxAndRpcInParallel(resource, command, fieldIn=None, expectedValue=None, code=None, payload={}):
-        Print("runReadOnlyTrxAndRpcInParallel: ", command)
-
-        numRuns = 10
-        trxThread = threading.Thread(target = sendReadOnlyTrxOnThread, args = (0, numRuns ))
-        rpcThread = threading.Thread(target = doRpc, args = (resource, command, numRuns, fieldIn, expectedValue, code, payload))
-        trxThread.start()
-        rpcThread.start()
-
-        trxThread.join()
-        rpcThread.join()
-        assert(not errorInThread)
-
-    def mixedOpsTest(opt=None):
-        Print("mixedOpsTest -- opt = ", opt)
-
-        numRuns = 300
-        readOnlyThread = threading.Thread(target = sendReadOnlyTrxOnThread, args = (0, numRuns ))
-        readOnlyThread.start()
-        sendTrxThread = threading.Thread(target = sendTrxsOnThread, args = (numRuns, numRuns, opt))
-        sendTrxThread.start()
-        pushBlockThread = threading.Thread(target = doRpc, args = ("chain", "push_block", numRuns, None, None, 202, {"block":"signed_block"}))
-        pushBlockThread.start()
-
-        readOnlyThread.join()
-        sendTrxThread.join()
-        pushBlockThread.join()
-        assert(not errorInThread)
-
-    def sendMulReadOnlyTrx(numThreads):
-        threadList = []
-        num_trxs_per_thread = 500
-        for i in range(numThreads):
-            thr = threading.Thread(target = sendReadOnlyTrxOnThread, args = (i * num_trxs_per_thread, num_trxs_per_thread ))
-            thr.start()
-            threadList.append(thr)
-        for thr in threadList:
-            thr.join()
-
-    def basicTests():
-        Print("Insert a user")
-        results = sendTransaction('insert', {"user": userAccountName, "id": 1, "age": 10}, auth=[{"actor": userAccountName, "permission":"active"}])
-        assert(results[0])
-        apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
-
-        # verify the return value (age) from read-only is the same as created.
-        Print("Send a read-only Get transaction to verify previous Insert")
-        results = sendTransaction('getage', {"user": userAccountName}, opts='--read')
-        assert(results[0])
-        assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 10)
-
-        # verify non-read-only modification works
-        Print("Send a non-read-only Modify transaction")
-        results = sendTransaction('modify', {"user": userAccountName, "age": 25}, auth=[{"actor": userAccountName, "permission": "active"}])
-        assert(results[0])
-        apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
-
-    def multiReadOnlyTests():
-        Print("Verify multiple read-only Get actions work after Modify")
-        sendMulReadOnlyTrx(numThreads=5)
-
-    def chainApiTests():
-        # verify chain APIs can run in parallel with read-ony transactions
-        runReadOnlyTrxAndRpcInParallel("chain", "get_info", "server_version")
-        runReadOnlyTrxAndRpcInParallel("chain", "get_consensus_parameters", "chain_config")
-        runReadOnlyTrxAndRpcInParallel("chain", "get_activated_protocol_features", "activated_protocol_features")
-        runReadOnlyTrxAndRpcInParallel("chain", "get_block", "block_num", expectedValue=1, payload={"block_num_or_id":1})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_block_info", "block_num", expectedValue=1, payload={"block_num":1})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_account", "account_name", expectedValue=userAccountName, payload = {"account_name":userAccountName})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_code", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_code_hash", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_raw_code_and_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_raw_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_producers", "rows", payload = {"json":"true","lower_bound":""})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", "rows", payload = {"json":"true","code":"eosio","scope":"eosio","table":"global"})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_table_by_scope", fieldIn="rows", payload = {"json":"true","table":"noauth"})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_currency_balance", code=200, payload = {"code":"eosio.token", "account":testAccountName})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_currency_stats", fieldIn="SYS", payload = {"code":"eosio.token", "symbol":"SYS"})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_required_keys", code=400)
-        runReadOnlyTrxAndRpcInParallel("chain", "get_transaction_id", code=200, payload = {"ref_block_num":"1"})
-        runReadOnlyTrxAndRpcInParallel("chain", "push_block", code=202, payload = {"block":"signed_block"})
-        runReadOnlyTrxAndRpcInParallel("chain", "get_producer_schedule", "active")
-        runReadOnlyTrxAndRpcInParallel("chain", "get_scheduled_transactions", "transactions", payload = {"json":"true","lower_bound":""})
-
-    def netApiTests():
-        # NET APIs
-        runReadOnlyTrxAndRpcInParallel("net", "status", code=201, payload = "localhost")
-        runReadOnlyTrxAndRpcInParallel("net", "connections", code=201)
-        runReadOnlyTrxAndRpcInParallel("net", "connect", code=201, payload = "localhost")
-        runReadOnlyTrxAndRpcInParallel("net", "disconnect", code=201, payload = "localhost")
-
-    basicTests()
-
-    if args.read_only_threads > 0: # Save test time. No need to run other tests if multi-threaded is not enabled
-        for i in range(numTestRuns):
-            multiReadOnlyTests()
-            chainApiTests()
-            netApiTests()
-            mixedOpsTest()
+    # verify 'cleos push action getage "user": "user" --read' works
+    Print("Send a read-only Get action")
+    results = apiNode.pushMessage(testAccountName, 'getage', "{{\"user\": \"{}\"}}".format(userAccountName), opts='--read');
+    assert(results[0])
+    assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 25)

    testSuccessful = True
 finally: