Merge pull request #1307 from AntelopeIO/bump_to_4_0_2

[4.0] Bump Leap version to 4.0.2

.github/actions/parallel-ctest-containers/main.mjs

@@ -11,8 +11,10 @@ const error_log_paths = JSON.parse(core.getInput('error-log-paths', {required: t
 const log_tarball_prefix = core.getInput('log-tarball-prefix', {required: true});
 const tests_label = core.getInput('tests-label', {required: true});
 
+const repo_name = process.env.GITHUB_REPOSITORY.split('/')[1];
+
 try {
-   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", "/__w/leap/leap", container, "sh", "-c", "zstdcat /build.tar.zst | tar x"], {stdio:"inherit"}).status)
+   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", `/__w/${repo_name}/${repo_name}`, container, "sh", "-c", "zstdcat /build.tar.zst | tar x"], {stdio:"inherit"}).status)
       throw new Error("Failed to create base container");
    if(child_process.spawnSync("docker", ["commit", "base", "baseimage"], {stdio:"inherit"}).status)
       throw new Error("Failed to create base image");
@@ -45,13 +47,13 @@ try {
       let packer = tar.pack();
 
       extractor.on('entry', (header, stream, next) => {
-         if(!header.name.startsWith(`__w/leap/leap/build`)) {
+         if(!header.name.startsWith(`__w/${repo_name}/${repo_name}/build`)) {
             stream.on('end', () => next());
             stream.resume();
             return;
          }
 
-         header.name = header.name.substring(`__w/leap/leap/`.length);
+         header.name = header.name.substring(`__w/${repo_name}/${repo_name}/`.length);
          if(header.name !== "build/" && error_log_paths.filter(p => header.name.startsWith(p)).length === 0) {
             stream.on('end', () => next());
             stream.resume();

CMakeLists.txt

@@ -15,8 +15,8 @@ set( CXX_STANDARD_REQUIRED ON)
 
 set(VERSION_MAJOR 4)
 set(VERSION_MINOR 0)
-set(VERSION_PATCH 0)
-set(VERSION_SUFFIX dev)
+set(VERSION_PATCH 2)
+#set(VERSION_SUFFIX rc3)
 
 if(VERSION_SUFFIX)
     set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}")
@@ -77,29 +77,9 @@ if(ENABLE_OC AND CMAKE_SIZEOF_VOID_P EQUAL 8 AND NOT WIN32)
       list(APPEND EOSIO_WASM_RUNTIMES eos-vm-oc)
       # EOS VM OC requires LLVM, but move the check up here to a central location so that the EosioTester.cmakes
       # can be created with the exact version found
-      # find_package VERSION range is only available in 3.19 and newer.
-      if(${CMAKE_VERSION} VERSION_LESS "3.19")
-         find_package(LLVM REQUIRED CONFIG)
-      else()
-         # Preferring the latest LLVM version, search through major 11..7 and minor 9..0 in reverse order.
-         # Minor versions may break API. This manifests in the LLVM config files such that a range of `11...<12`
-         # will NOT find LLVM `11.1`. We loop over minor as well as major to resolve this.
-         set(max_minor 9)
-         set(major 11)
-         set(minor ${max_minor})
-         while(NOT LLVM_FOUND AND major GREATER_EQUAL 7)
-            math(EXPR end "${major} + 1")
-            find_package(LLVM ${major}.${minor}...<${end} CONFIG QUIET)
-            if(minor GREATER 0)
-               math(EXPR minor "${minor} - 1")
-            else()
-               math(EXPR major "${major} - 1")
-               set(minor ${max_minor})
-            endif()
-         endwhile()
-      endif()
+      find_package(LLVM REQUIRED CONFIG)
       if(LLVM_VERSION_MAJOR VERSION_LESS 7 OR LLVM_VERSION_MAJOR VERSION_GREATER_EQUAL 12)
-         message(FATAL_ERROR "Leap requires an LLVM version 7 through 11")
+	      message(FATAL_ERROR "Leap requires an LLVM version 7 through 11")
       endif()
    endif()
 endif()
@@ -288,18 +268,8 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib/python3/dist-packages/TestHarness DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages COMPONENT dev EXCLUDE_FROM_ALL)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/share/leap_testing/bin DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing COMPONENT dev EXCLUDE_FROM_ALL)
 else()
-   # The following install(CODE ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
-   # However, it can flag as an error if using `make package` instead of `make dev-install` for installation.
-   # This is due to those directories and symbolic links being created in the postinit script during the package install instead.
-
-   # Note If/when doing `make package`, it is not a true error if you see:
-   # Error creating directory "/<leap_install_dir>/lib/python3/dist-packages".
-   # CMake Error: failed to create symbolic link '/<leap_install_dir>/lib/python3/dist-packages/TestHarness': no such file or directory
-   # CMake Error: failed to create symbolic link '/<leap_install_dir>/share/leap_testing/bin': no such file or directory
-
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages)" COMPONENT dev EXCLUDE_FROM_ALL)
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)" COMPONENT dev EXCLUDE_FROM_ALL)
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)" COMPONENT dev EXCLUDE_FROM_ALL)
+   # The following install(SCRIPT ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
+   install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/scripts/install_testharness_symlinks.cmake COMPONENT dev EXCLUDE_FROM_ALL)
 
    # The `make package` installation of symlinks happens via the `postinst` script installed in cmake.package via the line below
 endif()

README.md

@@ -72,7 +72,7 @@ git clone --recursive https://github.com/AntelopeIO/leap.git
 git clone --recursive git@github.com:AntelopeIO/leap.git
 ```
 
-> ℹ️ **HTTPS vs. SSH Clone** ℹ️
+> ℹ️ **HTTPS vs. SSH Clone** ℹ️  
 Both an HTTPS or SSH git clone will yield the same result - a folder named `leap` containing our source code. It doesn't matter which type you use.
 
 Navigate into that folder:
@@ -96,13 +96,13 @@ git submodule update --init --recursive
 ### Step 3 - Build
 Select build instructions below for a [pinned build](#pinned-build) (preferred) or an [unpinned build](#unpinned-build).
 
-> ℹ️ **Pinned vs. Unpinned Build** ℹ️
+> ℹ️ **Pinned vs. Unpinned Build** ℹ️  
 We have two types of builds for Leap: "pinned" and "unpinned." The only difference is that pinned builds use specific versions for some dependencies hand-picked by the Leap engineers - they are "pinned" to those versions. In contrast, unpinned builds use the default dependency versions available on the build system at the time. We recommend performing a "pinned" build to ensure the compiler and boost versions remain the same between builds of different Leap versions. Leap requires these versions to remain the same, otherwise its state might need to be recovered from a portable snapshot or the chain needs to be replayed.
 
-> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️
+> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️  
 When building C/C++ software, often the build is performed in parallel via a command such as `make -j "$(nproc)"` which uses all available CPU threads. However, be aware that some compilation units (`*.cpp` files) in Leap will consume nearly 4GB of memory. Failures due to memory exhaustion will typically, but not always, manifest as compiler crashes. Using all available CPU threads may also prevent you from doing other things on your computer during compilation. For these reasons, consider reducing this value.
 
-> 🐋 **Docker and `sudo`** 🐋
+> 🐋 **Docker and `sudo`** 🐋  
 If you are in an Ubuntu docker container, omit `sudo` from all commands because you run as `root` by default. Most other docker containers also exclude `sudo`, especially Debian-family containers. If your shell prompt is a hash tag (`#`), omit `sudo`.
 
 #### Pinned Build
@@ -111,12 +111,14 @@ Make sure you are in the root of the `leap` repo, then run the `install_depts.sh
 sudo scripts/install_deps.sh
 ```
 
-Next, run the pinned build script. You have to give it three arguments, in the following order:
-  - A temporary folder, for all dependencies that need to be built from source.
-  - A build folder, where the binaries you need to install will be built to.
-  - The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
+Next, run the pinned build script. You have to give it three arguments in the following order:
+1. A temporary folder, for all dependencies that need to be built from source.
+1. A build folder, where the binaries you need to install will be built to.
+1. The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
 
-The following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads (Note: you don't need `sudo` for this command):
+> 🔒 You do not need to run this script with `sudo` or as root.
+
+For example, the following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads:
 ```bash
 scripts/pinned_build.sh deps build "$(nproc)"
 ```
@@ -146,11 +148,9 @@ To build, make sure you are in the root of the `leap` repo, then run the followi
 ```bash
 mkdir -p build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Release ..
+cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/usr/lib/llvm-11 ..
 make -j "$(nproc)" package
 ```
-
-If CMake fails to find a valid llvm configure file, it may be necessary to add `-DCMAKE_PREFIX_PATH=/usr/lib/llvm-11` to the cmake command.
 </details>
 
 <details> <summary>Ubuntu 18.04 Bionic</summary>

docs/01_nodeos/03_plugins/chain_plugin/index.md

@@ -120,7 +120,7 @@ Config Options for eosio::chain_plugin:
                                         applied to them (may specify multiple
                                         times)
   --read-mode arg (=head)               Database read mode ("head",
-                                        "irreversible").
+                                        "irreversible", "speculative").
                                         In "head" mode: database contains state
                                         changes up to the head block;
                                         transactions received by the node are
@@ -131,7 +131,14 @@ Config Options for eosio::chain_plugin:
                                         received via the P2P network are not
                                         relayed and transactions cannot be
                                         pushed via the chain API.
-
+                                        In "speculative" mode: (DEPRECATED:
+                                        head mode recommended) database
+                                        contains state changes by transactions
+                                        in the blockchain up to the head block
+                                        as well as some transactions not yet
+                                        included in the blockchain;
+                                        transactions received by the node are
+                                        relayed if valid.                                        
   --api-accept-transactions arg (=1)    Allow API transactions to be evaluated
                                         and relayed if valid.
   --validation-mode arg (=full)         Chain validation mode ("full" or

docs/01_nodeos/07_concepts/05_storage-and-read-modes.md

@@ -29,6 +29,7 @@ The `nodeos` service can be run in different "read" modes. These modes control h
 
 - `head`: this only includes the side effects of confirmed transactions, this mode processes unconfirmed transactions but does not include them.
 - `irreversible`: this mode also includes confirmed transactions only up to those included in the last irreversible block.
+- `speculative`: this includes the side effects of confirmed and unconfirmed transactions.
 
 A transaction is considered confirmed when a `nodeos` instance has received, processed, and written it to a block on the blockchain, i.e. it is in the head block or an earlier block.
 
@@ -44,6 +45,16 @@ When `nodeos` is configured to be in irreversible read mode, it will still track
 
 Clients such as `cleos` and the RPC API will see database state as of the current head block of the chain. It **will not** include changes made by transactions known to this node but not included in the chain, such as unconfirmed transactions.
 
+### Speculative Mode ( Deprecated )
+
+Clients such as `cleos` and the RPC API, will see database state as of the current head block plus changes made by all transactions known to this node but potentially not included in the chain, unconfirmed transactions for example.
+
+Speculative mode is low latency but fragile, there is no guarantee that the transactions reflected in the state will be included in the chain OR that they will reflected in the same order the state implies.
+
+This mode features the lowest latency, but is the least consistent.
+
+In speculative mode `nodeos` is able to execute transactions which have TaPoS (Transaction as Proof of Stake) pointing to any valid block in a fork considered to be the best fork by this node.
+
 ## How To Specify the Read Mode
 
 The mode in which `nodeos` is run can be specified using the `--read-mode` option from the `eosio::chain_plugin`.

libraries/chain/authorization_manager.cpp

@@ -559,8 +559,8 @@ namespace eosio { namespace chain {
 
       }
 
-      if( !allow_unused_keys  || check_but_dont_fail) {
-         EOS_ASSERT( checker.all_keys_used(), tx_irrelevant_sig,
+      if( !allow_unused_keys ) {
+         EOS_ASSERT( checker.all_keys_used() || check_but_dont_fail, tx_irrelevant_sig,
                      "transaction bears irrelevant signatures from these keys: ${keys}",
                      ("keys", checker.unused_keys()) );
       }

libraries/chain/block_log.cpp

@@ -496,19 +496,21 @@ namespace eosio { namespace chain {
 
       /// Would remove pre-existing block log and index, never write blocks into disk.
       struct empty_block_log final : block_log_impl {
+         uint32_t first_block_number = std::numeric_limits<uint32_t>::max();
+
          explicit empty_block_log(const bfs::path& log_dir) {
             fc::remove(log_dir / "blocks.log");
             fc::remove(log_dir / "blocks.index");
          }
 
-         uint32_t first_block_num() final { return head ? head->block_num() : 1; }
+         uint32_t first_block_num() final { return head ? head->block_num() : first_block_number; }
          void append(const signed_block_ptr& b, const block_id_type& id, const std::vector<char>& packed_block) final {
             update_head(b, id);
          }
 
          uint64_t get_block_pos(uint32_t block_num) final { return block_log::npos; }
          void reset(const genesis_state& gs, const signed_block_ptr& first_block) final { update_head(first_block); }
-         void reset(const chain_id_type& chain_id, uint32_t first_block_num) final {}
+         void reset(const chain_id_type& chain_id, uint32_t first_block_num) final { first_block_number = first_block_num; }
          void flush() final {}
 
          signed_block_ptr read_block_by_num(uint32_t block_num) final { return {}; };
@@ -1370,16 +1372,57 @@ namespace eosio { namespace chain {
    }
 
    // static
-   std::optional<genesis_state> block_log::extract_genesis_state(const fc::path& block_dir) {
-      boost::filesystem::path p(block_dir / "blocks.log");
-      for_each_file_in_dir_matches(block_dir, R"(blocks-1-\d+\.log)",
-                                   [&p](boost::filesystem::path log_path) { p = std::move(log_path); });
-      return block_log_data(p).get_genesis_state();
+   std::optional<block_log::chain_context> block_log::extract_chain_context( const fc::path& block_dir, const fc::path& retained_dir) {
+      boost::filesystem::path first_block_file;
+      if (!retained_dir.empty() && fc::exists(retained_dir)) {
+         for_each_file_in_dir_matches(retained_dir, R"(blocks-1-\d+\.log)",
+                                      [&](boost::filesystem::path log_path) {
+                                          first_block_file = std::move(log_path);
+                                      });
+      }
+
+      if (first_block_file.empty() && fc::exists(block_dir / "blocks.log")) {
+         first_block_file = block_dir / "blocks.log";
+      }
+
+      if (!first_block_file.empty()) {
+         return block_log_data(first_block_file).get_preamble().chain_context;
+      }
+      
+      if (!retained_dir.empty() && fc::exists(retained_dir)) {
+         const std::regex        my_filter(R"(blocks-\d+-\d+\.log)");
+         std::smatch             what;
+         bfs::directory_iterator end_itr; // Default ctor yields past-the-end
+         for (bfs::directory_iterator p(retained_dir); p != end_itr; ++p) {
+            // Skip if not a file
+            if (!bfs::is_regular_file(p->status()))
+               continue;
+            // skip if it does not match the pattern
+            if (!std::regex_match(p->path().filename().string(), what, my_filter))
+               continue;
+            return block_log_data(p->path()).chain_id();
+         }
+      }
+      return {};
    }
 
    // static
-   chain_id_type block_log::extract_chain_id(const fc::path& data_dir) {
-      return block_log_data(data_dir / "blocks.log").chain_id();
+   std::optional<genesis_state> block_log::extract_genesis_state(const fc::path& block_dir, const fc::path& retained_dir) {
+      auto context = extract_chain_context(block_dir, retained_dir);
+      if (!context || std::holds_alternative<chain_id_type>(*context))
+         return {};
+      return std::get<genesis_state>(*context);
+   }
+
+   // static
+   std::optional<chain_id_type> block_log::extract_chain_id(const fc::path& block_dir, const fc::path& retained_dir) {
+      auto context = extract_chain_context(block_dir, retained_dir);
+      if (!context)
+         return {};
+      return std::visit(overloaded{
+         [](const chain_id_type& id){ return id; },
+         [](const genesis_state& gs){ return gs.compute_chain_id(); }
+          } , *context);
    }
 
    // static
@@ -1530,10 +1573,14 @@ namespace eosio { namespace chain {
 
       block_log_bundle log_bundle(block_dir);
 
+      ilog("block log version= ${version}",("version", log_bundle.log_data.version()));
+      ilog("first block= ${first}",("first", log_bundle.log_data.first_block_num()));
+      ilog("last block= ${last}",("last", log_bundle.log_data.last_block_num()));
+
       ilog("blocks.log and blocks.index agree on number of blocks");
 
       if (interval == 0) {
-         interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1);
+         interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1U);
       }
       uint32_t expected_block_num = log_bundle.log_data.first_block_num();
 

libraries/chain/controller.cpp

@@ -210,6 +210,12 @@ struct pending_state {
 };
 
 struct controller_impl {
+   enum class app_window_type {
+      write, // Only main thread is running; read-only threads are not running.
+             // All read-write and read-only tasks are sequentially executed.
+      read   // Main thread and read-only threads are running read-ony tasks in parallel.
+             // Read-write tasks are not being executed.
+   };
 
    // LLVM sets the new handler, we need to reset this to throw a bad_alloc exception so we can possibly exit cleanly
    // and not just abort.
@@ -246,11 +252,10 @@ struct controller_impl {
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
    thread_local static vm::wasm_allocator wasm_alloc; // a copy for main thread and each read-only thread
 #endif
-   // Ideally wasmif should be thread_local which must be a static.
-   // Unittests can create multiple controller objects (testers) at the same time,
-   // which overwrites the same static wasmif, is used for eosvmoc too.
    wasm_interface  wasmif;  // used by main thread and all threads for EOSVMOC
-   thread_local static std::unique_ptr<wasm_interface> wasmif_thread_local; // a copy for each read-only thread, used by eos-vm and eos-vm-jit
+   std::mutex threaded_wasmifs_mtx;
+   std::unordered_map<std::thread::id, std::unique_ptr<wasm_interface>> threaded_wasmifs; // one for each read-only thread, used by eos-vm and eos-vm-jit
+   app_window_type app_window = app_window_type::write;
 
    typedef pair<scope_name,action_name>                   handler_key;
    map< account_name, map<handler_key, apply_handler> >   apply_handlers;
@@ -265,9 +270,7 @@ struct controller_impl {
          prev = fork_db.root();
       }
 
-      if ( read_mode == db_read_mode::HEAD ) {
-         EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
-      }
+      EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
 
       head = prev;
 
@@ -337,7 +340,12 @@ struct controller_impl {
       set_activation_handler<builtin_protocol_feature_t::crypto_primitives>();
 
       self.irreversible_block.connect([this](const block_state_ptr& bsp) {
-         get_wasm_interface().current_lib(bsp->block_num);
+         // producer_plugin has already asserted irreversible_block signal is
+         // called in write window
+         wasmif.current_lib(bsp->block_num);
+         for (auto& w: threaded_wasmifs) {
+            w.second->current_lib(bsp->block_num);
+         }
       });
 
 
@@ -415,7 +423,8 @@ struct controller_impl {
          EOS_ASSERT( root_id == log_head_id, fork_database_exception, "fork database root does not match block log head" );
       } else {
          EOS_ASSERT( fork_db.root()->block_num == lib_num, fork_database_exception,
-                     "empty block log expects the first appended block to build off a block that is not the fork database root. root block number: ${block_num}, lib: ${lib_num}", ("block_num", fork_db.root()->block_num) ("lib_num", lib_num) );
+                     "The first block ${lib_num} when starting with an empty block log should be the block after fork database root ${bn}.",
+                     ("lib_num", lib_num)("bn", fork_db.root()->block_num) );
       }
 
       const auto fork_head = fork_db_head();
@@ -497,9 +506,10 @@ struct controller_impl {
 
    void replay(std::function<bool()> check_shutdown) {
       auto blog_head = blog.head();
-      if( !blog_head && !fork_db.root() ) {
+      if( !fork_db.root() ) {
          fork_db.reset( *head );
-         return;
+         if (!blog_head)
+            return;
       }
 
       replaying = true;
@@ -549,7 +559,10 @@ struct controller_impl {
          ilog( "no irreversible blocks need to be replayed" );
       }
 
-      if( !except_ptr && !check_shutdown() && fork_db.head() ) {
+      if (snapshot_head_block != 0 && !blog_head) {
+         // loading from snapshot without a block log so fork_db can't be considered valid
+         fork_db.reset( *head );
+      } else if( !except_ptr && !check_shutdown() && fork_db.head() ) {
          auto head_block_num = head->block_num;
          auto branch = fork_db.fetch_branch( fork_db.head()->id );
          int rev = 0;
@@ -580,19 +593,21 @@ struct controller_impl {
    void startup(std::function<void()> shutdown, std::function<bool()> check_shutdown, const snapshot_reader_ptr& snapshot) {
       EOS_ASSERT( snapshot, snapshot_exception, "No snapshot reader provided" );
       this->shutdown = shutdown;
-      ilog( "Starting initialization from snapshot, this may take a significant amount of time" );
       try {
          snapshot->validate();
          if( auto blog_head = blog.head() ) {
+            ilog( "Starting initialization from snapshot and block log ${b}-${e}, this may take a significant amount of time",
+                  ("b", blog.first_block_num())("e", blog_head->block_num()) );
             read_from_snapshot( snapshot, blog.first_block_num(), blog_head->block_num() );
          } else {
+            ilog( "Starting initialization from snapshot and no block log, this may take a significant amount of time" );
             read_from_snapshot( snapshot, 0, std::numeric_limits<uint32_t>::max() );
-            const uint32_t lib_num = head->block_num;
-            EOS_ASSERT( lib_num > 0, snapshot_exception,
+            EOS_ASSERT( head->block_num > 0, snapshot_exception,
                         "Snapshot indicates controller head at block number 0, but that is not allowed. "
                         "Snapshot is invalid." );
-            blog.reset( chain_id, lib_num + 1 );
+            blog.reset( chain_id, head->block_num + 1 );
          }
+         ilog( "Snapshot loaded, lib: ${lib}", ("lib", head->block_num) );
 
          init(check_shutdown);
          ilog( "Finished initialization from snapshot" );
@@ -1624,7 +1639,7 @@ struct controller_impl {
             if ( trx->is_transient() ) {
                // remove trx from pending block by not canceling 'restore'
                trx_context.undo(); // this will happen automatically in destructor, but make it more explicit
-            } else if ( pending->_block_status == controller::block_status::ephemeral ) {
+            } else if ( read_mode != db_read_mode::SPECULATIVE && pending->_block_status == controller::block_status::ephemeral ) {
                // An ephemeral block will never become a full block, but on a producer node the trxs should be saved
                // in the un-applied transaction queue for execution during block production. For a non-producer node
                // save the trxs in the un-applied transaction queue for use during block validation to skip signature
@@ -2680,28 +2695,51 @@ struct controller_impl {
    // only called from non-main threads (read-only trx execution threads)
    // when producer_plugin starts them
    void init_thread_local_data() {
-      EOS_ASSERT( !is_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
+      EOS_ASSERT( !is_on_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if ( is_eos_vm_oc_enabled() )
          // EOSVMOC needs further initialization of its thread local data
          wasmif.init_thread_local_data();
       else
 #endif
+      {
+         std::lock_guard g(threaded_wasmifs_mtx);
          // Non-EOSVMOC needs a wasmif per thread
-         wasmif_thread_local = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+         threaded_wasmifs[std::this_thread::get_id()]  = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+      }
    }
 
-   bool is_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+   bool is_on_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+
+   void set_to_write_window() {
+      app_window = app_window_type::write;
+   }
+   void set_to_read_window() {
+      app_window = app_window_type::read;
+   }
+   bool is_write_window() const {
+      return app_window == app_window_type::write;
+   }
 
    wasm_interface& get_wasm_interface() {
-      if ( is_main_thread()
+      if ( is_on_main_thread()
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
           || is_eos_vm_oc_enabled()
 #endif
          )
          return wasmif;
       else
-         return *wasmif_thread_local;
+         return *threaded_wasmifs[std::this_thread::get_id()];
+   }
+
+   void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+      // The caller of this function apply_eosio_setcode has already asserted that
+      // the transaction is not a read-only trx, which implies we are
+      // in write window. Safe to call threaded_wasmifs's code_block_num_last_used
+      wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      for (auto& w: threaded_wasmifs) {
+         w.second->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      }
    }
 
    block_state_ptr fork_db_head() const;
@@ -2711,7 +2749,6 @@ thread_local platform_timer controller_impl::timer;
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
 thread_local eosio::vm::wasm_allocator controller_impl::wasm_alloc;
 #endif
-thread_local std::unique_ptr<wasm_interface> controller_impl::wasmif_thread_local;
 
 const resource_limits_manager&   controller::get_resource_limits_manager()const
 {
@@ -3686,6 +3723,20 @@ void controller::init_thread_local_data() {
    my->init_thread_local_data();
 }
 
+void controller::set_to_write_window() {
+   my->set_to_write_window();
+}
+void controller::set_to_read_window() {
+   my->set_to_read_window();
+}
+bool controller::is_write_window() const {
+   return my->is_write_window();
+}
+
+void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+   return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+}
+
 /// Protocol feature activation handlers:
 
 template<>

libraries/chain/eosio_contract.cpp

@@ -160,7 +160,7 @@ void apply_eosio_setcode(apply_context& context) {
       old_size  = (int64_t)old_code_entry.code.size() * config::setcode_ram_bytes_multiplier;
       if( old_code_entry.code_ref_count == 1 ) {
          db.remove(old_code_entry);
-         context.control.get_wasm_interface().code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
+         context.control.code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
       } else {
          db.modify(old_code_entry, [](code_object& o) {
             --o.code_ref_count;

libraries/chain/include/eosio/chain/block_log.hpp

@@ -82,9 +82,12 @@ namespace eosio { namespace chain {
 
          static fc::path repair_log( const fc::path& data_dir, uint32_t truncate_at_block = 0, const char* reversible_block_dir_name="" );
 
-         static std::optional<genesis_state> extract_genesis_state( const fc::path& data_dir );
+         using chain_context = std::variant<genesis_state, chain_id_type>;
+         static std::optional<chain_context> extract_chain_context( const fc::path& data_dir, const fc::path& retained_dir);
 
-         static chain_id_type extract_chain_id( const fc::path& data_dir );
+         static std::optional<genesis_state> extract_genesis_state( const fc::path& data_dir, const fc::path& retained_dir = fc::path{});
+
+         static std::optional<chain_id_type> extract_chain_id( const fc::path& data_dir, const fc::path& retained_dir = fc::path{});
 
          static void construct_index(const fc::path& block_file_name, const fc::path& index_file_name);
 

libraries/chain/include/eosio/chain/controller.hpp

@@ -49,7 +49,8 @@ namespace eosio { namespace chain {
 
    enum class db_read_mode {
       HEAD,
-      IRREVERSIBLE
+      IRREVERSIBLE,
+      SPECULATIVE
    };
 
    enum class validation_mode {
@@ -373,6 +374,10 @@ namespace eosio { namespace chain {
       void set_db_read_only_mode();
       void unset_db_read_only_mode();
       void init_thread_local_data();
+      void set_to_write_window();
+      void set_to_read_window();
+      bool is_write_window() const;
+      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
 
       private:
          friend class apply_context;

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -361,6 +361,10 @@ namespace eosio { namespace chain {
                                     3080007, "Transaction exceeded the current greylisted account network usage limit" )
       FC_DECLARE_DERIVED_EXCEPTION( greylist_cpu_usage_exceeded, resource_exhausted_exception,
                                     3080008, "Transaction exceeded the current greylisted account CPU usage limit" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_temporary_failure, resource_exhausted_exception,
+                                    3080009, "Read-only transaction eos-vm-oc compile temporary failure" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_permanent_failure, resource_exhausted_exception,
+                                    3080010, "Read-only transaction eos-vm-oc compile permanent failure" )
 
       FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception,
                                     3081001, "Transaction reached the deadline set due to leeway on account CPU limits" )

libraries/chain/include/eosio/chain/log_index.hpp

@@ -32,7 +32,7 @@ class log_index {
    bool is_open() const { return file_.is_open(); }
 
    uint64_t back() { return nth_block_position(num_blocks()-1); }
-   int      num_blocks() const { return num_blocks_; }
+   uint32_t num_blocks() const { return num_blocks_; }
    uint64_t nth_block_position(uint32_t n) {
       file_.seek(n*sizeof(uint64_t));
       uint64_t r;

libraries/chain/include/eosio/chain/thread_utils.hpp

@@ -37,9 +37,13 @@ namespace eosio { namespace chain {
 
       /// Spawn threads, can be re-started after stop().
       /// Assumes start()/stop() called from the same thread or externally protected.
+      /// Blocks until all threads are created and completed their init function, or an exception is thrown
+      ///  during thread startup or an init function. Exceptions thrown during these stages are rethrown from start()
+      ///  but some threads might still have been started. Calling stop() after such a failure is safe.
       /// @param num_threads is number of threads spawned
       /// @param on_except is the function to call if io_context throws an exception, is called from thread pool thread.
-      ///                  if an empty function then logs and rethrows exception on thread which will terminate.
+      ///                  if an empty function then logs and rethrows exception on thread which will terminate. Not called
+      ///                  for exceptions during the init function (such exceptions are rethrown from start())
       /// @param init is an optional function to call at startup to initialize any data.
       /// @throw assert_exception if already started and not stopped.
       void start( size_t num_threads, on_except_t on_except, init_t init = {} ) {
@@ -47,9 +51,25 @@ namespace eosio { namespace chain {
          _ioc_work.emplace( boost::asio::make_work_guard( _ioc ) );
          _ioc.restart();
          _thread_pool.reserve( num_threads );
-         for( size_t i = 0; i < num_threads; ++i ) {
-            _thread_pool.emplace_back( std::thread( &named_thread_pool::run_thread, this, i, on_except, init )  );
+
+         std::promise<void> start_complete;
+         std::atomic<uint32_t> threads_remaining = num_threads;
+         std::exception_ptr pending_exception;
+         std::mutex pending_exception_mutex;
+
+         try {
+            for( size_t i = 0; i < num_threads; ++i ) {
+               _thread_pool.emplace_back( std::thread( &named_thread_pool::run_thread, this, i, on_except, init, std::ref(start_complete),
+                                                       std::ref(threads_remaining), std::ref(pending_exception), std::ref(pending_exception_mutex) ) );
+            }
          }
+         catch( ... ) {
+            /// only an exception from std::thread's ctor should end up here. shut down all threads to ensure no
+            ///  potential access to the promise, atomic, etc above performed after throwing out of start
+            stop();
+            throw;
+         }
+         start_complete.get_future().get();
       }
 
       /// destroy work guard, stop io_context, join thread_pool
@@ -63,16 +83,42 @@ namespace eosio { namespace chain {
       }
 
    private:
-      void run_thread( size_t i, const on_except_t& on_except, const init_t& init ) {
-         std::string tn = boost::core::demangle(typeid(this).name());
-         auto offset = tn.rfind("::");
-         if (offset != std::string::npos)
-            tn.erase(0, offset+2);
-         tn = tn.substr(0, tn.find('>')) + "-" + std::to_string( i );
+      void run_thread( size_t i, const on_except_t& on_except, const init_t& init, std::promise<void>& start_complete,
+                       std::atomic<uint32_t>& threads_remaining, std::exception_ptr& pending_exception, std::mutex& pending_exception_mutex ) {
+
+         std::string tn;
+
+         auto decrement_remaining = [&]() {
+            if( !--threads_remaining ) {
+               if( pending_exception )
+                  start_complete.set_exception( pending_exception );
+               else
+                  start_complete.set_value();
+            }
+         };
+
+         try {
+            try {
+               tn = boost::core::demangle(typeid(this).name());
+               auto offset = tn.rfind("::");
+               if (offset != std::string::npos)
+                  tn.erase(0, offset+2);
+               tn = tn.substr(0, tn.find('>')) + "-" + std::to_string( i );
+               fc::set_os_thread_name( tn );
+               if ( init )
+                  init();
+            } FC_LOG_AND_RETHROW()
+         }
+         catch( ... ) {
+            std::lock_guard<std::mutex> l( pending_exception_mutex );
+            pending_exception = std::current_exception();
+            decrement_remaining();
+            return;
+         }
+
+         decrement_remaining();
+
          try {
-            fc::set_os_thread_name( tn );
-            if ( init )
-               init();
             _ioc.run();
          } catch( const fc::exception& e ) {
             if( on_except ) {

libraries/chain/include/eosio/chain/wasm_interface.hpp

@@ -63,9 +63,6 @@ namespace eosio { namespace chain {
          //Calls apply or error on a given code
          void apply(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, apply_context& context);
 
-         //Immediately exits currently running wasm. UB is called when no wasm running
-         void exit();
-
          //Returns true if the code is cached
          bool is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const;
 

libraries/chain/include/eosio/chain/wasm_interface_private.hpp

@@ -37,7 +37,6 @@ namespace eosio { namespace chain {
    struct wasm_interface_impl {
       struct wasm_cache_entry {
          digest_type                                          code_hash;
-         uint32_t                                             first_block_num_used;
          uint32_t                                             last_block_num_used;
          std::unique_ptr<wasm_instantiated_module_interface>  module;
          uint8_t                                              vm_type = 0;
@@ -109,24 +108,6 @@ namespace eosio { namespace chain {
          return it != wasm_instantiation_cache.end();
       }
 
-      std::vector<uint8_t> parse_initial_memory(const Module& module) {
-         std::vector<uint8_t> mem_image;
-
-         for(const DataSegment& data_segment : module.dataSegments) {
-            EOS_ASSERT(data_segment.baseOffset.type == InitializerExpression::Type::i32_const, wasm_exception, "");
-            EOS_ASSERT(module.memories.defs.size(), wasm_exception, "");
-            const U32 base_offset = data_segment.baseOffset.i32;
-            const Uptr memory_size = (module.memories.defs[0].type.size.min << IR::numBytesPerPageLog2);
-            if(base_offset >= memory_size || base_offset + data_segment.data.size() > memory_size)
-               FC_THROW_EXCEPTION(wasm_execution_error, "WASM data segment outside of valid memory range");
-            if(base_offset + data_segment.data.size() > mem_image.size())
-               mem_image.resize(base_offset + data_segment.data.size(), 0x00);
-            memcpy(mem_image.data() + base_offset, data_segment.data.data(), data_segment.data.size());
-         }
-
-         return mem_image;
-      }
-
       void code_block_num_last_used(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, const uint32_t& block_num) {
          wasm_cache_index::iterator it = wasm_instantiation_cache.find(boost::make_tuple(code_hash, vm_type, vm_version));
          if(it != wasm_instantiation_cache.end())
@@ -157,7 +138,6 @@ namespace eosio { namespace chain {
 
             it = wasm_instantiation_cache.emplace( wasm_interface_impl::wasm_cache_entry{
                                                       .code_hash = code_hash,
-                                                      .first_block_num_used = codeobject->first_block_used,
                                                       .last_block_num_used = UINT32_MAX,
                                                       .module = nullptr,
                                                       .vm_type = vm_type,
@@ -173,24 +153,8 @@ namespace eosio { namespace chain {
                trx_context.resume_billing_timer();
             });
             trx_context.pause_billing_timer();
-            IR::Module module;
-            std::vector<U8> bytes = {
-                (const U8*)codeobject->code.data(),
-                (const U8*)codeobject->code.data() + codeobject->code.size()};
-            try {
-               Serialization::MemoryInputStream stream((const U8*)bytes.data(),
-                                                       bytes.size());
-               WASM::scoped_skip_checks no_check;
-               WASM::serialize(stream, module);
-               module.userSections.clear();
-            } catch (const Serialization::FatalSerializationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            } catch (const IR::ValidationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            }
-
             wasm_instantiation_cache.modify(it, [&](auto& c) {
-               c.module = runtime_interface->instantiate_module((const char*)bytes.data(), bytes.size(), parse_initial_memory(module), code_hash, vm_type, vm_version);
+               c.module = runtime_interface->instantiate_module(codeobject->code.data(), codeobject->code.size(), code_hash, vm_type, vm_version);
             });
          }
          return it->module;
@@ -209,7 +173,6 @@ namespace eosio { namespace chain {
                   member<wasm_cache_entry, uint8_t,     &wasm_cache_entry::vm_version>
                >
             >,
-            ordered_non_unique<tag<by_first_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::first_block_num_used>>,
             ordered_non_unique<tag<by_last_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::last_block_num_used>>
          >
       > wasm_cache_index;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp

@@ -30,10 +30,9 @@ class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~eosvmoc_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
       void init_thread_local_data() override;
 
       friend eosvmoc_instantiated_module;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -37,6 +37,7 @@ using allocator_t = bip::rbtree_best_fit<bip::null_mutex_family, bip::offset_ptr
 
 struct config;
 
+
 class code_cache_base {
    public:
       code_cache_base(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
@@ -46,6 +47,12 @@ class code_cache_base {
 
       void free_code(const digest_type& code_id, const uint8_t& vm_version);
 
+      // get_descriptor_for_code failure reasons
+      enum class get_cd_failure {
+         temporary, // oc compile not done yet, users like read-only trxs can retry
+         permanent  // oc will not start, users should not retry
+      };
+
    protected:
       struct by_hash;
 
@@ -85,9 +92,6 @@ class code_cache_base {
 
       template <typename T>
       void serialize_cache_index(fc::datastream<T>& ds);
-
-      std::thread::id _main_thread_id;
-      bool is_main_thread() const;
 };
 
 class code_cache_async : public code_cache_base {
@@ -98,7 +102,7 @@ class code_cache_async : public code_cache_base {
       //If code is in cache: returns pointer & bumps to front of MRU list
       //If code is not in cache, and not blacklisted, and not currently compiling: return nullptr and kick off compile
       //otherwise: return nullptr
-      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window, get_cd_failure& failure);
 
    private:
       std::thread _monitor_reply_thread;
@@ -115,7 +119,7 @@ class code_cache_sync : public code_cache_base {
       ~code_cache_sync();
 
       //Can still fail and return nullptr if, for example, there is an expected instantiation failure
-      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window);
 };
 
 }}}

libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp

@@ -43,15 +43,11 @@ template<typename Backend>
 class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
-
    private:
       // todo: managing this will get more complicated with sync calls;
-      //       immediately_exit_currently_running_module() should probably
-      //       move from wasm_runtime_interface to wasm_instantiated_module_interface.
       eos_vm_backend_t<Backend>* _bkend = nullptr;  // non owning pointer to allow for immediate exit
 
    template<typename Impl>
@@ -61,10 +57,8 @@ class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
 class eos_vm_profile_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_profile_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
-
-      void immediately_exit_currently_running_module() override;
 };
 
 }}}}// eosio::chain::webassembly::eos_vm_runtime

libraries/chain/include/eosio/chain/webassembly/runtime_interface.hpp

@@ -20,12 +20,9 @@ class wasm_instantiated_module_interface {
 
 class wasm_runtime_interface {
    public:
-      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                                      const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) = 0;
 
-      //immediately exit the currently running wasm_instantiated_module_interface. Yep, this assumes only one can possibly run at a time.
-      virtual void immediately_exit_currently_running_module() = 0;
-
       virtual ~wasm_runtime_interface();
 
       // eosvmoc_runtime needs this

libraries/chain/wasm_interface.cpp

@@ -92,8 +92,9 @@ namespace eosio { namespace chain {
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if(my->eosvmoc) {
          const chain::eosvmoc::code_descriptor* cd = nullptr;
+         chain::eosvmoc::code_cache_base::get_cd_failure failure = chain::eosvmoc::code_cache_base::get_cd_failure::temporary;
          try {
-            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version);
+            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version, context.control.is_write_window(), failure);
          }
          catch(...) {
             //swallow errors here, if EOS VM OC has gone in to the weeds we shouldn't bail: continue to try and run baseline
@@ -107,15 +108,18 @@ namespace eosio { namespace chain {
             my->eosvmoc->exec->execute(*cd, my->eosvmoc->mem, context);
             return;
          }
+         else if (context.trx_context.is_read_only()) {
+            if (failure == chain::eosvmoc::code_cache_base::get_cd_failure::temporary) {
+               EOS_ASSERT(false, ro_trx_vm_oc_compile_temporary_failure, "get_descriptor_for_code failed with temporary failure");
+            } else {
+               EOS_ASSERT(false, ro_trx_vm_oc_compile_permanent_failure, "get_descriptor_for_code failed with permanent failure");
+            }
+         }
       }
 #endif
       my->get_instantiated_module(code_hash, vm_type, vm_version, context.trx_context)->apply(context);
    }
 
-   void wasm_interface::exit() {
-      my->runtime_interface->immediately_exit_currently_running_module();
-   }
-
    bool wasm_interface::is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const {
       return my->is_code_cached(code_hash, vm_type, vm_version);
    }

libraries/chain/webassembly/cf_system.cpp

@@ -45,7 +45,8 @@ namespace eosio { namespace chain { namespace webassembly {
       }
    }
 
+   //be aware that EOS VM OC handles eosio_exit internally and this function will not be called by OC
    void interface::eosio_exit( int32_t code ) const {
-      context.control.get_wasm_interface().exit();
+      throw wasm_exit{};
    }
 }}} // ns eosio::chain::webassembly

libraries/chain/webassembly/runtimes/eos-vm-oc.cpp

@@ -28,7 +28,7 @@ class eosvmoc_instantiated_module : public wasm_instantiated_module_interface {
       bool is_main_thread() { return _main_thread_id == std::this_thread::get_id(); };
 
       void apply(apply_context& context) override {
-         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version);
+         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version, context.control.is_write_window());
          EOS_ASSERT(cd, wasm_execution_error, "EOS VM OC instantiation failed");
 
          if ( is_main_thread() )
@@ -50,14 +50,11 @@ eosvmoc_runtime::eosvmoc_runtime(const boost::filesystem::path data_dir, const e
 eosvmoc_runtime::~eosvmoc_runtime() {
 }
 
-std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                         const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) {
    return std::make_unique<eosvmoc_instantiated_module>(code_hash, vm_type, *this);
 }
 
-//never called. EOS VM OC overrides eosio_exit to its own implementation
-void eosvmoc_runtime::immediately_exit_currently_running_module() {}
-
 void eosvmoc_runtime::init_thread_local_data() {
    exec_thread_local = std::make_unique<eosvmoc::executor>(cc);
 }

libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp

@@ -107,10 +107,11 @@ std::tuple<size_t, size_t> code_cache_async::consume_compile_thread_queue() {
 }
 
 
-const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version) {
+const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window, get_cd_failure& failure) {
    //if there are any outstanding compiles, process the result queue now
-   //do this only on main thread (which is in single threaded write window)
-   if(is_main_thread() && _outstanding_compiles_and_poison.size()) {
+   //When app is in write window, all tasks are running sequentially and read-only threads
+   //are not running. Safe to update cache entries.
+   if(is_write_window && _outstanding_compiles_and_poison.size()) {
       auto [count_processed, bytes_remaining] = consume_compile_thread_queue();
 
       if(count_processed)
@@ -136,37 +137,48 @@ const code_descriptor* const code_cache_async::get_descriptor_for_code(const dig
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_main_thread())
+      if (is_write_window)
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_main_thread()) // on read-only thread
+   if(!is_write_window) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
+   }
 
    const code_tuple ct = code_tuple{code_id, vm_version};
 
-   if(_blacklist.find(ct) != _blacklist.end())
+   if(_blacklist.find(ct) != _blacklist.end()) {
+      failure = get_cd_failure::permanent; // Compile will not start
       return nullptr;
+   }
    if(auto it = _outstanding_compiles_and_poison.find(ct); it != _outstanding_compiles_and_poison.end()) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       it->second = false;
       return nullptr;
    }
-   if(_queued_compiles.find(ct) != _queued_compiles.end())
+   if(_queued_compiles.find(ct) != _queued_compiles.end()) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
+   }
 
    if(_outstanding_compiles_and_poison.size() >= _threads) {
       _queued_compiles.emplace(ct);
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
    }
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
-   if(!codeobject) //should be impossible right?
+   if(!codeobject) { //should be impossible right?
+      failure = get_cd_failure::permanent; // Compile will not start
       return nullptr;
+   }
 
    _outstanding_compiles_and_poison.emplace(ct, false);
    std::vector<wrapped_fd> fds_to_pass;
    fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code));
    write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ ct }, fds_to_pass);
+   failure = get_cd_failure::temporary; // Compile might not be done yet
    return nullptr;
 }
 
@@ -179,15 +191,15 @@ code_cache_sync::~code_cache_sync() {
       elog("unexpected response from EOS VM OC compile monitor during shutdown");
 }
 
-const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version) {
+const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window) {
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_main_thread())
+      if (is_write_window)
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_main_thread())
+   if(!is_write_window)
       return nullptr;
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
@@ -212,8 +224,7 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const
 
 code_cache_base::code_cache_base(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
    _db(db),
-   _cache_file_path(data_dir/"code_cache.bin"),
-   _main_thread_id(std::this_thread::get_id())
+   _cache_file_path(data_dir/"code_cache.bin")
 {
    static_assert(sizeof(allocator_t) <= header_offset, "header offset intersects with allocator");
 
@@ -390,8 +401,4 @@ void code_cache_base::check_eviction_threshold(size_t free_bytes) {
    if(free_bytes < _free_bytes_eviction_threshold)
       run_eviction_round();
 }
-
-bool code_cache_base::is_main_thread() const {
-   return _main_thread_id == std::this_thread::get_id();
-}
 }}}

libraries/chain/webassembly/runtimes/eos-vm.cpp

@@ -234,12 +234,7 @@ template<typename Impl>
 eos_vm_runtime<Impl>::eos_vm_runtime() {}
 
 template<typename Impl>
-void eos_vm_runtime<Impl>::immediately_exit_currently_running_module() {
-   throw wasm_exit{};
-}
-
-template<typename Impl>
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                              const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eos_vm_backend_t<Impl>;
@@ -261,11 +256,7 @@ template class eos_vm_runtime<eosio::vm::jit>;
 
 eos_vm_profile_runtime::eos_vm_profile_runtime() {}
 
-void eos_vm_profile_runtime::immediately_exit_currently_running_module() {
-   throw wasm_exit{};
-}
-
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                                const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eosio::vm::backend<eos_vm_host_functions_t, eosio::vm::jit_profile, webassembly::eos_vm_runtime::apply_options, vm::profile_instr_map>;

libraries/custom_appbase/include/eosio/chain/application.hpp

@@ -1,30 +1,33 @@
 #pragma once
 
 #include <appbase/application_base.hpp>
-#include <appbase/execution_priority_queue.hpp>
+#include <eosio/chain/exec_pri_queue.hpp>
+#include <mutex>
 
 /*
- * Custmomize appbase to support two-queue exector.
+ * Customize appbase to support two-queue executor.
 */
 namespace appbase { 
 
 enum class exec_window {
-   read,  // the window during which operations from read_only_trx_safe queue
-          // can be executed in app thread in parallel with read-only transactions
-          // in read-only transaction excuting threads.
-   write, // the window during which operations from both general and
-          // read_only_trx_safe queues can be executed in app thread,
-          // while read-only transactions are not executed in read-only
-          // transaction excuting threads.
+   read,  // the window during which operations from read_only queue
+          // can be executed in parallel in the read-only thread pool
+          // as well as in the app thread.
+   write, // the window during which operations from both read_write and
+          // parallel queues can be executed in app thread,
+          // while read-only operations are not executed in read-only
+          // thread pool. The read-only thread pool is not active; only
+          // the main app thread is active.
 };
 
 enum class exec_queue {
-   read_only_trx_safe, // the queue storing operations which are safe to execute
-                       // on app thread in parallel with read-only transactions
-                       // in read-only transaction excuting threads.
-   general             // the queue storing operations which can be only executed
-                       // on the app thread while read-only transactions are
-                       // not being executed in read-only threads
+   read_only,          // the queue storing tasks which are safe to execute
+                       // in parallel with other read-only tasks in the read-only
+                       // thread pool as well as on the main app thread.
+                       // Multi-thread safe as long as nothing is executed from the read_write queue.
+   read_write          // the queue storing tasks which can be only executed
+                       // on the app thread while read-only tasks are
+                       // not being executed in read-only threads. Single threaded.
 };
 
 class two_queue_executor {
@@ -32,56 +35,64 @@ public:
 
    template <typename Func>
    auto post( int priority, exec_queue q, Func&& func ) {
-      if ( q == exec_queue::general )
-         return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      if ( q == exec_queue::read_write )
+         return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
       else
-         return boost::asio::post(io_serv_, read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+         return boost::asio::post( io_serv_, read_only_queue_.wrap( priority, --order_, std::forward<Func>( func)));
    }
 
    // Legacy and deprecated. To be removed after cleaning up its uses in base appbase
    template <typename Func>
    auto post( int priority, Func&& func ) {
-      // safer to use general queue for unknown type of operation since operations
-      // from general queue are not executed in parallel with read-only transactions
-      return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      // safer to use read_write queue for unknown type of operation since operations
+      // from read_write queue are not executed in parallel with read-only operations
+      return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
    }
 
    boost::asio::io_service& get_io_service() { return io_serv_; }
 
    bool execute_highest() {
       if ( exec_window_ == exec_window::write ) {
-         if( !general_queue_.empty() && ( read_only_trx_safe_queue_.empty() || *read_only_trx_safe_queue_.top() < *general_queue_.top()) )  {
-            // general_queue_'s top function's priority greater than read_only_trx_safe_queue_'s top function's, or general_queue_ empty
-            general_queue_.execute_highest();
-         } else if( !read_only_trx_safe_queue_.empty() ) {
-            read_only_trx_safe_queue_.execute_highest();
+         // During write window only main thread is accessing anything in two_queue_executor, no locking required
+         if( !read_write_queue_.empty() && (read_only_queue_.empty() || *read_only_queue_.top() < *read_write_queue_.top()) )  {
+            // read_write_queue_'s top function's priority greater than read_only_queue_'s top function's, or read_only_queue_ empty
+            read_write_queue_.execute_highest();
+         } else if( !read_only_queue_.empty() ) {
+            read_only_queue_.execute_highest();
          }
-         return !read_only_trx_safe_queue_.empty() || !general_queue_.empty();
+         return !read_only_queue_.empty() || !read_write_queue_.empty();
       } else {
-         return read_only_trx_safe_queue_.execute_highest();
+         // When in read window, multiple threads including main app thread are accessing two_queue_executor, locking required
+         return read_only_queue_.execute_highest_locked(false);
       }
    }
 
+   bool execute_highest_read_only() {
+      return read_only_queue_.execute_highest_locked(true);
+   }
+
    template <typename Function>
-   boost::asio::executor_binder<Function, appbase::execution_priority_queue::executor>
+   boost::asio::executor_binder<Function, appbase::exec_pri_queue::executor>
    wrap(int priority, exec_queue q, Function&& func ) {
-      if ( q == exec_queue::general )
-         return general_queue_.wrap(priority, --order_, std::forward<Function>(func));
+      if ( q == exec_queue::read_write )
+         return read_write_queue_.wrap(priority, --order_, std::forward<Function>(func));
       else
-         return read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Function>(func));
+         return read_only_queue_.wrap( priority, --order_, std::forward<Function>( func));
    }
      
    void clear() {
-      read_only_trx_safe_queue_.clear();
-      general_queue_.clear();
+      read_only_queue_.clear();
+      read_write_queue_.clear();
    }
 
-   void set_to_read_window() {
+   void set_to_read_window(uint32_t num_threads, std::function<bool()> should_exit) {
       exec_window_ = exec_window::read;
+      read_only_queue_.enable_locking(num_threads, std::move(should_exit));
    }
 
    void set_to_write_window() {
       exec_window_ = exec_window::write;
+      read_only_queue_.disable_locking();
    }
 
    bool is_read_window() const {
@@ -92,15 +103,15 @@ public:
       return exec_window_ == exec_window::write;
    }
 
-   auto& read_only_trx_safe_queue() { return read_only_trx_safe_queue_; }
+   auto& read_only_queue() { return read_only_queue_; }
 
-   auto& general_queue() { return general_queue_; }
+   auto& read_write_queue() { return read_write_queue_; }
 
    // members are ordered taking into account that the last one is destructed first
 private:
    boost::asio::io_service            io_serv_;
-   appbase::execution_priority_queue  read_only_trx_safe_queue_;
-   appbase::execution_priority_queue  general_queue_;
+   appbase::exec_pri_queue            read_only_queue_;
+   appbase::exec_pri_queue            read_write_queue_;
    std::atomic<std::size_t>           order_ { std::numeric_limits<size_t>::max() }; // to maintain FIFO ordering in both queues within priority
    exec_window                        exec_window_ { exec_window::write };
 };

libraries/custom_appbase/include/eosio/chain/exec_pri_queue.hpp

@@ -0,0 +1,233 @@
+#pragma once
+#include <boost/asio.hpp>
+
+#include <condition_variable>
+#include <mutex>
+#include <queue>
+
+namespace appbase {
+// adapted from: https://www.boost.org/doc/libs/1_69_0/doc/html/boost_asio/example/cpp11/invocation/prioritised_handlers.cpp
+
+// Locking has to be coordinated by caller, use with care.
+class exec_pri_queue : public boost::asio::execution_context
+{
+public:
+
+   void enable_locking(uint32_t num_threads, std::function<bool()> should_exit) {
+      assert(num_threads > 0 && num_waiting_ == 0);
+      lock_enabled_ = true;
+      max_waiting_ = num_threads;
+      should_exit_ = std::move(should_exit);
+      exiting_blocking_ = false;
+   }
+
+   void disable_locking() {
+      lock_enabled_ = false;
+      should_exit_ = [](){ assert(false); return true; }; // should not be called when locking is disabled
+   }
+
+   // called from appbase::application_base::exec poll_one() or run_one()
+   template <typename Function>
+   void add(int priority, size_t order, Function function)
+   {
+      std::unique_ptr<queued_handler_base> handler(new queued_handler<Function>(priority, order, std::move(function)));
+      if (lock_enabled_) {
+         std::lock_guard g( mtx_ );
+         handlers_.push( std::move( handler ) );
+         if (num_waiting_)
+            cond_.notify_one();
+      } else {
+         handlers_.push( std::move( handler ) );
+      }
+   }
+
+   // only call when no lock required
+   void clear()
+   {
+      handlers_ = prio_queue();
+   }
+
+   // only call when no lock required
+   bool execute_highest()
+   {
+      if( !handlers_.empty() ) {
+         handlers_.top()->execute();
+         handlers_.pop();
+      }
+
+      return !handlers_.empty();
+   }
+
+private:
+   // has to be defined before use, auto return type
+   auto pop() {
+      auto t = std::move(const_cast<std::unique_ptr<queued_handler_base>&>(handlers_.top()));
+      handlers_.pop();
+      return t;
+   }
+
+public:
+
+   bool execute_highest_locked(bool should_block) {
+      std::unique_lock g(mtx_);
+      if (should_block) {
+         ++num_waiting_;
+         cond_.wait(g, [this](){
+            bool exit = exiting_blocking_ || should_exit_();
+            bool empty = handlers_.empty();
+            if (empty || exit) {
+               if (((empty && num_waiting_ == max_waiting_) || exit) && !exiting_blocking_) {
+                  cond_.notify_all();
+                  exiting_blocking_ = true;
+               }
+               return exit || exiting_blocking_; // same as calling should_exit(), but faster
+            }
+            return true;
+         });
+         --num_waiting_;
+         if (exiting_blocking_ || should_exit_())
+            return false;
+      }
+      if( handlers_.empty() )
+         return false;
+      auto t = pop();
+      g.unlock();
+      t->execute();
+      return true;
+   }
+
+   // Only call when locking disabled
+   size_t size() const { return handlers_.size(); }
+
+   // Only call when locking disabled
+   bool empty() const { return handlers_.empty(); }
+
+   // Only call when locking disabled
+   const auto& top() const { return handlers_.top(); }
+
+   class executor
+   {
+   public:
+      executor(exec_pri_queue& q, int p, size_t o)
+            : context_(q), priority_(p), order_(o)
+      {
+      }
+
+      exec_pri_queue& context() const noexcept
+      {
+         return context_;
+      }
+
+      template <typename Function, typename Allocator>
+      void dispatch(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void post(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void defer(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      void on_work_started() const noexcept {}
+      void on_work_finished() const noexcept {}
+
+      bool operator==(const executor& other) const noexcept
+      {
+         return order_ == other.order_ && &context_ == &other.context_ && priority_ == other.priority_;
+      }
+
+      bool operator!=(const executor& other) const noexcept
+      {
+         return !operator==(other);
+      }
+
+   private:
+      exec_pri_queue& context_;
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   boost::asio::executor_binder<Function, executor>
+   wrap(int priority, size_t order, Function&& func)
+   {
+      return boost::asio::bind_executor( executor(*this, priority, order), std::forward<Function>(func) );
+   }
+
+private:
+   class queued_handler_base
+   {
+   public:
+      queued_handler_base( int p, size_t order )
+            : priority_( p )
+            , order_( order )
+      {
+      }
+
+      virtual ~queued_handler_base() = default;
+
+      virtual void execute() = 0;
+
+      int priority() const { return priority_; }
+      // C++20
+      // friend std::weak_ordering operator<=>(const queued_handler_base&,
+      //                                       const queued_handler_base&) noexcept = default;
+      friend bool operator<(const queued_handler_base& a,
+                            const queued_handler_base& b) noexcept
+      {
+         return std::tie( a.priority_, a.order_ ) < std::tie( b.priority_, b.order_ );
+      }
+
+   private:
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   class queued_handler : public queued_handler_base
+   {
+   public:
+      queued_handler(int p, size_t order, Function f)
+            : queued_handler_base( p, order )
+            , function_( std::move(f) )
+      {
+      }
+
+      void execute() override
+      {
+         function_();
+      }
+
+   private:
+      Function function_;
+   };
+
+   struct deref_less
+   {
+      template<typename Pointer>
+      bool operator()(const Pointer& a, const Pointer& b) noexcept(noexcept(*a < *b))
+      {
+         return *a < *b;
+      }
+   };
+
+   bool lock_enabled_ = false;
+   std::mutex mtx_;
+   std::condition_variable cond_;
+   uint32_t num_waiting_{0};
+   uint32_t max_waiting_{0};
+   bool exiting_blocking_{false};
+   std::function<bool()> should_exit_; // called holding mtx_
+   using prio_queue = std::priority_queue<std::unique_ptr<queued_handler_base>, std::deque<std::unique_ptr<queued_handler_base>>, deref_less>;
+   prio_queue handlers_;
+};
+
+} // appbase

libraries/custom_appbase/tests/custom_appbase_tests.cpp

@@ -27,28 +27,28 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
 
    // Stop app. Use the lowest priority to make sure this function to execute the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_only_trx_safe_queue should only contain the current lambda function,
-      // and general_queue should have execute all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_only_queue should only contain the current lambda function,
+      // and read_write_queue should have executed all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of both queues' functions processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 8 );
@@ -64,40 +64,40 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    BOOST_CHECK_LT( rslts[6], rslts[7] );
 }
 
-// verify functions only from read_only_trx_safe queue are processed during read window
+// verify functions only from read_only queue are processed during read window
 BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 4 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 4 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 6 );
@@ -109,40 +109,40 @@ BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    BOOST_CHECK_LT( rslts[3], rslts[4] );
 }
 
-// verify no functions are executed during  read window if read_only_trx_safe queue is empty
+// verify no functions are executed during  read window if read_only queue is empty
 BOOST_AUTO_TEST_CASE( execute_from_empty_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // Stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 10 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 10 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // no results
    BOOST_REQUIRE_EQUAL( rslts.size(), 0 );
@@ -159,32 +159,32 @@ BOOST_AUTO_TEST_CASE( execute_from_both_queues ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[9]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[10]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[11]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[10]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[11]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
       // read_queue should have current function and write_queue's functions are all executed 
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
 
    app_thread.join();
 
    // queues are emptied after quit
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 12 );

libraries/libfc/CMakeLists.txt

@@ -57,8 +57,6 @@ set( fc_sources
      src/crypto/blake2.cpp
      src/crypto/k1_recover.cpp
      src/network/ip.cpp
-     src/network/resolve.cpp
-     src/network/udp_socket.cpp
      src/network/url.cpp
      src/network/http/http_client.cpp
      src/compress/zlib.cpp

libraries/libfc/include/fc/network/resolve.hpp

@@ -1,9 +0,0 @@
-#pragma once
-#include <fc/vector.hpp>
-#include <fc/network/ip.hpp>
-
-namespace fc
-{
-  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
-                                                      const std::string& host, uint16_t port);
-}

libraries/libfc/include/fc/network/udp_socket.hpp

@@ -1,39 +0,0 @@
-#pragma once
-#include <fc/utility.hpp>
-#include <memory>
-
-#include <boost/asio.hpp>
-
-namespace fc {
-  namespace ip {
-    class endpoint;
-    class address;
-  }
-
-  /**
-   *  The udp_socket class has reference semantics, all copies will
-   *  refer to the same underlying socket.
-   */
-  class udp_socket {
-    public:
-      udp_socket();
-      udp_socket( const udp_socket& s );
-      ~udp_socket();
-
-      void initialize(boost::asio::io_service &);
-      void open();
-      void send_to(const char* b, size_t l, boost::asio::ip::udp::endpoint &to);
-      void send_to(const std::shared_ptr<const char>& b, size_t l, boost::asio::ip::udp::endpoint &to);
-      void close();
-
-      void set_reuse_address(bool);
-
-      void connect(const boost::asio::ip::udp::endpoint& e);
-      const boost::asio::ip::udp::endpoint local_endpoint() const;
-
-    private:
-      class                impl;
-      std::shared_ptr<impl> my;
-  };
-
-}

libraries/libfc/src/log/gelf_appender.cpp

@@ -1,6 +1,3 @@
-#include <fc/network/udp_socket.hpp>
-#include <fc/network/ip.hpp>
-#include <fc/network/resolve.hpp>
 #include <fc/exception/exception.hpp>
 #include <fc/log/gelf_appender.hpp>
 #include <fc/reflect/variant.hpp>
@@ -10,6 +7,7 @@
 #include <fc/compress/zlib.hpp>
 #include <fc/log/logger_config.hpp>
 
+#include <boost/asio.hpp>
 #include <boost/lexical_cast.hpp>
 #include <iomanip>
 #include <iostream>
@@ -19,14 +17,6 @@
 
 namespace fc
 {
-  namespace detail
-  {
-    boost::asio::ip::udp::endpoint to_asio_ep( const fc::ip::endpoint& e )
-    {
-      return boost::asio::ip::udp::endpoint(boost::asio::ip::address_v4(e.get_address()), e.port() );
-    }
-  }
-
   const std::vector<std::string> gelf_appender::config::reserved_field_names  = {
       "_id",            // per GELF specification
       "_timestamp_ns",  // Remaining names all populated by appender
@@ -36,23 +26,21 @@ namespace fc
       "_method_name",
       "_thread_name",
       "_task_name"
-   };
+  };
 
-   const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
+  const std::regex gelf_appender::config::user_field_name_pattern{"^_[\\w\\.\\-]*$"}; // per GELF specification
 
   class gelf_appender::impl
   {
   public:
     using work_guard_t = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
     config                                         cfg;
-    std::optional<boost::asio::ip::udp::endpoint>  gelf_endpoint;
     std::thread                                    thread;
     boost::asio::io_context                        io_context;
     work_guard_t work_guard =                      boost::asio::make_work_guard(io_context);
-    udp_socket                                     gelf_socket;
+    boost::asio::ip::udp::socket                   gelf_socket;
 
-
-    impl(const variant& c)
+    impl(const variant& c) : gelf_socket(io_context)
     {
       mutable_variant_object mvo;
       from_variant(c, mvo);
@@ -84,6 +72,35 @@ namespace fc
           thread.join();
        }
     }
+
+    static std::shared_ptr<std::vector<char>> make_new_bufer(boost::asio::const_buffer buf) {
+      const char* p = static_cast<const char*>(buf.data());
+      return std::make_shared<std::vector<char>>(p, p+buf.size());
+    }
+
+    static std::shared_ptr<std::vector<char>> make_new_bufer(const std::array<boost::asio::const_buffer, 2>& bufs) {
+      auto new_buf = std::make_shared<std::vector<char>>();
+      new_buf->reserve(bufs[0].size() + bufs[1].size());
+      for (int i = 0; i < 2; ++i) {
+          const char* p = static_cast<const char*>(bufs[i].data());
+          new_buf->insert(new_buf->end(), p, p + bufs[i].size());
+      }
+      return new_buf;
+    }
+
+    template <typename Buffers>
+    void send(Buffers&& bufs) {
+      boost::system::error_code ec;
+      gelf_socket.send(std::forward<Buffers>(bufs), 0, ec);
+      if (ec == boost::asio::error::would_block) {
+          auto new_buf = make_new_bufer(std::forward<Buffers>(bufs));
+          gelf_socket.async_send(boost::asio::buffer(*new_buf),
+                                    [new_buf](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/) {
+                                        // Swallow errors.  Currently only used for GELF logging, so depend on local
+                                        // log to catch anything that doesn't make it across the network.
+                                    });
+      }
+    }
   };
 
   gelf_appender::gelf_appender(const variant& args) :
@@ -95,58 +112,40 @@ namespace fc
   {
     try
     {
-      try
-      {
-        // if it's a numeric address:port, this will parse it
-        my->gelf_endpoint = detail::to_asio_ep(ip::endpoint::from_string(my->cfg.endpoint));
+      if (my->cfg.endpoint.empty()) {
+        fprintf(stderr, "The logging destination is not specified\n");
+        return;
       }
-      catch (...)
-      {
-      }
-      if (!my->gelf_endpoint)
-      {
-        // couldn't parse as a numeric ip address, try resolving as a DNS name.
-        // This can yield, so don't do it in the catch block above
-        string::size_type colon_pos = my->cfg.endpoint.find(':');
-        try
-        {
-          FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
-          string port = my->cfg.endpoint.substr(colon_pos + 1);
 
-          string hostname = my->cfg.endpoint.substr( 0, colon_pos );
+      std::string_view endpoint = my->cfg.endpoint;
+      string::size_type colon_pos = endpoint.rfind(':');
+      FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
+      auto port = endpoint.substr(colon_pos + 1);
 
-          boost::asio::ip::udp::resolver resolver{ my->io_context };
-          auto endpoints = resolver.resolve(hostname, port);
+      auto hostname = (endpoint[0] == '[' && colon_pos >= 2)  ? endpoint.substr( 1, colon_pos-2 ) : endpoint.substr( 0, colon_pos );
 
-          if (endpoints.empty())
-              FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
-                                 ("hostname", hostname));
+      boost::asio::ip::udp::resolver resolver{ my->io_context };
+      auto endpoints = resolver.resolve(hostname, port);
 
-          my->gelf_endpoint = *endpoints.begin();
+      if (endpoints.empty())
+          FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
+                              ("hostname", std::string(hostname)));
+
+      my->gelf_socket.connect(*endpoints.begin());
+      std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
+      my->gelf_socket.non_blocking(true);
+
+      my->thread = std::thread([this] {
+        try {
+          fc::set_os_thread_name("gelf");
+          my->io_context.run();
+        } catch (std::exception& ex) {
+          fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
+        } catch (...) {
+          fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
         }
-        catch (const boost::bad_lexical_cast&)
-        {
-          FC_THROW("Bad port: ${port}", ("port", my->cfg.endpoint.substr(colon_pos + 1, my->cfg.endpoint.size())));
-        }
-      }
+      });
 
-      if (my->gelf_endpoint)
-      {
-        my->gelf_socket.initialize(my->io_context);
-        my->gelf_socket.open();
-        std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
-
-        my->thread = std::thread([this] {
-          try {
-            fc::set_os_thread_name("gelf");
-            my->io_context.run();
-          } catch (std::exception& ex) {
-            fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
-          } catch (...) {
-            fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
-          }
-        });
-      }
     }
     catch (...)
     {
@@ -221,57 +220,44 @@ namespace fc
 
     if (gelf_message_as_string.size() <= max_payload_size)
     {
-      // no need to split
-      std::shared_ptr<char> send_buffer(new char[gelf_message_as_string.size()],
-                                        [](char* p){ delete[] p; });
-      memcpy(send_buffer.get(), gelf_message_as_string.c_str(),
-             gelf_message_as_string.size());
-
-      my->gelf_socket.send_to(send_buffer, gelf_message_as_string.size(),
-                              *my->gelf_endpoint);
+      my->send(boost::asio::buffer(gelf_message_as_string));
     }
     else
     {
       // split the message
+      struct gelf_header {
+        uint8_t magic[2] = { 0x1e, 0x0f};
+        uint64_t message_id;
+        uint8_t seq = 0;
+        uint8_t count = 0;
+      } header;
+
       // we need to generate an 8-byte ID for this message.
       // city hash should do
-      uint64_t message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
-      const unsigned header_length = 2 /* magic */ + 8 /* msg id */ + 1 /* seq */ + 1 /* count */;
-      const unsigned body_length = max_payload_size - header_length;
-      unsigned total_number_of_packets = (gelf_message_as_string.size() + body_length - 1) / body_length;
+      header.message_id = city_hash64(gelf_message_as_string.c_str(), gelf_message_as_string.size());
+      const unsigned body_length = max_payload_size - sizeof(header);
+      header.count = (gelf_message_as_string.size() + body_length - 1) / body_length;
       unsigned bytes_sent = 0;
-      unsigned number_of_packets_sent = 0;
+
       while (bytes_sent < gelf_message_as_string.size())
       {
         unsigned bytes_to_send = std::min((unsigned)gelf_message_as_string.size() - bytes_sent,
                                           body_length);
 
-        std::shared_ptr<char> send_buffer(new char[max_payload_size],
-                                          [](char* p){ delete[] p; });
-        char* ptr = send_buffer.get();
-        // magic number for chunked message
-        *(unsigned char*)ptr++ = 0x1e;
-        *(unsigned char*)ptr++ = 0x0f;
-
-        // message id
-        memcpy(ptr, (char*)&message_id, sizeof(message_id));
-        ptr += sizeof(message_id);
-
-        *(unsigned char*)(ptr++) = number_of_packets_sent;
-        *(unsigned char*)(ptr++) = total_number_of_packets;
-        memcpy(ptr, gelf_message_as_string.c_str() + bytes_sent,
-               bytes_to_send);
-        my->gelf_socket.send_to(send_buffer, header_length + bytes_to_send,
-                                *my->gelf_endpoint);
-        ++number_of_packets_sent;
+        std::array<boost::asio::const_buffer,2> bufs = {
+          boost::asio::const_buffer(&header, sizeof(header)),
+          boost::asio::const_buffer(gelf_message_as_string.c_str() + bytes_sent, bytes_to_send)
+        };
+        my->send(bufs);
+        ++header.seq;
         bytes_sent += bytes_to_send;
       }
-      FC_ASSERT(number_of_packets_sent == total_number_of_packets);
+      FC_ASSERT(header.seq == header.count);
     }
   }
 
   void gelf_appender::log(const log_message& message) {
-    if (!my->gelf_endpoint)
+    if (!my->thread.joinable())
       return;
 
     // use now() instead of context.get_timestamp() because log_message construction can include user provided long running calls

libraries/libfc/src/network/resolve.cpp

@@ -1,33 +0,0 @@
-#include <boost/asio.hpp>
-#include <fc/exception/exception.hpp>
-
-namespace fc
-{
-  std::vector<boost::asio::ip::udp::endpoint> resolve(boost::asio::io_service& io_service,
-                                                     const std::string& host, uint16_t port)
-  {
-    using q = boost::asio::ip::udp::resolver::query;
-    using b = boost::asio::ip::resolver_query_base;
-    boost::asio::ip::udp::resolver res(io_service);
-    boost::system::error_code ec;
-    auto ep = res.resolve(q(host, std::to_string(uint64_t(port)),
-                            b::address_configured | b::numeric_service), ec);
-    if(!ec)
-    {
-      std::vector<boost::asio::ip::udp::endpoint> eps;
-      while(ep != boost::asio::ip::udp::resolver::iterator())
-      {
-        if(ep->endpoint().address().is_v4())
-        {
-         eps.push_back(*ep);
-        }
-        // TODO: add support for v6
-        ++ep;
-      }
-      return eps;
-    }
-    FC_THROW_EXCEPTION(unknown_host_exception,
-                       "name resolution failed: ${reason}",
-                       ("reason", ec.message()));
-  }
-}

libraries/libfc/src/network/udp_socket.cpp

@@ -1,111 +0,0 @@
-#include <fc/network/udp_socket.hpp>
-#include <fc/network/ip.hpp>
-
-
-namespace fc
-{
-  class udp_socket::impl
-  {
-    public:
-      impl(){}
-      ~impl(){}
-
-      std::shared_ptr<boost::asio::ip::udp::socket> _sock;
-  };
-
-  udp_socket::udp_socket()
-    : my(new impl())
-  {
-  }
-
-  void udp_socket::initialize(boost::asio::io_service& service)
-  {
-    my->_sock.reset(new boost::asio::ip::udp::socket(service));
-  }
-
-  udp_socket::~udp_socket() 
-  {
-    try 
-    {
-      if(my->_sock)
-        my->_sock->close(); //close boost socket to make any pending reads run their completion handler
-    }
-    catch (...) //avoid destructor throw and likely this is just happening because socket wasn't open.
-    {
-    }
-  }
-
-  void udp_socket::send_to(const char* buffer, size_t length, boost::asio::ip::udp::endpoint& to)
-  {
-    try 
-    {
-      my->_sock->send_to(boost::asio::buffer(buffer, length), to);
-      return;
-    } 
-    catch(const boost::system::system_error& e)
-    {
-      if(e.code() == boost::asio::error::would_block)
-      {
-          auto send_buffer_ptr = std::make_shared<std::vector<char>>(buffer, buffer+length);
-          my->_sock->async_send_to(boost::asio::buffer(send_buffer_ptr.get(), length), to,
-                                   [send_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
-          {
-            // Swallow errors.  Currently only used for GELF logging, so depend on local
-            // log to catch anything that doesn't make it across the network.
-          });
-       }
-       // All other exceptions ignored.
-    }
-  }
-
-  void udp_socket::send_to(const std::shared_ptr<const char>& buffer, size_t length,
-                           boost::asio::ip::udp::endpoint& to)
-  {
-    try
-    {
-      my->_sock->send_to(boost::asio::buffer(buffer.get(), length), to);
-      return;
-    } 
-    catch(const boost::system::system_error& e)
-    {
-      if(e.code() == boost::asio::error::would_block)
-      {
-          auto preserved_buffer_ptr = buffer;
-          my->_sock->async_send_to(boost::asio::buffer(preserved_buffer_ptr.get(), length), to,
-                                   [preserved_buffer_ptr](const boost::system::error_code& /*ec*/, std::size_t /*bytes_transferred*/)
-          {
-            // Swallow errors.  Currently only used for GELF logging, so depend on local
-            // log to catch anything that doesn't make it across the network.
-          });
-       }
-       // All other exceptions ignored.
-    }
-  }
-
-  void udp_socket::open()
-  {
-    my->_sock->open(boost::asio::ip::udp::v4());
-    my->_sock->non_blocking(true);
-  }
-
-  void udp_socket::close()
-  {
-    my->_sock->close();
-  }
-
-  const boost::asio::ip::udp::endpoint udp_socket::local_endpoint() const
-  {
-    return my->_sock->local_endpoint();
-  }
-
-  void udp_socket::connect(const boost::asio::ip::udp::endpoint& e)
-  {
-    my->_sock->connect(e);
-  }
-
-  void udp_socket::set_reuse_address( bool s )
-  {
-    my->_sock->set_option( boost::asio::ip::udp::socket::reuse_address(s) );
-  }
-
-}

libraries/state_history/include/eosio/state_history/log.hpp

@@ -89,7 +89,7 @@ namespace state_history {
       fc::path retained_dir       = "retained";
       fc::path archive_dir        = "archive";
       uint32_t stride             = 1000000;
-      uint32_t max_retained_files = 10;
+      uint32_t max_retained_files = UINT32_MAX;
    };
 } // namespace state_history
 
@@ -129,7 +129,7 @@ struct locked_decompress_stream {
 
 namespace detail {
 
-std::vector<char> zlib_decompress(fc::cfile& file, uint64_t compressed_size) {
+inline std::vector<char> zlib_decompress(fc::cfile& file, uint64_t compressed_size) {
    if (compressed_size) {
       std::vector<char> compressed(compressed_size);
       file.read(compressed.data(), compressed_size);
@@ -138,7 +138,7 @@ std::vector<char> zlib_decompress(fc::cfile& file, uint64_t compressed_size) {
    return {};
 }
 
-std::vector<char> zlib_decompress(fc::datastream<const char*>& strm, uint64_t compressed_size) {
+inline std::vector<char> zlib_decompress(fc::datastream<const char*>& strm, uint64_t compressed_size) {
    if (compressed_size) {
       return state_history::zlib_decompress({strm.pos(), compressed_size});
    }
@@ -283,7 +283,7 @@ private:
 class state_history_log {
  private:
    const char* const       name = "";
-   state_history_log_config config;
+   state_history_log_config _config;
 
    // provide exclusive access to all data of this object since accessed from the main thread and the ship thread
    mutable std::mutex      _mx;
@@ -305,7 +305,7 @@ class state_history_log {
    state_history_log(const char* name, const fc::path& log_dir,
                      state_history_log_config conf = {})
        : name(name)
-       , config(std::move(conf)) {
+       , _config(std::move(conf)) {
 
       log.set_file_path(log_dir/(std::string(name) + ".log"));
       index.set_file_path(log_dir/(std::string(name) + ".index"));
@@ -327,7 +327,7 @@ class state_history_log {
                _begin_block = _end_block = catalog.last_block_num() +1;
             }
          }
-      }, config);
+      }, _config);
 
       //check for conversions to/from pruned log, as long as log contains something
       if(_begin_block != _end_block) {
@@ -335,7 +335,7 @@ class state_history_log {
          log.seek(0);
          read_header(first_header);
 
-         auto prune_config = std::get_if<state_history::prune_config>(&config);
+         auto prune_config = std::get_if<state_history::prune_config>(&_config);
 
          if((is_ship_log_pruned(first_header.magic) == false) && prune_config) {
             //need to convert non-pruned to pruned; first prune any ranges we can (might be none)
@@ -362,7 +362,7 @@ class state_history_log {
       if(_begin_block == _end_block)
          return;
 
-      auto prune_config = std::get_if<state_history::prune_config>(&config);
+      auto prune_config = std::get_if<state_history::prune_config>(&_config);
       if(!prune_config || !prune_config->vacuum_on_close)
          return;
 
@@ -372,6 +372,10 @@ class state_history_log {
          vacuum();
    }
 
+   const state_history_log_config& config() const {
+      return _config;
+   }
+
    //        begin     end
    std::pair<uint32_t, uint32_t> block_range() const {
       std::lock_guard g(_mx);
@@ -500,7 +504,7 @@ class state_history_log {
          }
       }
 
-      auto prune_config = std::get_if<state_history::prune_config>(&config);
+      auto prune_config = std::get_if<state_history::prune_config>(&_config);
       if (block_num < _end_block) {
          // This is typically because of a fork, and we need to truncate the log back to the beginning of the fork.
          static uint32_t start_block_num = block_num;
@@ -550,7 +554,10 @@ class state_history_log {
          fc::raw::pack(log, num_blocks_in_log);
       }
 
-      auto partition_config = std::get_if<state_history::partition_config>(&config);
+      log.flush();
+      index.flush();
+
+      auto partition_config = std::get_if<state_history::partition_config>(&_config);
       if (partition_config && block_num % partition_config->stride == 0) {
          split_log();
       }
@@ -606,7 +613,7 @@ class state_history_log {
    }
 
    void prune(const fc::log_level& loglevel) {
-      auto prune_config = std::get_if<state_history::prune_config>(&config);
+      auto prune_config = std::get_if<state_history::prune_config>(&_config);
 
       if(!prune_config)
          return;

libraries/testing/tester.cpp

@@ -272,11 +272,14 @@ namespace eosio { namespace testing {
       if( !expected_chain_id ) {
          expected_chain_id = controller::extract_chain_id_from_db( cfg.state_dir );
          if( !expected_chain_id ) {
-            if( fc::is_regular_file( cfg.blocks_dir / "blocks.log" ) ) {
-               expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir );
-            } else {
-               expected_chain_id = genesis_state().compute_chain_id();
+            fc::path retained_dir;
+            auto partitioned_config = std::get_if<partitioned_blocklog_config>(&cfg.blog);
+            if (partitioned_config) {
+               retained_dir = partitioned_config->retained_dir;
+               if (retained_dir.is_relative())
+                  retained_dir = cfg.blocks_dir/retained_dir;
             }
+            expected_chain_id = block_log::extract_chain_id( cfg.blocks_dir, retained_dir );
          }
       }
 

plugins/chain_api_plugin/chain.swagger.yaml

@@ -122,10 +122,10 @@ paths:
                   description: Compression used, usually false
                 packed_context_free_data:
                   type: string
-                  description: json to hex
+                  description: JSON to hex
                 packed_trx:
                   type: string
-                  description: Transaction object json to hex
+                  description: Transaction object JSON to hex
       responses:
         "200":
           description: OK
@@ -154,10 +154,10 @@ paths:
                   description: Compression used, usually false
                 packed_context_free_data:
                   type: string
-                  description: json to hex
+                  description: JSON to hex
                 packed_trx:
                   type: string
-                  description: Transaction object json to hex
+                  description: Transaction object JSON to hex
 
       responses:
         "200":
@@ -345,29 +345,18 @@ paths:
               schema:
                 title: "GetProducersResponse"
                 type: object
-                additionalProperties: false
-                minProperties:  3
-                required:
-                  - active
-                  - pending
-                  - proposed
                 properties:
-                  active:
+                  rows:
                     type: array
                     nullable: true
                     items:
-                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
-                  pending:
-                    type: array
-                    nullable: true
-                    items:
-                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
-                  proposed:
-                    type: array
-                    nullable: true
-                    items:
-                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
-
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Producer.yaml"
+                  total_producer_vote_weight: 
+                    type: string
+                    description: The sum of all producer votes.
+                  more:
+                    type: string
+                    description: If not all producers were returned with the first request, more contains the lower bound to use for the next request.
 
   /get_raw_code_and_abi:
     post:
@@ -724,3 +713,218 @@ paths:
                         threshold:
                           type: "integer"
                           description: the sum of weights that must be met or exceeded to satisfy the permission
+  /get_transaction_status:
+    post:
+      description: Attempts to get current blockchain state and, if available, transaction information given the transaction id. For query to work, the transaction finality status feature must be enabled by configuring the chain plugin with the config option '--transaction-finality-status-max-storage-size-gb' in nodeos.
+      operationId: get_transaction_status
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - id
+              properties:
+                id:
+                  type: string
+                  description: The transaction ID of the transaction to retrieve the status for. 
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: "https://docs.eosnetwork.com/openapi/v2.0/TransactionStatus.yaml"
+
+  /send_transaction2:
+    post:
+      description: Attempts to apply a transaction to the blockchain specified in JSON format. It supports returning the full trace of a failed transaction and automatic nodeos-mediated retry if it is enabled on the node. When transaction retry is enabled on an API node, it will monitor incoming API transactions and ensure they are resubmitted additional times into the P2P network until they expire or are included in a block. Warning, full failure traces are now returned by default instead of exceptions. Be careful to not confuse a returned trace as an indication of speculative execution success. Verify 'receipt' and 'except' fields of the returned trace.
+      operationId: send_transaction2
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                return_failure_trace:
+                  type: boolean
+                  description: If true, then embed transaction exceptions into the returned transaction trace.
+                retry_trx:
+                  type: boolean
+                  description: If true, requests to retry transaction until gets in a block of given height, see retry_trx_num_blocks as well, or it is irreversible or expires.
+                retry_trx_num_blocks:
+                  type: integer
+                  description: If retry_trx is true, requests to retry transaction until in a block of given height, or lib if not specified.
+                transaction:
+                  type: object
+                  properties:
+                    signatures:
+                      type: array
+                      description: array of signatures required to authorize transaction.
+                      items:
+                        $ref: "https://docs.eosnetwork.com/openapi/v2.0/Signature.yaml"
+                    compression:
+                      type: boolean
+                      description: Compression used, usually false
+                    packed_context_free_data:
+                      type: string
+                      description: JSON to hex
+                    packed_trx:
+                      type: string
+                      description: Transaction object JSON to hex
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                description: Returns Nothing
+
+  /compute_transaction:
+    post:
+      description: Executes specified transaction and creates a transaction trace, including resource usage, and then reverts all state changes but not contribute to the subjective billing for the account. If the transaction has signatures, they are processed, but any failures are ignored. Transactions which fail always include the transaction failure trace. Warning, users with exposed nodes who have enabled the compute_transaction endpoint should implement some throttling to protect from Denial of Service attacks.
+      operationId: compute_transaction
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                signatures:
+                  type: array
+                  description: array of signatures required to authorize transaction
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Signature.yaml"
+                compression:
+                  type: boolean
+                  description: Compression used, usually false
+                packed_context_free_data:
+                  type: string
+                  description: JSON to hex
+                packed_trx:
+                  type: string
+                  description: Transaction object, JSON to hex
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                description: Returns Nothing
+
+  /get_code_hash:
+    post:
+      description: Retrieves the code hash for a smart contract deployed on the blockchain. Once you have the code hash of a contract, you can compare it with a known or expected value to ensure that the contract code has not been modified or tampered with.
+      operationId: get_code_hash
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                account_name:
+                  description: The name of the account for which you want to retrieve the code hash. It represents the account that owns the smart contract code.
+                  type: string
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  account_name:
+                    description: The name of the account where the smart contract was deployed.
+                    type: string
+                  code_hash:
+                    type: string
+                    description: A string that represents the hash value of the specified account's smart contract code.
+
+  /get_transaction_id:
+    post:
+      description: Retrieves the transaction ID (also known as the transaction hash) of a specified transaction on the blockchain.
+      operationId: get_transaction_id
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              description: The transaction in JSON format for which the ID should be retrieved.
+              $ref: "https://docs.eosnetwork.com/openapi/v2.0/Transaction.yaml"
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: string
+                description: The transaction ID.
+
+  /get_producer_schedule:
+    post:
+      description: Retrieves the current producer schedule from the blockchain, which includes the list of active producers and their respective rotation schedule.
+      operationId: get_producer_schedule
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  active:
+                    description: A JSON object that encapsulates the list of active producers schedule and its version.
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"                              
+                  pending:
+                    description: A JSON object that encapsulates the list of pending producers schedule and its version.
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
+                  proposed:
+                    description: A JSON object that encapsulates the list of proposed producers schedule and its version.
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/ProducerSchedule.yaml"
+
+  /send_read_only_transaction:
+    post:
+      description: Sends a read-only transaction in JSON format to the blockchain. This transaction is not intended for inclusion in the blockchain. When a user sends a transaction, which modifies the blockchain state, the connected node will fail the transaction.
+      operationId: send_read_only_transaction
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                transaction:
+                  type: object
+                  properties:
+                    compression:
+                      type: boolean
+                      description: Compression used, usually false
+                    packed_context_free_data:
+                      type: string
+                      description: JSON to hex
+                    packed_trx:
+                      type: string
+                      description: Transaction object JSON to hex
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                description: Returns Nothing
+
+  /push_block:
+    post:
+      description: Sends a block to the blockchain.
+      operationId: push_block
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "https://docs.eosnetwork.com/openapi/v2.0/Block.yaml"
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                description: Returns Nothing

plugins/chain_api_plugin/chain_api_plugin.cpp

@@ -41,6 +41,55 @@ parse_params<chain_apis::read_only::get_transaction_status_params, http_params_t
    }
 }
 
+// if actions.data & actions.hex_data provided, use the hex_data since only currently support unexploded data
+template<>
+chain_apis::read_only::get_transaction_id_params
+parse_params<chain_apis::read_only::get_transaction_id_params, http_params_types::params_required>(const std::string& body) {
+   if (body.empty()) {
+      EOS_THROW(chain::invalid_http_request, "A Request body is required");
+   }
+
+   try {
+      fc::variant trx_var =  fc::json::from_string( body );
+      if( trx_var.is_object() ) {
+         fc::variant_object& vo = trx_var.get_object();
+         if( vo.contains("actions") && vo["actions"].is_array() ) {
+            fc::mutable_variant_object mvo = vo;
+            fc::variants& action_variants = mvo["actions"].get_array();
+            for( auto& action_v : action_variants ) {
+               if( action_v.is_object() ) {
+                 fc::variant_object& action_vo = action_v.get_object();
+                  if( action_vo.contains( "data" ) && action_vo.contains( "hex_data" ) ) {
+                     fc::mutable_variant_object maction_vo = action_vo;
+                     maction_vo["data"] = maction_vo["hex_data"];
+                     action_vo = maction_vo;
+                     vo = mvo;
+                  } else if( action_vo.contains( "data" ) ) {
+                     if( !action_vo["data"].is_string() ) {
+                        EOS_THROW(chain::invalid_http_request, "Request supports only un-exploded 'data' (hex form)");
+                     }
+                  }
+               }
+               else {
+                  EOS_THROW(chain::invalid_http_request, "Transaction contains invalid or empty action");
+               }
+            } 
+         }
+         else {
+            EOS_THROW(chain::invalid_http_request, "Transaction actions are missing or invalid");
+         }
+      }
+      else {
+         EOS_THROW(chain::invalid_http_request, "Transaction object is missing or invalid");
+      }
+      auto trx = trx_var.as<chain_apis::read_only::get_transaction_id_params>();
+      if( trx.id() == transaction().id() ) {
+         EOS_THROW(chain::invalid_http_request, "Invalid transaction object");
+      }
+      return trx;
+   } EOS_RETHROW_EXCEPTIONS(chain::invalid_http_request, "Invalid transaction");
+}
+
 #define CALL_WITH_400(api_name, api_handle, api_namespace, call_name, http_response_code, params_type) \
 {std::string("/v1/" #api_name "/" #call_name), \
    [api_handle](string&&, string&& body, url_response_callback&& cb) mutable { \
@@ -75,7 +124,7 @@ void chain_api_plugin::plugin_startup() {
    ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() );
 
    _http_plugin.add_api( {
-      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only, appbase::priority::medium_high);
    _http_plugin.add_api({
       CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params),
       CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required),
@@ -96,19 +145,19 @@ void chain_api_plugin::plugin_startup() {
       CHAIN_RO_CALL(get_scheduled_transactions, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_required_keys, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_transaction_id, 200, http_params_types::params_required),
+      // transaction related APIs will be posted to read_write queue after keys are recovered, they are safe to run in parallel until they post to the read_write queue
       CHAIN_RO_CALL_ASYNC(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required),
       CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required),
-      // transaction related APIs will be posted to general queue after keys are recovered
       CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 
    // Not safe to run in parallel with read-only transactions
    _http_plugin.add_api({
       CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required)
-   }, appbase::exec_queue::general, appbase::priority::medium_low );
+   }, appbase::exec_queue::read_write, appbase::priority::medium_low );
 
    if (chain.account_queries_enabled()) {
       _http_plugin.add_async_api({
@@ -124,7 +173,7 @@ void chain_api_plugin::plugin_startup() {
    if (chain.transaction_finality_status_enabled()) {
       _http_plugin.add_api({
          CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required),
-      }, appbase::exec_queue::read_only_trx_safe);
+      }, appbase::exec_queue::read_only);
    }
 
    _http_plugin.add_api({
@@ -160,7 +209,7 @@ void chain_api_plugin::plugin_startup() {
            }
         }
       }
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 }
 
 void chain_api_plugin::plugin_shutdown() {}

plugins/chain_plugin/chain_plugin.cpp

@@ -51,6 +51,8 @@ std::ostream& operator<<(std::ostream& osm, eosio::chain::db_read_mode m) {
       osm << "head";
    } else if ( m == eosio::chain::db_read_mode::IRREVERSIBLE ) {
       osm << "irreversible";
+   } else if ( m == eosio::chain::db_read_mode::SPECULATIVE ) {
+      osm << "speculative";
    }
 
    return osm;
@@ -70,10 +72,12 @@ void validate(boost::any& v,
   // one string, it's an error, and exception will be thrown.
   std::string const& s = validators::get_single_string(values);
 
- if ( s == "head" ) {
+  if ( s == "head" ) {
      v = boost::any(eosio::chain::db_read_mode::HEAD);
   } else if ( s == "irreversible" ) {
      v = boost::any(eosio::chain::db_read_mode::IRREVERSIBLE);
+  } else if ( s == "speculative" ) {
+     v = boost::any(eosio::chain::db_read_mode::SPECULATIVE);
   } else {
      throw validation_error(validation_error::invalid_option_value);
   }
@@ -286,10 +290,12 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("sender-bypass-whiteblacklist", boost::program_options::value<vector<string>>()->composing()->multitoken(),
           "Deferred transactions sent by accounts in this list do not have any of the subjective whitelist/blacklist checks applied to them (may specify multiple times)")
          ("read-mode", boost::program_options::value<eosio::chain::db_read_mode>()->default_value(eosio::chain::db_read_mode::HEAD),
-          "Database read mode (\"head\", \"irreversible\").\n"
+          "Database read mode (\"head\", \"irreversible\", \"speculative\").\n"
           "In \"head\" mode: database contains state changes up to the head block; transactions received by the node are relayed if valid.\n"
           "In \"irreversible\" mode: database contains state changes up to the last irreversible block; "
           "transactions received via the P2P network are not relayed and transactions cannot be pushed via the chain API.\n"
+          "In \"speculative\" mode: (DEPRECATED: head mode recommended) database contains state changes by transactions in the blockchain "
+          "up to the head block as well as some transactions not yet included in the blockchain; transactions received by the node are relayed if valid.\n"
           )
          ( "api-accept-transactions", bpo::value<bool>()->default_value(true), "Allow API transactions to be evaluated and relayed if valid.")
          ("validation-mode", boost::program_options::value<eosio::chain::validation_mode>()->default_value(eosio::chain::validation_mode::FULL),
@@ -638,21 +644,56 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
       if( options.count( "terminate-at-block" ))
          my->chain_config->terminate_at_block = options.at( "terminate-at-block" ).as<uint32_t>();
 
+      // move fork_db to new location
+      upgrade_from_reversible_to_fork_db( my.get() );
+
+      bool has_partitioned_block_log_options = options.count("blocks-retained-dir") ||  options.count("blocks-archive-dir")
+         || options.count("blocks-log-stride") || options.count("max-retained-block-files");
+      bool has_retain_blocks_option = options.count("block-log-retain-blocks");
+
+      EOS_ASSERT(!has_partitioned_block_log_options || !has_retain_blocks_option, plugin_config_exception,
+         "block-log-retain-blocks cannot be specified together with blocks-retained-dir, blocks-archive-dir or blocks-log-stride or max-retained-block-files.");
+
+      fc::path retained_dir;
+      if (has_partitioned_block_log_options) {
+         retained_dir = options.count("blocks-retained-dir") ? options.at("blocks-retained-dir").as<bfs::path>()
+                                                                 : bfs::path("");
+         if (retained_dir.is_relative())
+            retained_dir = fc::path{my->blocks_dir}/retained_dir;
+         my->chain_config->blog = eosio::chain::partitioned_blocklog_config{
+            .retained_dir = retained_dir,
+            .archive_dir  = options.count("blocks-archive-dir") ? options.at("blocks-archive-dir").as<bfs::path>()
+                                                                : bfs::path("archive"),
+            .stride       = options.count("blocks-log-stride") ? options.at("blocks-log-stride").as<uint32_t>()
+                                                               : UINT32_MAX,
+            .max_retained_files = options.count("max-retained-block-files")
+                                        ? options.at("max-retained-block-files").as<uint32_t>()
+                                        : UINT32_MAX,
+         };
+      } else if(has_retain_blocks_option) {
+         uint32_t block_log_retain_blocks = options.at("block-log-retain-blocks").as<uint32_t>();
+         if (block_log_retain_blocks == 0)
+            my->chain_config->blog = eosio::chain::empty_blocklog_config{};
+         else {
+            EOS_ASSERT(cfile::supports_hole_punching(), plugin_config_exception,
+                       "block-log-retain-blocks cannot be greater than 0 because the file system does not support hole "
+                       "punching");
+            my->chain_config->blog = eosio::chain::prune_blocklog_config{ .prune_blocks = block_log_retain_blocks };
+         }
+      }
+
+      
+
       if( options.count( "extract-genesis-json" ) || options.at( "print-genesis-json" ).as<bool>()) {
          std::optional<genesis_state> gs;
-
-         if( fc::exists( my->blocks_dir / "blocks.log" )) {
-            gs = block_log::extract_genesis_state( my->blocks_dir );
-            EOS_ASSERT( gs,
-                        plugin_config_exception,
-                        "Block log at '${path}' does not contain a genesis state, it only has the chain-id.",
-                        ("path", (my->blocks_dir / "blocks.log").generic_string())
-            );
-         } else {
-            wlog( "No blocks.log found at '${p}'. Using default genesis state.",
-                  ("p", (my->blocks_dir / "blocks.log").generic_string()));
-            gs.emplace();
-         }
+         
+         gs = block_log::extract_genesis_state( my->blocks_dir, retained_dir );
+         EOS_ASSERT( gs,
+                     plugin_config_exception,
+                     "Block log at '${path}' does not contain a genesis state, it only has the chain-id.",
+                     ("path", (my->blocks_dir / "blocks.log").generic_string())
+         );
+         
 
          if( options.at( "print-genesis-json" ).as<bool>()) {
             ilog( "Genesis JSON:\n${genesis}", ("genesis", json::to_pretty_string( *gs )));
@@ -677,41 +718,6 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
          EOS_THROW( extract_genesis_state_exception, "extracted genesis state from blocks.log" );
       }
 
-      // move fork_db to new location
-      upgrade_from_reversible_to_fork_db( my.get() );
-
-      bool has_partitioned_block_log_options = options.count("blocks-retained-dir") ||  options.count("blocks-archive-dir")
-         || options.count("blocks-log-stride") || options.count("max-retained-block-files");
-      bool has_retain_blocks_option = options.count("block-log-retain-blocks");
-
-      EOS_ASSERT(!has_partitioned_block_log_options || !has_retain_blocks_option, plugin_config_exception,
-         "block-log-retain-blocks cannot be specified together with blocks-retained-dir, blocks-archive-dir or blocks-log-stride or max-retained-block-files.");
-
-
-      if (has_partitioned_block_log_options) {
-         my->chain_config->blog = eosio::chain::partitioned_blocklog_config{
-            .retained_dir = options.count("blocks-retained-dir") ? options.at("blocks-retained-dir").as<bfs::path>()
-                                                                 : bfs::path(""),
-            .archive_dir  = options.count("blocks-archive-dir") ? options.at("blocks-archive-dir").as<bfs::path>()
-                                                                : bfs::path("archive"),
-            .stride       = options.count("blocks-log-stride") ? options.at("blocks-log-stride").as<uint32_t>()
-                                                               : UINT32_MAX,
-            .max_retained_files = options.count("max-retained-block-files")
-                                        ? options.at("max-retained-block-files").as<uint32_t>()
-                                        : UINT32_MAX,
-         };
-      } else if(has_retain_blocks_option) {
-         uint32_t block_log_retain_blocks = options.at("block-log-retain-blocks").as<uint32_t>();
-         if (block_log_retain_blocks == 0)
-            my->chain_config->blog = eosio::chain::empty_blocklog_config{};
-         else {
-            EOS_ASSERT(cfile::supports_hole_punching(), plugin_config_exception,
-                       "block-log-retain-blocks cannot be greater than 0 because the file system does not support hole "
-                       "punching");
-            my->chain_config->blog = eosio::chain::prune_blocklog_config{ .prune_blocks = block_log_retain_blocks };
-         }
-      }
-
       if( options.at( "delete-all-blocks" ).as<bool>()) {
          ilog( "Deleting state database and blocks" );
          if( options.at( "truncate-at-block" ).as<uint32_t>() > 0 )
@@ -755,41 +761,35 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
                  plugin_config_exception,
                  "Snapshot can only be used to initialize an empty database." );
 
-         if( fc::is_regular_file( my->blocks_dir / "blocks.log" )) {
-            auto block_log_genesis = block_log::extract_genesis_state(my->blocks_dir);
-            if( block_log_genesis ) {
-               const auto& block_log_chain_id = block_log_genesis->compute_chain_id();
-               EOS_ASSERT( *chain_id == block_log_chain_id,
-                           plugin_config_exception,
-                           "snapshot chain ID (${snapshot_chain_id}) does not match the chain ID from the genesis state in the block log (${block_log_chain_id})",
-                           ("snapshot_chain_id",  *chain_id)
-                           ("block_log_chain_id", block_log_chain_id)
-               );
-            } else {
-               const auto& block_log_chain_id = block_log::extract_chain_id(my->blocks_dir);
-               EOS_ASSERT( *chain_id == block_log_chain_id,
+         auto block_log_chain_id = block_log::extract_chain_id(my->blocks_dir, retained_dir);
+
+         if (block_log_chain_id) {
+            EOS_ASSERT( *chain_id == *block_log_chain_id,
                            plugin_config_exception,
                            "snapshot chain ID (${snapshot_chain_id}) does not match the chain ID (${block_log_chain_id}) in the block log",
                            ("snapshot_chain_id",  *chain_id)
-                           ("block_log_chain_id", block_log_chain_id)
+                           ("block_log_chain_id", *block_log_chain_id)
                );
-            }
          }
 
       } else {
 
          chain_id = controller::extract_chain_id_from_db( my->chain_config->state_dir );
 
+         auto chain_context = block_log::extract_chain_context( my->blocks_dir, retained_dir );
          std::optional<genesis_state> block_log_genesis;
-         std::optional<chain_id_type> block_log_chain_id;
+         std::optional<chain_id_type> block_log_chain_id;  
 
-         if( fc::is_regular_file( my->blocks_dir / "blocks.log" ) ) {
-            block_log_genesis = block_log::extract_genesis_state( my->blocks_dir );
-            if( block_log_genesis ) {
-               block_log_chain_id = block_log_genesis->compute_chain_id();
-            } else {
-               block_log_chain_id = block_log::extract_chain_id( my->blocks_dir );
-            }
+         if (chain_context) {
+            std::visit(overloaded {
+               [&](const genesis_state& gs) {
+                  block_log_genesis = gs;
+                  block_log_chain_id = gs.compute_chain_id();
+               },
+               [&](const chain_id_type& id) {
+                  block_log_chain_id = id;
+               } 
+            }, *chain_context);
 
             if( chain_id ) {
                EOS_ASSERT( *block_log_chain_id == *chain_id, block_log_exception,
@@ -2686,6 +2686,11 @@ fc::variant chain_plugin::get_log_trx(const transaction& trx) const {
     }
     return pretty_output;
 }
+
+const controller::config& chain_plugin::chain_config() const {
+   EOS_ASSERT(my->chain_config.has_value(), plugin_exception, "chain_config not initialized");
+   return *my->chain_config;
+}
 } // namespace eosio
 
 FC_REFLECT( eosio::chain_apis::detail::ram_market_exchange_state_t, (ignore1)(ignore2)(ignore3)(core_symbol)(ignore4) )

plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp

@@ -829,7 +829,7 @@ public:
             // the following will convert the input to array of 2 uint128_t in little endian, i.e. 50f0fa8360ec998f4bb65b00c86282f5 fb54b91bfed2fe7fe39a92d999d002c5
             // which is the format used by secondary index
             chain::key256_t k;
-            uint8_t buffer[32];
+            uint8_t buffer[32] = {};
             boost::multiprecision::export_bits(v, buffer, 8, false);
             memcpy(&k[0], buffer + 16, 16);
             memcpy(&k[1], buffer, 16);
@@ -886,6 +886,8 @@ public:
    // return variant of trx for logging, trace is modified to minimize log output
    fc::variant get_log_trx(const transaction& trx) const;
 
+   const controller::config& chain_config() const;
+
 private:
    static void log_guard_exception(const chain::guard_exception& e);
 

plugins/chain_plugin/test/CMakeLists.txt

@@ -6,6 +6,6 @@ add_executable( test_trx_retry_db test_trx_retry_db.cpp )
 target_link_libraries( test_trx_retry_db chain_plugin eosio_testing)
 add_test(NAME test_trx_retry_db COMMAND plugins/chain_plugin/test/test_trx_retry_db WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 
-add_executable( test_trx_finality_status_processing test_trx_finality_status_processing.cpp )
+add_executable( test_trx_finality_status_processing test_trx_finality_status_processing.cpp plugin_config_test.cpp)
 target_link_libraries( test_trx_finality_status_processing chain_plugin eosio_testing)
 add_test(NAME test_trx_finality_status_processing COMMAND plugins/chain_plugin/test/test_trx_finality_status_processing WORKING_DIRECTORY ${CMAKE_BINARY_DIR})

plugins/chain_plugin/test/plugin_config_test.cpp

@@ -0,0 +1,22 @@
+#include <array>
+#include <boost/test/unit_test.hpp>
+#include <eosio/chain/application.hpp>
+#include <eosio/chain_plugin/chain_plugin.hpp>
+#include <stdint.h>
+
+BOOST_AUTO_TEST_CASE(chain_plugin_default_tests) {
+   appbase::scoped_app app;
+   fc::temp_directory  tmp;
+
+   auto tmp_path = tmp.path().string();
+   std::array          args = {
+       "test_chain_plugin", "--blocks-log-stride", "10", "--data-dir", tmp_path.c_str(),
+   };
+
+   BOOST_CHECK(app->initialize<eosio::chain_plugin>(args.size(), const_cast<char**>(args.data())));
+   auto& plugin = app->get_plugin<eosio::chain_plugin>();
+
+   auto* config = std::get_if<eosio::chain::partitioned_blocklog_config>(&plugin.chain_config().blog);
+   BOOST_REQUIRE(config);
+   BOOST_CHECK_EQUAL(config->max_retained_files, UINT32_MAX);
+}

plugins/db_size_api_plugin/db_size_api_plugin.cpp

@@ -28,7 +28,7 @@ using namespace eosio;
 void db_size_api_plugin::plugin_startup() {
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(db_size, this, get,  INVOKE_R_V(this, get), 200),
-   }, appbase::exec_queue::read_only_trx_safe);
+   }, appbase::exec_queue::read_only);
 }
 
 db_size_stats db_size_api_plugin::get() {

plugins/http_plugin/http_plugin.cpp

@@ -28,7 +28,7 @@ namespace eosio {
    using std::regex;
    using boost::asio::ip::tcp;
    using std::shared_ptr;
-   
+
    static http_plugin_defaults current_http_plugin_defaults;
    static bool verbose_http_errors = false;
 
@@ -36,6 +36,10 @@ namespace eosio {
       current_http_plugin_defaults = config;
    }
 
+   std::string http_plugin::get_server_header() {
+      return current_http_plugin_defaults.server_header;
+   }
+
    using http_plugin_impl_ptr = std::shared_ptr<class http_plugin_impl>;
 
    class http_plugin_impl : public std::enable_shared_from_this<http_plugin_impl> {
@@ -47,9 +51,9 @@ namespace eosio {
 
          http_plugin_impl& operator=(const http_plugin_impl&) = delete;
          http_plugin_impl& operator=(http_plugin_impl&&) = delete;
-         
+
          std::optional<tcp::endpoint>  listen_endpoint;
-         
+
          std::optional<asio::local::stream_protocol::endpoint> unix_endpoint;
 
          shared_ptr<beast_http_listener<plain_session, tcp, tcp_socket_t > >  beast_server;
@@ -272,7 +276,7 @@ namespace eosio {
 
          my->plugin_state->server_header = current_http_plugin_defaults.server_header;
 
-         
+
          //watch out for the returns above when adding new code here
       } FC_LOG_AND_RETHROW()
    }
@@ -309,7 +313,7 @@ namespace eosio {
             if(my->unix_endpoint) {
                try {
                   my->create_beast_server(true);
-                  
+
                   my->beast_unix_server->listen(*my->unix_endpoint);
                   my->beast_unix_server->start_accept();
                } catch ( const fc::exception& e ){
@@ -334,8 +338,8 @@ namespace eosio {
                      handle_exception("node", "get_supported_apis", body.empty() ? "{}" : body, cb);
                   }
                }
-            }}, appbase::exec_queue::read_only_trx_safe);
-            
+            }}, appbase::exec_queue::read_only);
+
          } catch (...) {
             fc_elog(logger(), "http_plugin startup fails, shutting down");
             app().quit();
@@ -381,22 +385,30 @@ namespace eosio {
          boost::asio::post( my->plugin_state->thread_pool.get_executor(), f );
    }
 
-   void http_plugin::handle_exception( const char *api_name, const char *call_name, const string& body, const url_response_callback& cb) {
+   void http_plugin::handle_exception( const char* api_name, const char* call_name, const string& body, const url_response_callback& cb) {
       try {
          try {
             throw;
          } catch (chain::unknown_block_exception& e) {
             error_results results{400, "Unknown Block", error_results::error_info(e, verbose_http_errors)};
             cb( 400, fc::time_point::maximum(), fc::variant( results ));
+            fc_dlog( logger(), "Unknown block while processing ${api}.${call}: ${e}",
+                     ("api", api_name)("call", call_name)("e", e.to_detail_string()) );
          } catch (chain::invalid_http_request& e) {
             error_results results{400, "Invalid Request", error_results::error_info(e, verbose_http_errors)};
             cb( 400, fc::time_point::maximum(), fc::variant( results ));
+            fc_dlog( logger(), "Invalid http request while processing ${api}.${call}: ${e}",
+                     ("api", api_name)("call", call_name)("e", e.to_detail_string()) );
          } catch (chain::unsatisfied_authorization& e) {
             error_results results{401, "UnAuthorized", error_results::error_info(e, verbose_http_errors)};
             cb( 401, fc::time_point::maximum(), fc::variant( results ));
+            fc_dlog( logger(), "Auth error while processing ${api}.${call}: ${e}",
+                     ("api", api_name)("call", call_name)("e", e.to_detail_string()) );
          } catch (chain::tx_duplicate& e) {
             error_results results{409, "Conflict", error_results::error_info(e, verbose_http_errors)};
             cb( 409, fc::time_point::maximum(), fc::variant( results ));
+            fc_dlog( logger(), "Duplicate trx while processing ${api}.${call}: ${e}",
+                     ("api", api_name)("call", call_name)("e", e.to_detail_string()) );
          } catch (fc::eof_exception& e) {
             error_results results{422, "Unprocessable Entity", error_results::error_info(e, verbose_http_errors)};
             cb( 422, fc::time_point::maximum(), fc::variant( results ));
@@ -454,9 +466,9 @@ namespace eosio {
    fc::microseconds http_plugin::get_max_response_time()const {
       return my->plugin_state->max_response_time;
    }
-   
+
    size_t http_plugin::get_max_body_size()const {
       return my->plugin_state->max_body_size;
    }
-   
+
 }

plugins/http_plugin/include/eosio/http_plugin/beast_http_listener.hpp

@@ -121,9 +121,13 @@ private:
                fail(ec, "accept", self->plugin_state_->logger, "closing connection");
             } else {
                // Create the session object and run it
+               boost::system::error_code re_ec;
+               auto re = self->socket_.remote_endpoint(re_ec);
+               std::string remote_endpoint = re_ec ? "unknown" : boost::lexical_cast<std::string>(re);
                std::make_shared<session_type>(
                   std::move(self->socket_),
-                  self->plugin_state_)
+                  self->plugin_state_,
+                  std::move(remote_endpoint))
                   ->run_session();
             }
             

plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp

@@ -3,6 +3,9 @@
 #include <eosio/http_plugin/common.hpp>
 #include <fc/io/json.hpp>
 
+#include <boost/iostreams/device/array.hpp>
+#include <boost/iostreams/stream.hpp>
+
 #include <memory>
 #include <string>
 #include <charconv>
@@ -74,6 +77,26 @@ bool allow_host(const http::request<http::string_body>& req, T& session,
 
 // Handle HTTP connection using boost::beast for TCP communication
 // Subclasses of this class (plain_session, ssl_session (now removed), etc.)
+// T can be request or response or anything serializable to boost iostreams
+template<typename T>
+std::string to_log_string(const T& req, size_t max_size = 1024) {
+   assert( max_size > 4 );
+   std::string buffer( max_size, '\0' );
+   {
+      boost::iostreams::array_sink sink( buffer.data(), buffer.size() );
+      boost::iostreams::stream stream( sink );
+      stream << req;
+   }
+   buffer.resize( std::strlen( buffer.data() ) );
+   if( buffer.size() == max_size ) {
+      buffer[max_size - 3] = '.';
+      buffer[max_size - 2] = '.';
+      buffer[max_size - 1] = '.';
+   }
+   std::replace_if( buffer.begin(), buffer.end(), []( unsigned char c ) { return c == '\r' || c == '\n'; }, ' ' );
+   return buffer;
+}
+
 // use the Curiously Recurring Template Pattern so that
 // the same code works with both regular TCP sockets and UNIX sockets
 template<class Derived>
@@ -92,6 +115,7 @@ protected:
    std::optional<http::response<http::string_body>> res_;
 
    std::shared_ptr<http_plugin_state> plugin_state_;
+   std::string remote_endpoint_;
 
    // whether response should be sent back to client when an exception occurs
    bool is_send_exception_response_ = true;
@@ -130,8 +154,12 @@ protected:
       }
 
       try {
-         if(!derived().allow_host(req))
+         if(!derived().allow_host(req)) {
+            error_results results{static_cast<uint16_t>(http::status::bad_request), "Disallowed HTTP HOST header in the request"};
+            send_response( fc::json::to_string( results, fc::time_point::maximum() ),
+                        static_cast<unsigned int>(http::status::bad_request) );
             return;
+         }
 
          if(!plugin_state_->access_control_allow_origin.empty()) {
             res_->set("Access-Control-Allow-Origin", plugin_state_->access_control_allow_origin);
@@ -152,6 +180,9 @@ protected:
             return;
          }
 
+         fc_dlog( plugin_state_->logger, "Request:  ${ep} ${r}",
+                  ("ep", remote_endpoint_)("r", to_log_string(req)) );
+
          std::string resource = std::string(req.target());
          // look for the URL handler to handle this resource
          auto handler_itr = plugin_state_->url_handlers.find(resource);
@@ -245,9 +276,9 @@ public:
    // shared_from_this() requires default constructor
    beast_http_session() = default;
 
-   beast_http_session(
-         std::shared_ptr<http_plugin_state> plugin_state)
-       : plugin_state_(std::move(plugin_state)) {
+   beast_http_session(std::shared_ptr<http_plugin_state> plugin_state, std::string remote_endpoint)
+       : plugin_state_(std::move(plugin_state)),
+         remote_endpoint_(std::move(remote_endpoint)) {
       plugin_state_->requests_in_flight += 1;
       req_parser_.emplace();
       req_parser_->body_limit(plugin_state_->max_body_size);
@@ -468,6 +499,9 @@ public:
       // Determine if we should close the connection after
       bool close = !(plugin_state_->keep_alive) || res_->need_eof();
 
+      fc_dlog( plugin_state_->logger, "Response: ${ep} ${b}",
+               ("ep", remote_endpoint_)("b", to_log_string(*res_)) );
+
       // Write the response
       http::async_write(
          derived().stream(),
@@ -498,8 +532,9 @@ public:
    // Create the session
    plain_session(
          tcp_socket_t socket,
-         std::shared_ptr<http_plugin_state> plugin_state)
-       : beast_http_session<plain_session>(std::move(plugin_state)), socket_(std::move(socket)) {}
+         std::shared_ptr<http_plugin_state> plugin_state,
+         std::string remote_endpoint)
+       : beast_http_session<plain_session>(std::move(plugin_state), std::move(remote_endpoint)), socket_(std::move(socket)) {}
 
    tcp_socket_t& stream() { return socket_; }
    tcp_socket_t& socket() { return socket_; }
@@ -542,8 +577,9 @@ class unix_socket_session
 
 public:
    unix_socket_session(stream_protocol::socket sock,
-                       std::shared_ptr<http_plugin_state> plugin_state)
-       : beast_http_session(std::move(plugin_state)), socket_(std::move(sock)) {}
+                       std::shared_ptr<http_plugin_state> plugin_state,
+                       std::string remote_endpoint)
+       : beast_http_session(std::move(plugin_state), std::move(remote_endpoint)), socket_(std::move(sock)) {}
 
    virtual ~unix_socket_session() = default;
 

plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp

@@ -78,6 +78,7 @@ namespace eosio {
 
         //must be called before initialize
         static void set_defaults(const http_plugin_defaults& config);
+        static std::string get_server_header();
 
         APPBASE_PLUGIN_REQUIRES()
         void set_program_options(options_description&, options_description& cfg) override;
@@ -121,7 +122,7 @@ namespace eosio {
         void register_metrics_listener(chain::plugin_interface::metrics_listener listener);
 
         size_t get_max_body_size()const;
-      
+
    private:
         std::shared_ptr<class http_plugin_impl> my;
    };
@@ -175,7 +176,7 @@ namespace eosio {
    };
 
    /**
-    * @brief Used to trim whitespace from body. 
+    * @brief Used to trim whitespace from body.
     * Returned string_view valid only for lifetime of body
     */
    inline std::string_view make_trimmed_string_view(const std::string& body) {

plugins/http_plugin/tests/unit_tests.cpp

@@ -52,7 +52,7 @@ public:
                   cb(200, fc::time_point::maximum(), fc::variant(ok ? string("yes") : string("no")));
                }
             },
-         }, appbase::exec_queue::general);
+         }, appbase::exec_queue::read_write);
    }
 
 private:

plugins/net_api_plugin/net_api_plugin.cpp

@@ -53,11 +53,7 @@ void net_api_plugin::plugin_startup() {
    ilog("starting net_api_plugin");
    // lifetime of plugin is lifetime of application
    auto& net_mgr = app().get_plugin<net_plugin>();
-   app().get_plugin<http_plugin>().add_api({
-    //   CALL(net, net_mgr, set_timeout,
-    //        INVOKE_V_R(net_mgr, set_timeout, int64_t), 200),
-    //   CALL(net, net_mgr, sign_transaction,
-    //        INVOKE_R_R_R_R(net_mgr, sign_transaction, chain::signed_transaction, flat_set<public_key_type>, chain::chain_id_type), 201),
+   app().get_plugin<http_plugin>().add_async_api({
        CALL_WITH_400(net, net_mgr, connect,
             INVOKE_R_R(net_mgr, connect, std::string), 201),
        CALL_WITH_400(net, net_mgr, disconnect,
@@ -66,9 +62,7 @@ void net_api_plugin::plugin_startup() {
             INVOKE_R_R(net_mgr, status, std::string), 201),
        CALL_WITH_400(net, net_mgr, connections,
             INVOKE_R_V(net_mgr, connections), 201),
-    //   CALL(net, net_mgr, open,
-    //        INVOKE_V_R(net_mgr, open, std::string), 200),
-   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+   } );
 }
 
 void net_api_plugin::plugin_initialize(const variables_map& options) {

plugins/net_plugin/net_plugin.cpp

@@ -743,7 +743,7 @@ namespace eosio {
        * day routine and converts to a (at least) 64 bit integer.
        */
       static tstamp get_time() {
-         return std::chrono::system_clock::now().time_since_epoch().count();
+         return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
       }
       /** @} */
 
@@ -1590,7 +1590,7 @@ namespace eosio {
          } );
          sync_known_lib_num = highest_lib_num;
 
-         // if closing the connection we are currently syncing from, then reset our last requested and next expected.
+         // if closing the connection we are currently syncing from then request from a diff peer
          if( c == sync_source ) {
             reset_last_requested_num(g);
             // if starting to sync need to always start from lib as we might be on our own fork
@@ -1777,7 +1777,9 @@ namespace eosio {
       auto current_time_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
       int64_t network_latency_ns = current_time_ns - msg.time; // net latency in nanoseconds
       if( network_latency_ns < 0 ) {
-         peer_wlog(c, "Peer sent a handshake with a timestamp skewed by at least ${t}ms", ("t", network_latency_ns/1000000));
+         if (network_latency_ns < -(block_interval_ns/2)) {
+            peer_wlog(c, "Peer sent a handshake with a timestamp skewed by at least ${t}ms", ("t", network_latency_ns / 1000000));
+         }
          network_latency_ns = 0;
       }
       // number of blocks syncing node is behind from a peer node, round up
@@ -1935,7 +1937,12 @@ namespace eosio {
                verify_catchup( c, msg.known_blocks.pending, id );
             } else {
                // we already have the block, so update peer with our view of the world
-               c->send_handshake();
+               auto chain_info = my_impl->get_chain_info();
+               std::unique_lock g_conn( c->conn_mtx );
+               if (chain_info.head_id != c->last_handshake_sent.head_id) { // no need to send handshake if nothing new to report
+                  g_conn.unlock();
+                  c->send_handshake();
+               }
             }
          }
       } else if (msg.known_blocks.mode == last_irr_catch_up) {
@@ -1953,7 +1960,10 @@ namespace eosio {
    void sync_manager::rejected_block( const connection_ptr& c, uint32_t blk_num ) {
       c->block_status_monitor_.rejected();
       std::unique_lock<std::mutex> g( sync_mtx );
-      reset_last_requested_num(g);
+      sync_last_requested_num = 0;
+      if (blk_num < sync_next_expected_num) {
+         sync_next_expected_num = my_impl->get_chain_lib_num();
+      }
       if( c->block_status_monitor_.max_events_violated()) {
          peer_wlog( c, "block ${bn} not accepted, closing connection", ("bn", blk_num) );
          sync_source.reset();
@@ -1961,6 +1971,7 @@ namespace eosio {
          c->close();
       } else {
          g.unlock();
+         peer_dlog(c, "rejected block ${bn}, sending handshake", ("bn", blk_num));
          c->send_handshake();
       }
    }
@@ -3056,15 +3067,13 @@ namespace eosio {
       switch (msg.known_trx.mode) {
       case none:
          break;
-      case last_irr_catch_up: {
+      case last_irr_catch_up:
+      case catch_up : {
          std::unique_lock<std::mutex> g_conn( conn_mtx );
          last_handshake_recv.head_num = msg.known_blocks.pending;
          g_conn.unlock();
          break;
       }
-      case catch_up : {
-         break;
-      }
       case normal: {
          my_impl->dispatcher->recv_notice( shared_from_this(), msg, false );
       }
@@ -3225,22 +3234,23 @@ namespace eosio {
             return;
          }
 
-         bool valid_block_header = !!bsp;
 
-         if( valid_block_header ) {
+         uint32_t block_num = bsp ? bsp->block_num : 0;
+
+         if( block_num != 0 ) {
             fc_dlog( logger, "validated block header, broadcasting immediately, connection ${cid}, blk num = ${num}, id = ${id}",
-                     ("cid", cid)("num", bsp->block_num)("id", bsp->id) );
+                     ("cid", cid)("num", block_num)("id", bsp->id) );
             my_impl->dispatcher->add_peer_block( bsp->id, cid ); // no need to send back to sender
             my_impl->dispatcher->bcast_block( bsp->block, bsp->id );
          }
 
-         app().executor().post(priority::medium, exec_queue::general, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
+         app().executor().post(priority::medium, exec_queue::read_write, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c{std::move(c)}]() mutable {
             c->process_signed_block( id, std::move(ptr), std::move(bsp) );
          });
 
-         if( valid_block_header ) {
+         if( block_num != 0 ) {
             // ready to process immediately, so signal producer to interrupt start_block
-            my_impl->producer_plug->received_block();
+            my_impl->producer_plug->received_block(block_num);
          }
       });
    }
@@ -3252,18 +3262,6 @@ namespace eosio {
       // use c in this method instead of this to highlight that all methods called on c-> must be thread safe
       connection_ptr c = shared_from_this();
 
-      // if we have closed connection then stop processing
-      if( !c->socket_is_open() ) {
-         if( bsp ) {
-            // valid bsp means add_peer_block already called, need to remove it since we are not going to process the block
-            // call on dispatch strand to serialize with the add_peer_block calls
-            my_impl->dispatcher->strand.post( [blk_id]() {
-               my_impl->dispatcher->rm_block( blk_id );
-            } );
-         }
-         return;
-      }
-
       try {
          if( cc.fetch_block_by_id(blk_id) ) {
             c->strand.post( [sync_master = my_impl->sync_master.get(),

plugins/producer_api_plugin/producer.swagger.yaml

@@ -342,11 +342,11 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/component/schema/Error"
+                $ref: "#/components/schemas/Error"
   /producer/schedule_snapshot:
     post:
       summary: schedule_snapshot
-      description: Submits a request to generate a schedule for automated snapshot with given parameters. If request body is empty, triest to execute snapshot immediately. Returns error when unable to create snapshot.
+      description: Submits a request to automatically generate snapshots according to a schedule specified with given parameters. If request body is empty, schedules immediate snapshot generation. Returns error when unable to accept schedule.
       operationId: schedule_snapshot
       requestBody:
         content:
@@ -377,6 +377,12 @@ paths:
             application/json:
               schema:
                 type: object
+                required:
+                    - snapshot_request_id
+                    - block_spacing
+                    - start_block_num
+                    - end_block_num
+                    - snapshot_description
                 properties:
                   snapshot_request_id:
                     type: integer
@@ -401,12 +407,12 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/component/schema/Error"
+                $ref: "#/components/schemas/Error"
   /producer/get_snapshot_requests:
     post:
       summary: get_snapshot_requests
-      description: Returns a list of scheduled snapshots
-      operationId: get_snapshot_status
+      description: Returns a list of scheduled snapshots.
+      operationId: get_snapshot_requests
       responses:
         "201":
           description: OK
@@ -417,9 +423,16 @@ paths:
                 properties:
                   snapshot_requests:
                     type: array
-                    description: An array of scheduled snapshots
+                    description: An array of scheduled snapshot requests
                     items:
                       type: object
+                      required:
+                          - snapshot_request_id
+                          - block_spacing
+                          - start_block_num
+                          - end_block_num
+                          - snapshot_description
+                          - pending_snapshots
                       properties:
                         snapshot_request_id:
                           type: integer
@@ -444,6 +457,12 @@ paths:
                           description: List of pending snapshots
                           items:
                             type: object
+                            required:
+                                - head_block_id
+                                - head_block_num
+                                - head_block_time
+                                - version
+                                - snapshot_name
                             properties:
                               head_block_id:
                                 $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
@@ -468,11 +487,11 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/component/schema/Error"
+                $ref: "#/components/schemas/Error"
   /producer/unschedule_snapshot:
     post:
       summary: unschedule_snapshot
-      description: Submits a request to remove identified by id recurring snapshot from schedule. Returns error when unable to create unschedule.
+      description: Removes snapshot request identified by id. Returns error if referenced snapshot request does not exist.
       operationId: unschedule_snapshot
       requestBody:
         required: true
@@ -491,6 +510,12 @@ paths:
             application/json:
               schema:
                 type: object
+                required:
+                  - snapshot_request_id
+                  - block_spacing
+                  - start_block_num
+                  - end_block_num
+                  - snapshot_description
                 properties:
                   snapshot_request_id:
                     type: integer

plugins/producer_api_plugin/producer_api_plugin.cpp

@@ -118,9 +118,11 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_R(producer, get_account_ram_corrections, producer_plugin::get_account_ram_corrections_params), 201),
        CALL_WITH_400(producer, producer, get_unapplied_transactions,
                      INVOKE_R_R_D(producer, get_unapplied_transactions, producer_plugin::get_unapplied_transactions_params), 200),
-   }, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+                     INVOKE_R_V(producer, get_snapshot_requests), 201),
+   }, appbase::exec_queue::read_only, appbase::priority::medium_high);
 
-   // Not safe to run in parallel with read-only transactions
+   // Not safe to run in parallel
    app().get_plugin<http_plugin>().add_api({
        CALL_WITH_400(producer, producer, pause,
             INVOKE_V_V(producer, pause), 201),
@@ -137,16 +139,14 @@ void producer_api_plugin::plugin_startup() {
        CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information,
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
        CALL_WITH_400(producer, producer, schedule_snapshot,
-            INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
-       CALL_WITH_400(producer, producer, get_snapshot_requests,
-            INVOKE_R_V(producer, get_snapshot_requests), 201),
+            INVOKE_R_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,
-            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
+            INVOKE_R_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
        CALL_WITH_400(producer, producer, get_integrity_hash,
             INVOKE_R_V(producer, get_integrity_hash), 201),
        CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
             INVOKE_V_R(producer, schedule_protocol_feature_activations, producer_plugin::scheduled_protocol_feature_activations), 201),
-   }, appbase::exec_queue::general, appbase::priority::medium_high);
+   }, appbase::exec_queue::read_write, appbase::priority::medium_high);
 }
 
 void producer_api_plugin::plugin_initialize(const variables_map& options) {

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -94,8 +94,11 @@ public:
       uint32_t snapshot_request_id = 0;
    };
 
+   struct snapshot_schedule_result : public snapshot_request_id_information, public snapshot_request_information {
+   };
+
    struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information {
-      std::optional<std::vector<snapshot_information>> pending_snapshots;
+      std::vector<snapshot_information> pending_snapshots;
    };
 
    struct get_snapshot_requests_result {
@@ -159,8 +162,8 @@ public:
    integrity_hash_information get_integrity_hash() const;
 
    void create_snapshot(next_function<snapshot_information> next);
-   void schedule_snapshot(const snapshot_request_information& schedule);
-   void unschedule_snapshot(const snapshot_request_id_information& schedule);
+   snapshot_schedule_result schedule_snapshot(const snapshot_request_information& schedule);
+   snapshot_schedule_result unschedule_snapshot(const snapshot_request_id_information& schedule);
    get_snapshot_requests_result get_snapshot_requests() const;
 
    scheduled_protocol_feature_activations get_scheduled_protocol_feature_activations() const;
@@ -202,11 +205,14 @@ public:
    void register_metrics_listener(metrics_listener listener);
 
    // thread-safe, called when a new block is received
-   void received_block();
+   void received_block(uint32_t block_num);
 
-const std::set<account_name>& producer_accounts() const;
+   const std::set<account_name>& producer_accounts() const;
 
+   static void set_test_mode(bool m) { test_mode_ = m; }
  private:
+   inline static bool test_mode_{false}; // to be moved into appbase (application_base)
+
    std::shared_ptr<class producer_plugin_impl> my;
 };
 
@@ -221,6 +227,7 @@ FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (block_spacing)
 FC_REFLECT(eosio::producer_plugin::snapshot_request_id_information, (snapshot_request_id))
 FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (snapshot_requests))
 FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_information, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information), (pending_snapshots))
+FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_result, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information),)
 FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate))
 FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable))
 FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse))

plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp

@@ -45,7 +45,7 @@ private:
    void x_serialize() {
       auto& vec = _snapshot_requests.get<as_vector>();
       std::vector<producer_plugin::snapshot_schedule_information> sr(vec.begin(), vec.end());
-      _snapshot_db << sr;   
+      _snapshot_db << sr;
    }
 
 public:
@@ -54,7 +54,7 @@ public:
    // snapshot_scheduler_listener
    void on_start_block(uint32_t height) {
       bool serialize_needed  = false;
-      bool snapshot_executed = false; 
+      bool snapshot_executed = false;
 
       auto execute_snapshot_with_log = [this, height, &snapshot_executed](const auto & req) {
          // one snapshot per height
@@ -64,18 +64,18 @@ public:
                ("end_block_num",   req.end_block_num)
                ("block_spacing",   req.block_spacing)
                ("height",          height));
-               
+
             execute_snapshot(req.snapshot_request_id);
             snapshot_executed = true;
          }
-      };    
+      };
 
       std::vector<uint32_t> unschedule_snapshot_request_ids;
       for(const auto& req: _snapshot_requests.get<0>()) {
          // -1 since its called from start block
          bool recurring_snapshot =  req.block_spacing &&  (height >= req.start_block_num + 1) && (!((height - req.start_block_num - 1) % req.block_spacing));
          bool onetime_snapshot   = (!req.block_spacing) && (height == req.start_block_num + 1);
-        
+
          // assume "asap" for snapshot with missed/zero start, it can have spacing
          if(!req.start_block_num) {
             // update start_block_num with current height only if this is recurring
@@ -84,16 +84,16 @@ public:
                auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
                auto it = snapshot_by_id.find(req.snapshot_request_id);
                _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height - 1; });
-               serialize_needed = true;              
+               serialize_needed = true;
             }
             execute_snapshot_with_log(req);
          } else if(recurring_snapshot || onetime_snapshot) {
             execute_snapshot_with_log(req);
-         }       
+         }
 
          // cleanup - remove expired (or invalid) request
-         if((!req.start_block_num && !req.block_spacing) ||         
-            (!req.block_spacing && height >= (req.start_block_num + 1)) || 
+         if((!req.start_block_num && !req.block_spacing) ||
+            (!req.block_spacing && height >= (req.start_block_num + 1)) ||
             (req.end_block_num > 0 && height >= (req.end_block_num + 1))) {
             unschedule_snapshot_request_ids.push_back(req.snapshot_request_id);
          }
@@ -108,7 +108,7 @@ public:
    }
 
    // snapshot_scheduler_handler
-   void schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
+   producer_plugin::snapshot_schedule_result schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
       auto& snapshot_by_value = _snapshot_requests.get<by_snapshot_value>();
       auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
       EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request");
@@ -122,19 +122,27 @@ public:
          }
       }
 
-      _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++},{sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
+      _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
       x_serialize();
+
+      // returning snapshot_schedule_result
+      return  producer_plugin::snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}};
    }
 
-   virtual void unschedule_snapshot(uint32_t sri) {
+   producer_plugin::snapshot_schedule_result unschedule_snapshot(uint32_t sri) {
       auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
       auto existing = snapshot_by_id.find(sri);
       EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found");
+
+      producer_plugin::snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}};
       _snapshot_requests.erase(existing);
       x_serialize();
+
+      // returning snapshot_schedule_result
+      return result;
    }
 
-   virtual producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
+   producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
       producer_plugin::get_snapshot_requests_result result;
       auto& asvector = _snapshot_requests.get<as_vector>();
       result.snapshot_requests.reserve(asvector.size());
@@ -160,11 +168,8 @@ public:
       auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
       auto  snapshot_req   = snapshot_by_id.find(_inflight_sid);
       if (snapshot_req != snapshot_by_id.end()) {
-         _snapshot_requests.modify(snapshot_req, [&si](auto& p) { 
-            if (!p.pending_snapshots) {
-               p.pending_snapshots = std::vector<producer_plugin::snapshot_information>();
-            }
-            p.pending_snapshots->emplace_back(si);       
+         _snapshot_requests.modify(snapshot_req, [&si](auto& p) {
+            p.pending_snapshots.emplace_back(si);
          });
       }
    }
@@ -193,14 +198,12 @@ public:
             auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
             auto  snapshot_req   = snapshot_by_id.find(srid);
 
-            if (snapshot_req != snapshot_by_id.end()) {               
-               if (auto pending = snapshot_req->pending_snapshots; pending) {                 
-                  auto it = std::remove_if(pending->begin(), pending->end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <=  snapshot_info.head_block_num; });
-                  pending->erase(it, pending->end());
-                  _snapshot_requests.modify(snapshot_req, [&pending](auto& p) { 
-                     p.pending_snapshots = std::move(pending);
-                  });
-               }
+            if (snapshot_req != snapshot_by_id.end()) {
+               _snapshot_requests.modify(snapshot_req, [&](auto& p) {
+                  auto & pending = p.pending_snapshots;
+                  auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <=  snapshot_info.head_block_num; });
+                  pending.erase(it, pending.end());
+               });
             }
          }
       };

plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp

@@ -157,7 +157,10 @@ public:
       const auto time_ordinal = time_ordinal_for(now);
       const auto orig_count = _account_subjective_bill_cache.size();
       remove_subjective_billing( bsp, time_ordinal );
-      fc_dlog( log, "Subjective billed accounts ${n} removed ${r}", ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
+      if (orig_count > 0) {
+         fc_dlog( log, "Subjective billed accounts ${n} removed ${r}",
+                  ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
+      }
    }
 
    template <typename Yield>

plugins/producer_plugin/test/test_read_only_trx.cpp

@@ -50,7 +50,9 @@ auto make_unique_trx( const chain_id_type& chain_id ) {
 
 BOOST_AUTO_TEST_SUITE(read_only_trxs)
 
-void error_handling_common(std::vector<const char*>& specific_args) {
+enum class app_init_status { failed, succeeded };
+
+void test_configs_common(std::vector<const char*>& specific_args, app_init_status expected_status) {
    appbase::scoped_app app;
    fc::temp_directory temp;
    auto temp_dir_str = temp.path().string();
@@ -59,19 +61,30 @@ void error_handling_common(std::vector<const char*>& specific_args) {
    std::vector<const char*> argv =
       {"test", "--data-dir", temp_dir_str.c_str(), "--config-dir", temp_dir_str.c_str()};
    argv.insert( argv.end(), specific_args.begin(), specific_args.end() );
-   BOOST_CHECK_EQUAL( app->initialize<producer_plugin>( argv.size(), (char**) &argv[0]), false );
+
+   // app->initialize() returns a boolean. BOOST_CHECK_EQUAL cannot compare
+   // a boolean with a app_init_status directly
+   bool rc = (expected_status == app_init_status::succeeded) ? true : false;
+   BOOST_CHECK_EQUAL( app->initialize<producer_plugin>( argv.size(), (char**) &argv[0]), rc );
 }
 
 // --read-only-thread not allowed on producer node
 BOOST_AUTO_TEST_CASE(read_only_on_producer) {
    std::vector<const char*> specific_args = {"-p", "eosio", "-e", "--read-only-threads", "2" };
-   error_handling_common(specific_args);
+   test_configs_common(specific_args, app_init_status::failed);
 }
 
 // read_window_time must be greater than max_transaction_time + 10ms
 BOOST_AUTO_TEST_CASE(invalid_read_window_time) {
    std::vector<const char*> specific_args = { "--read-only-threads", "2", "--max-transaction-time", "10", "--read-only-write-window-time-us", "50000", "--read-only-read-window-time-us", "20000" }; // 20000 not greater than --max-transaction-time (10ms) + 10000us (minimum margin)
-   error_handling_common(specific_args);
+   test_configs_common(specific_args, app_init_status::failed);
+}
+
+// if --read-only-threads is not configured, read-only trx related configs should
+// not be checked
+BOOST_AUTO_TEST_CASE(not_check_configs_if_no_read_only_threads) {
+   std::vector<const char*> specific_args = { "--max-transaction-time", "10", "--read-only-write-window-time-us", "50000", "--read-only-read-window-time-us", "20000" }; // 20000 not greater than --max-transaction-time (10ms) + 10000us (minimum margin)
+   test_configs_common(specific_args, app_init_status::succeeded);
 }
 
 void test_trxs_common(std::vector<const char*>& specific_args) {
@@ -79,6 +92,7 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    appbase::scoped_app app;
    fc::temp_directory temp;
    auto temp_dir_str = temp.path().string();
+   producer_plugin::set_test_mode(true);
    
    std::promise<std::tuple<producer_plugin*, chain_plugin*>> plugin_promise;
    std::future<std::tuple<producer_plugin*, chain_plugin*>> plugin_fut = plugin_promise.get_future();
@@ -96,14 +110,19 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    auto chain_id = chain_plug->get_chain_id();
 
    std::atomic<size_t> next_calls = 0;
+   std::atomic<size_t> num_get_account_calls = 0;
    std::atomic<size_t> num_posts = 0;
    std::atomic<size_t> trace_with_except = 0;
    std::atomic<bool> trx_match = true;
-   const size_t num_pushes = 2558;
+   const size_t num_pushes = 4242;
 
    for( size_t i = 1; i <= num_pushes; ++i ) {
       auto ptrx = make_unique_trx( chain_id );
-      app->executor().post( priority::low, exec_queue::general, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
+      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug, &num_get_account_calls]() {
+         chain_plug->get_read_only_api(fc::seconds(90)).get_account(chain_apis::read_only::get_account_params{.account_name=config::system_account_name}, fc::time_point::now()+fc::seconds(90));
+         ++num_get_account_calls;
+      });
+      app->executor().post( priority::low, exec_queue::read_write, [ptrx, &next_calls, &num_posts, &trace_with_except, &trx_match, &app]() {
          ++num_posts;
          bool return_failure_traces = true;
          app->get_method<plugin_interface::incoming::methods::transaction_async>()(ptrx,
@@ -126,12 +145,15 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
                ++next_calls;
         });
       });
+      app->executor().post( priority::low, exec_queue::read_only, [&chain_plug=chain_plug]() {
+         chain_plug->get_read_only_api(fc::seconds(90)).get_consensus_parameters(chain_apis::read_only::get_consensus_parameters_params{}, fc::time_point::now()+fc::seconds(90));
+      });
    }
 
    // Wait long enough such that all transactions are executed
    auto start = fc::time_point::now();
    auto hard_deadline = start + fc::seconds(10); // To protect against waiting forever
-   while ( next_calls < num_pushes && fc::time_point::now() < hard_deadline ){
+   while ( (next_calls < num_pushes || num_get_account_calls < num_pushes) && fc::time_point::now() < hard_deadline ){
       std::this_thread::sleep_for( 100ms );;
    }
 
@@ -141,13 +163,39 @@ void test_trxs_common(std::vector<const char*>& specific_args) {
    BOOST_CHECK_EQUAL( trace_with_except, 0 ); // should not have any traces with except in it
    BOOST_CHECK_EQUAL( num_pushes, num_posts );
    BOOST_CHECK_EQUAL( num_pushes, next_calls.load() );
-   BOOST_CHECK( trx_match.load() );  // trace should match the transaction  
+   BOOST_CHECK_EQUAL( num_pushes, num_get_account_calls.load() );
+   BOOST_CHECK( trx_match.load() );  // trace should match the transaction
 }
 
 // test read-only trxs on main thread (no --read-only-threads)
 BOOST_AUTO_TEST_CASE(no_read_only_threads) {
-   std::vector<const char*> specific_args = { "-p", "eosio", "-e" };
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e", "--abi-serializer-max-time-ms=999" };
    test_trxs_common(specific_args);
 }
 
+// test read-only trxs on 1 threads (with --read-only-threads)
+BOOST_AUTO_TEST_CASE(with_1_read_only_threads) {
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
+                                              "--read-only-threads=1",
+                                              "--max-transaction-time=10",
+                                              "--abi-serializer-max-time-ms=999",
+                                              "--read-only-write-window-time-us=100000",
+                                              "--read-only-read-window-time-us=40000",
+                                              "--disable-subjective-billing=true" };
+   test_trxs_common(specific_args);
+}
+
+// test read-only trxs on 8 separate threads (with --read-only-threads)
+BOOST_AUTO_TEST_CASE(with_8_read_only_threads) {
+   std::vector<const char*> specific_args = { "-p", "eosio", "-e",
+                                              "--read-only-threads=8",
+                                              "--max-transaction-time=10",
+                                              "--abi-serializer-max-time-ms=999",
+                                              "--read-only-write-window-time-us=100000",
+                                              "--read-only-read-window-time-us=40000",
+                                              "--disable-subjective-billing=true" };
+   test_trxs_common(specific_args);
+}
+
+
 BOOST_AUTO_TEST_SUITE_END()

plugins/producer_plugin/test/test_snapshot_scheduler.cpp

@@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
             fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
             std::vector<const char*> argv =
                   {"test", "--data-dir", temp.c_str(), "--config-dir", temp.c_str(),
-                   "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true"};
+                   "-p", "eosio", "-e", "--disable-subjective-billing=true"};
             appbase::app().initialize<chain_plugin, producer_plugin>(argv.size(), (char**) &argv[0]);
             appbase::app().startup();
             plugin_promise.set_value(
@@ -93,18 +93,20 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          auto bs = chain_plug->chain().block_start.connect([&pp](uint32_t bn) {
             // catching pending snapshot
             if (!pp->get_snapshot_requests().snapshot_requests.empty()) {
-               auto& pending = pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots;
-               if (pending && pending->size()==1) {
-                  // lets check the head block num of it, it should be 8 + 1 = 9
-                  // this means we are getting a snapshot for correct block # as well
-                  BOOST_CHECK_EQUAL(9, pending->begin()->head_block_num);   
+               const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests;
+               auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(), [](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;});
+               // we should have a pending snapshot for request id = 0
+               BOOST_REQUIRE(it != snapshot_requests.end());
+               auto& pending = it->pending_snapshots;
+               if (pending.size()==1) {
+                  BOOST_CHECK_EQUAL(9, pending.begin()->head_block_num);   
                }
             }
          });
        
-         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
-         producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
-         producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 1"};
+         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 1"};
+         producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2 that will never happen"};
+         producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 3 that will expire"};
 
          pp->schedule_snapshot(sri1);
          pp->schedule_snapshot(sri2);
@@ -118,8 +120,13 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          // one of the snapshots is done here and request, corresponding to it should be deleted
          BOOST_CHECK_EQUAL(2, pp->get_snapshot_requests().snapshot_requests.size());
 
-         // check whether no pending snapshots present
-         BOOST_CHECK_EQUAL(0, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->size());
+         // check whether no pending snapshots present for a snapshot with id 0
+         const auto& snapshot_requests = pp->get_snapshot_requests().snapshot_requests;
+         auto it = find_if(snapshot_requests.begin(), snapshot_requests.end(),[](const producer_plugin::snapshot_schedule_information& obj) {return obj.snapshot_request_id == 0;});
+
+         // snapshot request with id = 0 should be found and should not have any pending snapshots
+         BOOST_REQUIRE(it != snapshot_requests.end());
+         BOOST_CHECK(!it->pending_snapshots.size());
 
          // quit app
          appbase::app().quit();

plugins/producer_plugin/test/test_trx_full.cpp

@@ -112,7 +112,7 @@ BOOST_AUTO_TEST_CASE(producer) {
          fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
          std::vector<const char*> argv =
                {"test", "--data-dir", temp_dir_str.c_str(), "--config-dir", temp_dir_str.c_str(),
-                "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true" };
+                "-p", "eosio", "-e", "--disable-subjective-billing=true" };
          app->initialize<chain_plugin, producer_plugin>( argv.size(), (char**) &argv[0] );
          app->startup();
          plugin_promise.set_value(

plugins/prometheus_plugin/include/eosio/prometheus_plugin/simple_rest_server.hpp

@@ -0,0 +1,234 @@
+#pragma once
+
+#include <boost/beast/core.hpp>
+#include <boost/beast/http.hpp>
+#include <boost/beast/version.hpp>
+
+namespace eosio { namespace rest {
+
+   // The majority of the code here are derived from boost source
+   // libs/beast/example/http/server/async/http_server_async.cpp
+   // with minimum modification and yet reusable.
+
+   namespace beast = boost::beast;         // from <boost/beast.hpp>
+   namespace http  = beast::http;          // from <boost/beast/http.hpp>
+   namespace net   = boost::asio;          // from <boost/asio.hpp>
+   using tcp       = boost::asio::ip::tcp; // from <boost/asio/ip/tcp.hpp>
+   template <typename T>
+   class simple_server {
+      T* self() { return static_cast<T*>(this); }
+
+      void fail(beast::error_code ec, char const* what) { self()->log_error(what, ec.message()); }
+      // Return a response for the given request.
+      http::response<http::string_body> handle_request(http::request<http::string_body>&& req) {
+         auto server_header = self()->server_header();
+         // Returns a bad request response
+         auto const bad_request = [&req, &server_header](std::string_view why) {
+            http::response<http::string_body> res{ http::status::bad_request, req.version() };
+            res.set(http::field::server, server_header);
+            res.set(http::field::content_type, "text/plain");
+            res.keep_alive(req.keep_alive());
+            res.body() = std::string(why);
+            res.prepare_payload();
+            return res;
+         };
+
+         // Returns a not found response
+         auto const not_found = [&req, &server_header](std::string_view target) {
+            http::response<http::string_body> res{ http::status::not_found, req.version() };
+            res.set(http::field::server, server_header);
+            res.set(http::field::content_type, "text/plain");
+            res.keep_alive(req.keep_alive());
+            res.body() = "The resource '" + std::string(target) + "' was not found.";
+            res.prepare_payload();
+            return res;
+         };
+
+         // Returns a server error response
+         auto const server_error = [&req, &server_header](std::string_view what) {
+            http::response<http::string_body> res{ http::status::internal_server_error, req.version() };
+            res.set(http::field::server, server_header);
+            res.set(http::field::content_type, "text/plain");
+            res.keep_alive(req.keep_alive());
+            res.body() = "An error occurred: '" + std::string(what) + "'";
+            res.prepare_payload();
+            return res;
+         };
+
+         // Make sure we can handle the method
+         if (!self()->allow_method(req.method()))
+            return bad_request("Unknown HTTP-method");
+
+         // Request path must be absolute and not contain "..".
+         std::string_view target{req.target().data(), req.target().size()};
+         if (target.empty() || target[0] != '/' || target.find("..") != std::string_view::npos)
+            return bad_request("Illegal request-target");
+
+         try {
+            auto res = self()->on_request(std::move(req));
+            if (!res)
+               return not_found(target);
+            return *res;
+         } catch (std::exception& ex) { return server_error(ex.what()); }
+      }
+
+      class session : public std::enable_shared_from_this<session> {
+         tcp::socket                                        socket_;
+         boost::asio::io_context::strand                    strand_;
+         beast::flat_buffer                                 buffer_;
+         http::request<http::string_body>                   req_;
+         simple_server*                                     server_;
+         std::shared_ptr<http::response<http::string_body>> res_;
+
+       public:
+         // Take ownership of the stream
+         session(net::io_context& ioc, tcp::socket&& socket, simple_server* server)
+             : socket_(std::move(socket)), strand_(ioc), server_(server) {}
+
+         // Start the asynchronous operation
+         void run() { do_read(); }
+
+         void do_read() {
+            // Make the request empty before reading,
+            // otherwise the operation behavior is undefined.
+            req_ = {};
+
+            // Read a request
+            http::async_read(
+                  socket_, buffer_, req_,
+                  boost::asio::bind_executor(strand_, [self = this->shared_from_this()](beast::error_code ec,
+                                                                                        std::size_t bytes_transferred) {
+                     self->on_read(ec, bytes_transferred);
+                  }));
+         }
+
+         void on_read(beast::error_code ec, std::size_t bytes_transferred) {
+            boost::ignore_unused(bytes_transferred);
+
+            // This means they closed the connection
+            if (ec == http::error::end_of_stream)
+               return do_close();
+
+            if (ec)
+               return server_->fail(ec, "read");
+
+            // Send the response
+            send_response(server_->handle_request(std::move(req_)));
+         }
+
+         void send_response(http::response<http::string_body>&& msg) {
+            // The lifetime of the message has to extend
+            // for the duration of the async operation so
+            // we use a shared_ptr to manage it.
+            res_ = std::make_shared<http::response<http::string_body>>(std::move(msg));
+
+            // Write the response
+            http::async_write(socket_, *res_,
+                              boost::asio::bind_executor(socket_.get_executor(),
+                                                         [self = this->shared_from_this(), close = res_->need_eof()](
+                                                               beast::error_code ec, std::size_t bytes_transferred) {
+                                                            self->on_write(ec, bytes_transferred, close);
+                                                         }));
+         }
+
+         void on_write(boost::system::error_code ec, std::size_t bytes_transferred, bool close) {
+            boost::ignore_unused(bytes_transferred);
+
+            if (ec)
+               return server_->fail(ec, "write");
+
+            if (close) {
+               // This means we should close the connection, usually because
+               // the response indicated the "Connection: close" semantic.
+               return do_close();
+            }
+
+            // We're done with the response so delete it
+            res_ = nullptr;
+
+            // Read another request
+            do_read();
+         }
+
+         void do_close() {
+            // Send a TCP shutdown
+            beast::error_code ec;
+            socket_.shutdown(tcp::socket::shutdown_send, ec);
+
+            // At this point the connection is closed gracefully
+         }
+      };
+
+      //------------------------------------------------------------------------------
+
+      // Accepts incoming connections and launches the sessions
+      class listener : public std::enable_shared_from_this<listener> {
+         net::io_context& ioc_;
+         tcp::acceptor    acceptor_;
+         tcp::socket      socket_;
+         simple_server*   server_;
+
+       public:
+         listener(net::io_context& ioc, tcp::endpoint endpoint, simple_server* server)
+             : ioc_(ioc), acceptor_(ioc), socket_(ioc), server_(server) {
+            boost::system::error_code ec;
+
+            // Open the acceptor
+            acceptor_.open(endpoint.protocol(), ec);
+            if (ec) {
+               server_->fail(ec, "open");
+               return;
+            }
+
+            // Allow address reuse
+            acceptor_.set_option(net::socket_base::reuse_address(true), ec);
+            if (ec) {
+               server_->fail(ec, "set_option");
+               return;
+            }
+
+            // Bind to the server address
+            acceptor_.bind(endpoint, ec);
+            if (ec) {
+               server_->fail(ec, "bind");
+               return;
+            }
+
+            // Start listening for connections
+            acceptor_.listen(net::socket_base::max_listen_connections, ec);
+            if (ec) {
+               server_->fail(ec, "listen");
+               return;
+            }
+         }
+
+         // Start accepting incoming connections
+         void run() {
+            if (!acceptor_.is_open())
+               return;
+            do_accept();
+         }
+
+       private:
+         void do_accept() {
+            acceptor_.async_accept(
+                  socket_, [self = this->shared_from_this()](boost::system::error_code ec) { self->on_accept(ec); });
+         }
+
+         void on_accept(boost::system::error_code ec) {
+            if (ec) {
+               server_->fail(ec, "accept");
+            } else {
+               // Create the session and run it
+               std::make_shared<session>(ioc_, std::move(socket_), server_)->run();
+            }
+
+            // Accept another connection
+            do_accept();
+         }
+      };
+
+    public:
+      void run(net::io_context& ioc, tcp::endpoint endpoint) { std::make_shared<listener>(ioc, endpoint, this)->run(); }
+   };
+}} // namespace eosio::rest

plugins/prometheus_plugin/prometheus_plugin.cpp

@@ -1,8 +1,11 @@
 #include <eosio/prometheus_plugin/prometheus_plugin.hpp>
+#include <eosio/prometheus_plugin/simple_rest_server.hpp>
+
 #include <eosio/chain/plugin_interface.hpp>
 #include <eosio/chain/thread_utils.hpp>
 #include <eosio/http_plugin/macros.hpp>
 #include <eosio/net_plugin/net_plugin.hpp>
+#include <eosio/http_plugin/http_plugin.hpp>
 #include <eosio/producer_plugin/producer_plugin.hpp>
 
 #include <fc/log/logger.hpp>
@@ -14,8 +17,8 @@
 #include <prometheus/text_serializer.h>
 #include <prometheus/registry.h>
 
-
 namespace eosio {
+   static const char* prometheus_api_name = "/v1/prometheus/metrics";
    using namespace prometheus;
    using namespace chain::plugin_interface;
 
@@ -110,12 +113,41 @@ namespace eosio {
       }
    };
 
-   struct prometheus_plugin_impl {
+   namespace http = boost::beast::http;
+   struct prometheus_plugin_impl : rest::simple_server<prometheus_plugin_impl> {
+
+      std::string server_header() const {
+         return http_plugin::get_server_header();
+      }
+
+      void log_error(char const* what, const std::string& message) {
+         elog("${what}: ${message}", ("what", what)("message", message));
+      }
+
+      bool allow_method(http::verb method) const {
+         return method == http::verb::get;
+      }
+
+      std::optional<http::response<http::string_body>>
+       on_request(http::request<http::string_body>&& req) {
+         if(req.target() != prometheus_api_name)
+            return {};
+         http::response<http::string_body> res{ http::status::ok, req.version() };
+         // Respond to GET request
+         res.set(http::field::server, server_header());
+         res.set(http::field::content_type, "text/plain");
+         res.keep_alive(req.keep_alive());
+         res.body() = metrics();
+         res.prepare_payload();
+         return res;
+      }
+
       eosio::chain::named_thread_pool<struct prom> _prometheus_thread_pool;
       boost::asio::io_context::strand _prometheus_strand;
       prometheus_plugin_metrics _metrics;
 
       map<std::string, vector<runtime_metric>> _plugin_metrics;
+      boost::asio::ip::tcp::endpoint           _endpoint;
 
       prometheus_plugin_impl(): _prometheus_strand(_prometheus_thread_pool.get_executor()){ }
 
@@ -179,32 +211,11 @@ namespace eosio {
          return body;
       }
 
-      void metrics_async(chain::plugin_interface::next_function<std::string> results) {
-         _prometheus_strand.post([self=this, results=std::move(results)]() {
-            results(self->metrics());
-         });
+      void start() {
+         run(_prometheus_thread_pool.get_executor(), _endpoint);
+         _prometheus_thread_pool.start(
+               1, [](const fc::exception& e) { elog("Prometheus exception ${e}", ("e", e)); });
       }
-
-   };
-
-   using metrics_params = fc::variant_object;
-
-   struct prometheus_api {
-      prometheus_plugin_impl& _pp;
-      fc::microseconds _max_response_time_us;
-
-      fc::time_point start() const {
-         return fc::time_point::now() + _max_response_time_us;
-      }
-
-      void metrics(const metrics_params&, chain::plugin_interface::next_function<std::string> results) {
-         _pp.metrics_async(std::move(results));
-      }
-
-      prometheus_api(prometheus_plugin_impl& plugin, const fc::microseconds& max_response_time)
-      : _pp(plugin)
-      , _max_response_time_us(max_response_time){}
-
    };
 
    prometheus_plugin::prometheus_plugin()
@@ -214,24 +225,35 @@ namespace eosio {
    prometheus_plugin::~prometheus_plugin() = default;
 
    void prometheus_plugin::set_program_options(options_description&, options_description& cfg) {
+      cfg.add_options()
+         ("prometheus-exporter-address", bpo::value<string>()->default_value("127.0.0.1:9101"),
+            "The local IP and port to listen for incoming prometheus metrics http request.");
    }
 
    void prometheus_plugin::plugin_initialize(const variables_map& options) {
       my->initialize_metrics();
 
-      auto& _http_plugin = app().get_plugin<http_plugin>();
-      fc::microseconds max_response_time = _http_plugin.get_max_response_time();
+      string lipstr = options.at("prometheus-exporter-address").as<string>();
+      EOS_ASSERT(lipstr.size() > 0, chain::plugin_config_exception, "prometheus-exporter-address must have a value");
 
-      prometheus_api handle(*my, max_response_time);
-      app().get_plugin<http_plugin>().add_async_api({
-        CALL_ASYNC_WITH_400(prometheus, handle, eosio, metrics, std::string, 200, http_params_types::no_params)}, http_content_type::plaintext);
+      string host = lipstr.substr(0, lipstr.find(':'));
+      string port = lipstr.substr(host.size() + 1, lipstr.size());
+
+      boost::system::error_code ec;
+      using tcp = boost::asio::ip::tcp;
+      tcp::resolver resolver(app().get_io_service());
+
+      my->_endpoint = *resolver.resolve(tcp::v4(), host, port, ec);
+      if (!ec) {
+         fc_ilog(logger(), "configured prometheus metrics exporter to listen on ${h}", ("h", lipstr));
+      } else {
+         fc_elog(logger(), "failed to configure prometheus metrics exporter to listen on ${h} (${m})",
+                 ("h", lipstr)("m", ec.message()));
+      }
    }
 
    void prometheus_plugin::plugin_startup() {
-      my->_prometheus_thread_pool.start(1, []( const fc::exception& e ) {
-         elog("Prometheus excpetion ${e}:${l}", ("e", e));
-      } );
-
+      my->start();
       ilog("Prometheus plugin started.");
    }
 

plugins/state_history_plugin/include/eosio/state_history_plugin/session.hpp

@@ -103,7 +103,7 @@ public:
    void pop_entry(bool call_send = true) {
       send_queue.erase(send_queue.begin());
       sending = false;
-      if (call_send)
+      if (call_send || !send_queue.empty())
          send();
    }
 
@@ -491,7 +491,10 @@ private:
          return;
       }
 
-      auto block_id = plugin->get_block_id(to_send_block_num);
+      // not just an optimization, on accepted_block signal may not be able to find block_num in forkdb as it has not been validated
+      // until after the accepted_block signal
+      std::optional<chain::block_id_type> block_id =
+          (block_state && block_state->block_num == to_send_block_num) ? block_state->id : plugin->get_block_id(to_send_block_num);
 
       if (block_id && position_it && (*position_it)->block_num == to_send_block_num) {
          // This branch happens when the head block of nodeos is behind the head block of connecting client.
@@ -550,6 +553,7 @@ private:
 
       state_history::get_blocks_result_v0 result;
       result.head = {block_state->block_num, block_state->id};
+      to_send_block_num = std::min(block_state->block_num, to_send_block_num);
       send_update(std::move(result), block_state);
    }
 

plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp

@@ -3,6 +3,7 @@
 
 #include <eosio/chain_plugin/chain_plugin.hpp>
 #include <eosio/state_history/types.hpp>
+#include <eosio/state_history/log.hpp>
 
 namespace fc {
 class variant;
@@ -11,7 +12,6 @@ class variant;
 namespace eosio {
 using chain::bytes;
 using std::shared_ptr;
-
 typedef shared_ptr<struct state_history_plugin_impl> state_history_ptr;
 
 class state_history_plugin : public plugin<state_history_plugin> {
@@ -29,6 +29,9 @@ class state_history_plugin : public plugin<state_history_plugin> {
 
    void handle_sighup() override;
 
+   const state_history_log* trace_log() const;
+   const state_history_log* chain_state_log() const;
+
  private:
    state_history_ptr my;
 };

plugins/state_history_plugin/state_history_plugin.cpp

@@ -100,6 +100,8 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
 
    named_thread_pool<struct ship> thread_pool;
 
+   bool  plugin_started = false;
+
    static fc::logger& logger() { return _log; }
 
    std::optional<state_history_log>& get_trace_log() { return trace_log; }
@@ -292,9 +294,15 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
              "the process");
       }
 
-      boost::asio::post(get_ship_executor(), [self = this->shared_from_this(), block_state]() {
-         self->session_mgr.send_update(block_state);
-      });
+      // avoid accumulating all these posts during replay before ship threads started
+      // that can lead to a large memory consumption and failures
+      // this is safe as there are no clients connected until after replay is complete
+      // this method is called from the main thread and "plugin_started" is set on the main thread as well when plugin is started 
+      if (plugin_started) {
+         boost::asio::post(get_ship_executor(), [self = this->shared_from_this(), block_state]() {
+            self->session_mgr.send_update(block_state);
+         });
+      }
 
    }
 
@@ -455,7 +463,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) {
 
       state_history_log_config ship_log_conf;
       if (options.count("state-history-log-retain-blocks")) {
-         auto ship_log_prune_conf = ship_log_conf.emplace<state_history::prune_config>();
+         auto& ship_log_prune_conf = ship_log_conf.emplace<state_history::prune_config>();
          ship_log_prune_conf.prune_blocks = options.at("state-history-log-retain-blocks").as<uint32_t>();
          //the arbitrary limit of 1000 here is mainly so that there is enough buffer for newly applied forks to be delivered to clients
          // before getting pruned out. ideally pruning would have been smart enough to know not to prune reversible blocks
@@ -495,7 +503,8 @@ void state_history_plugin::plugin_startup() {
       my->thread_pool.start( 1, [](const fc::exception& e) {
          fc_elog( _log, "Exception in SHiP thread pool, exiting: ${e}", ("e", e.to_detail_string()) );
          app().quit();
-      } );
+      });
+      my->plugin_started = true; 
    } catch (std::exception& ex) {
       appbase::app().quit();
    }
@@ -512,4 +521,12 @@ void state_history_plugin::handle_sighup() {
    fc::logger::update(logger_name, _log);
 }
 
+const state_history_log* state_history_plugin::trace_log() const {
+   return my->trace_log ? std::addressof(*my->trace_log) : nullptr;
+}
+
+const state_history_log* state_history_plugin::chain_state_log() const {
+   return my->chain_state_log ? std::addressof(*my->chain_state_log) : nullptr;
+}
+
 } // namespace eosio

plugins/state_history_plugin/tests/CMakeLists.txt

@@ -1,5 +1,5 @@
-add_executable( test_state_history_session session_test.cpp )
-target_link_libraries(test_state_history_session state_history Boost::unit_test_framework)
-target_include_directories( test_state_history_session PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include" )
+add_executable( test_state_history session_test.cpp plugin_config_test.cpp)
+target_link_libraries(test_state_history state_history_plugin Boost::unit_test_framework)
+target_include_directories( test_state_history PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include" )
 
-add_test(test_state_history_session test_state_history_session)
+add_test(test_state_history test_state_history)

plugins/state_history_plugin/tests/plugin_config_test.cpp

@@ -0,0 +1,40 @@
+#include <array>
+#include <boost/test/unit_test.hpp>
+#include <eosio/state_history/log.hpp>
+#include <eosio/state_history_plugin/state_history_plugin.hpp>
+#include <fc/filesystem.hpp>
+#include <stdint.h>
+
+BOOST_AUTO_TEST_CASE(state_history_plugin_default_tests) {
+   fc::temp_directory  tmp;
+   appbase::scoped_app app;
+
+   auto tmp_path = tmp.path().string();
+   std::array args = {"test_state_history",    "--trace-history", "--state-history-stride", "10",
+                      "--disable-replay-opts", "--data-dir",      tmp_path.c_str()};
+
+   BOOST_CHECK(app->initialize<eosio::state_history_plugin>(args.size(), const_cast<char**>(args.data())));
+   auto& plugin = app->get_plugin<eosio::state_history_plugin>();
+
+   BOOST_REQUIRE(plugin.trace_log());
+   auto* config = std::get_if<eosio::state_history::partition_config>(&plugin.trace_log()->config());
+   BOOST_REQUIRE(config);
+   BOOST_CHECK_EQUAL(config->max_retained_files, UINT32_MAX);
+}
+
+BOOST_AUTO_TEST_CASE(state_history_plugin_retain_blocks_tests) {
+   fc::temp_directory  tmp;
+   appbase::scoped_app app;
+
+   auto tmp_path = tmp.path().string();
+   std::array args = {"test_state_history",    "--trace-history", "--state-history-log-retain-blocks", "4242",
+                      "--disable-replay-opts", "--data-dir",      tmp_path.c_str()};
+
+   BOOST_CHECK(app->initialize<eosio::state_history_plugin>(args.size(), const_cast<char**>(args.data())));
+   auto& plugin = app->get_plugin<eosio::state_history_plugin>();
+
+   BOOST_REQUIRE(plugin.trace_log());
+   auto* config = std::get_if<eosio::state_history::prune_config>(&plugin.trace_log()->config());
+   BOOST_REQUIRE(config);
+   BOOST_CHECK_EQUAL(config->prune_blocks, 4242);
+}

plugins/test_control_api_plugin/test_control_api_plugin.cpp

@@ -51,7 +51,7 @@ void test_control_api_plugin::plugin_startup() {
 
    app().get_plugin<http_plugin>().add_api({
       TEST_CONTROL_RW_CALL(kill_node_on_producer, 202, http_params_types::params_required)
-   }, appbase::exec_queue::general);
+   }, appbase::exec_queue::read_write);
 }
 
 void test_control_api_plugin::plugin_shutdown() {}

plugins/wallet_api_plugin/wallet_api_plugin.cpp

@@ -115,7 +115,7 @@ void wallet_api_plugin::plugin_startup() {
             INVOKE_R_R_R(wallet_mgr, list_keys, std::string, std::string), 200),
        CALL_WITH_400(wallet, wallet_mgr, get_public_keys,
             INVOKE_R_V(wallet_mgr, get_public_keys), 200)
-   }, appbase::exec_queue::general);
+   }, appbase::exec_queue::read_write);
 }
 
 void wallet_api_plugin::plugin_initialize(const variables_map& options) {

programs/keosd/main.cpp

@@ -111,7 +111,7 @@ int main(int argc, char** argv)
                        [&a=app](string, string, url_response_callback cb) {
          cb(200, fc::time_point::maximum(), fc::variant(fc::variant_object()));
          a->quit();
-      }, appbase::exec_queue::general );
+      }, appbase::exec_queue::read_write );
       app->startup();
       app->exec();
    } catch (const fc::exception& e) {

programs/leap-util/actions/blocklog.cpp

@@ -88,19 +88,19 @@ void blocklog_actions::setup(CLI::App& app) {
    auto* extract_blocks = sub->add_subcommand("extract-blocks", "Extract range of blocks from blocks.log and write to output-dir.  Must give 'first' and/or 'last'.")->callback([err_guard]() { err_guard(&blocklog_actions::extract_blocks); });
    extract_blocks->add_option("--first,-f", opt->first_block, "The first block number to keep.")->required();
    extract_blocks->add_option("--last,-l", opt->last_block, "The last block number to keep.")->required();
-   extract_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the block log extracted from blocks-dir.");
+   extract_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the block log extracted from blocks-dir.")->required();
 
    // subcommand - split blocks
    auto* split_blocks = sub->add_subcommand("split-blocks", "Split the blocks.log based on the stride and store the result in the specified 'output-dir'.")->callback([err_guard]() { err_guard(&blocklog_actions::split_blocks); });
    split_blocks->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
-   split_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the split block log.");
+   split_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the split block log.")->required();
    split_blocks->add_option("--stride", opt->stride, "The number of blocks to split into each file.")->required();
 
    // subcommand - merge blocks
    auto* merge_blocks = sub->add_subcommand("merge-blocks", "Merge block log files in 'blocks-dir' with the file pattern 'blocks-\\d+-\\d+.[log,index]' to 'output-dir' whenever possible."
           "The files in 'blocks-dir' will be kept without change.")->callback([err_guard]() { err_guard(&blocklog_actions::merge_blocks); });
    merge_blocks->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
-   merge_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the merged block log.");
+   merge_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the merged block log.")->required();
 
    // subcommand - smoke test
    sub->add_subcommand("smoke-test", "Quick test that blocks.log and blocks.index are well formed and agree with each other.")->callback([err_guard]() { err_guard(&blocklog_actions::smoke_test); });
@@ -169,32 +169,33 @@ int blocklog_actions::extract_blocks() {
 }
 
 int blocklog_actions::do_genesis() {
-   std::optional<genesis_state> gs;
    bfs::path bld = opt->blocks_dir;
-   auto full_path = (bld / "blocks.log").generic_string();
 
-   if(fc::exists(bld / "blocks.log")) {
-      gs = block_log::extract_genesis_state(opt->blocks_dir);
-      if(!gs) {
-         std::cerr << "Block log at '" << full_path
-                   << "' does not contain a genesis state, it only has the chain-id." << std::endl;
-         return -1;
-      }
-   } else {
-      std::cerr << "No blocks.log found at '" << full_path << "'." << std::endl;
+   auto context = block_log::extract_chain_context(opt->blocks_dir,opt->blocks_dir);
+   
+   if (!context) {
+      std::cerr << "No blocks log found at '" << opt->blocks_dir.c_str() << "'." << std::endl;
       return -1;
    }
 
+   if(!std::holds_alternative<genesis_state>(*context)) {
+      std::cerr << "Block log at '" << opt->blocks_dir.c_str()
+                  << "' does not contain a genesis state, it only has the chain-id." << std::endl;
+      return -1;
+   } 
+
+   const genesis_state& gs = std::get<genesis_state>(*context);
+
    // just print if output not set
    if(opt->output_file.empty()) {
-      std::cout << json::to_pretty_string(*gs) << std::endl;
+      std::cout << json::to_pretty_string(gs) << std::endl;
    } else {
       bfs::path p = opt->output_file;
       if(p.is_relative()) {
          p = bfs::current_path() / p;
       }
 
-      if(!fc::json::save_to_file(*gs, p, true)) {
+      if(!fc::json::save_to_file(gs, p, true)) {
          std::cerr << "Error occurred while writing genesis JSON to '" << p.generic_string() << "'" << std::endl;
          return -1;
       }

scripts/CMakeLists.txt

@@ -6,3 +6,4 @@ configure_file(eosio-tn_up.sh eosio-tn_up.sh COPYONLY)
 configure_file(abi_is_json.py abi_is_json.py COPYONLY)
 configure_file(postinst .)
 configure_file(prerm .)
+configure_file(install_testharness_symlinks.cmake .)

scripts/install_deps.sh

@@ -1,6 +1,27 @@
-#!/usr/bin/env bash
-
+#!/bin/bash
 apt-get update
 apt-get update --fix-missing
-DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
-apt-get -y install zip unzip libncurses5 wget git build-essential cmake curl libgmp-dev libssl-dev libzstd-dev time zlib1g-dev libtinfo-dev bzip2 libbz2-dev python3 python3-numpy file
+export DEBIAN_FRONTEND='noninteractive'
+export TZ='Etc/UTC'
+apt-get install -y \
+    build-essential \
+    bzip2 \
+    cmake \
+    curl \
+    file \
+    git \
+    libbz2-dev \
+    libcurl4-openssl-dev \
+    libgmp-dev \
+    libncurses5 \
+    libssl-dev \
+    libtinfo-dev \
+    libzstd-dev \
+    python3 \
+    python3-numpy \
+    time \
+    tzdata \
+    unzip \
+    wget \
+    zip \
+    zlib1g-dev

scripts/install_testharness_symlinks.cmake

@@ -0,0 +1,5 @@
+execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages  ERROR_QUIET RESULT_VARIABLE ret)
+if(ret EQUAL "0")
+    execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)
+    execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)
+endif()

scripts/pinned_build.sh

@@ -1,151 +1,185 @@
-#!/usr/bin/env bash
+#!/bin/bash
+set -eo pipefail
 
 echo "Leap Pinned Build"
 
 if [[ "$(uname)" == "Linux" ]]; then
-   if [[ -e /etc/os-release ]]; then
-      # obtain NAME and other information
-      . /etc/os-release
-      if [[ ${NAME} != "Ubuntu" ]]; then
-         echo "Currently only supporting Ubuntu based builds. Proceed at your own risk."
-      fi
-   else
-       echo "Currently only supporting Ubuntu based builds. /etc/os-release not found. Your Linux distribution is not supported. Proceed at your own risk."
-   fi
+    if [[ -e /etc/os-release ]]; then
+        # obtain NAME and other information
+        . /etc/os-release
+        if [[ "${NAME}" != "Ubuntu" ]]; then
+            echo "Currently only supporting Ubuntu based builds. Proceed at your own risk."
+        fi
+    else
+        echo "Currently only supporting Ubuntu based builds. /etc/os-release not found. Your Linux distribution is not supported. Proceed at your own risk."
+    fi
 else
     echo "Currently only supporting Ubuntu based builds. Your architecture is not supported. Proceed at your own risk."
 fi
 
-if [ $# -eq 0 ] || [ -z "$1" ]
-   then
-      echo "Please supply a directory for the build dependencies to be placed and a directory for leap build and a value for the number of jobs to use for building."
-      echo "The binary packages will be created and placed into the leap build directory."
-      echo "./pinned_build.sh <dependencies directory> <leap build directory> <1-100>"
-      exit -1
+if [ $# -eq 0 ] || [ -z "$1" ]; then
+    echo "Please supply a directory for the build dependencies to be placed and a directory for leap build and a value for the number of jobs to use for building."
+    echo "The binary packages will be created and placed into the leap build directory."
+    echo "./pinned_build.sh <dependencies directory> <leap build directory> <1-100>"
+    exit 255
 fi
 
-CORE_SYM=EOS
+export CORE_SYM='EOS'
 # CMAKE_C_COMPILER requires absolute path
-DEP_DIR=`realpath $1`
-LEAP_DIR=$2
-JOBS=$3
+DEP_DIR="$(realpath "$1")"
+LEAP_DIR="$2"
+JOBS="$3"
 CLANG_VER=11.0.1
 BOOST_VER=1.70.0
 LLVM_VER=7.1.0
-ARCH=`uname -m`
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
 START_DIR="$(pwd)"
 
 
 pushdir() {
-   DIR=$1
-   mkdir -p ${DIR}
-   pushd ${DIR} &> /dev/null
+    DIR="$1"
+    mkdir -p "${DIR}"
+    pushd "${DIR}" &> /dev/null
 }
 
 popdir() {
-   EXPECTED=$1
-   D=`popd`
-   popd &> /dev/null
-   echo ${D}
-   D=`eval echo $D | head -n1 | cut -d " " -f1`
+    EXPECTED="$1"
+    D="$(popd)"
+    popd &> /dev/null
+    echo "${D}"
+    D="$(eval echo "$D" | head -n1 | cut -d " " -f1)"
 
-   # -ef compares absolute paths
-   if ! [[ ${D} -ef ${EXPECTED} ]]; then
-     echo "Directory is not where expected EXPECTED=${EXPECTED} at ${D}"
-     exit 1 
-   fi
+    # -ef compares absolute paths
+    if ! [[ "${D}" -ef "${EXPECTED}" ]]; then
+        echo "Directory is not where expected EXPECTED=${EXPECTED} at ${D}"
+        exit 1
+    fi
 }
 
 try(){
-   output=$($@)
-   res=$?
-   if [[ ${res} -ne 0 ]]; then
-      exit -1
-   fi
+    "$@"
+    res=$?
+    if [[ ${res} -ne 0 ]]; then
+        exit 255
+    fi
 }
 
 install_clang() {
-   CLANG_DIR=$1
-   if [ ! -d "${CLANG_DIR}" ]; then
-      echo "Installing Clang ${CLANG_VER} @ ${CLANG_DIR}"
-      mkdir -p ${CLANG_DIR}
-      CLANG_FN=clang+llvm-${CLANG_VER}-x86_64-linux-gnu-ubuntu-16.04.tar.xz
-      try wget -O ${CLANG_FN} https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VER}/${CLANG_FN}
-      try tar -xvf ${CLANG_FN} -C ${CLANG_DIR}
-      pushdir ${CLANG_DIR}
-      mv clang+*/* .
-      popdir ${DEP_DIR}
-      rm ${CLANG_FN}
-   fi
-   export PATH=${CLANG_DIR}/bin:$PATH
-   export CLANG_DIR=${CLANG_DIR}
+    CLANG_DIR="$1"
+    if [ ! -d "${CLANG_DIR}" ]; then
+        echo "Installing Clang ${CLANG_VER} @ ${CLANG_DIR}"
+        mkdir -p "${CLANG_DIR}"
+        CLANG_FN="clang+llvm-${CLANG_VER}-x86_64-linux-gnu-ubuntu-16.04.tar.xz"
+        try wget -O "${CLANG_FN}" "https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VER}/${CLANG_FN}"
+        try tar -xvf "${CLANG_FN}" -C "${CLANG_DIR}"
+        pushdir "${CLANG_DIR}"
+        mv clang+*/* .
+        popdir "${DEP_DIR}"
+        rm "${CLANG_FN}"
+    fi
+    export PATH="${CLANG_DIR}/bin:$PATH"
+    export CLANG_DIR="${CLANG_DIR}"
 }
 
 install_llvm() {
-   LLVM_DIR=$1
-   if [ ! -d "${LLVM_DIR}" ]; then
-      echo "Installing LLVM ${LLVM_VER} @ ${LLVM_DIR}"
-      mkdir -p ${LLVM_DIR}
-      try wget -O llvm-${LLVM_VER}.src.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/llvm-${LLVM_VER}.src.tar.xz
-      try tar -xvf llvm-${LLVM_VER}.src.tar.xz
-      pushdir "${LLVM_DIR}.src"
-      pushdir build
-      try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=${LLVM_DIR} -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=Off -DLLVM_ENABLE_RTTI=On -DLLVM_ENABLE_TERMINFO=Off -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO ..
-      try make -j${JOBS} 
-      try make -j${JOBS} install
-      popdir "${LLVM_DIR}.src"
-      popdir ${DEP_DIR}
-      rm -rf ${LLVM_DIR}.src 
-      rm llvm-${LLVM_VER}.src.tar.xz
-   fi
-   export LLVM_DIR=${LLVM_DIR}
+    LLVM_DIR="$1"
+    if [ ! -d "${LLVM_DIR}" ]; then
+        echo "Installing LLVM ${LLVM_VER} @ ${LLVM_DIR}"
+        mkdir -p "${LLVM_DIR}"
+        try wget -O "llvm-${LLVM_VER}.src.tar.xz" "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/llvm-${LLVM_VER}.src.tar.xz"
+        try tar -xvf "llvm-${LLVM_VER}.src.tar.xz"
+        pushdir "${LLVM_DIR}.src"
+        pushdir build
+        try cmake -DCMAKE_TOOLCHAIN_FILE="${SCRIPT_DIR}/pinned_toolchain.cmake" -DCMAKE_INSTALL_PREFIX="${LLVM_DIR}" -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=Off -DLLVM_ENABLE_RTTI=On -DLLVM_ENABLE_TERMINFO=Off -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO ..
+        try make -j "${JOBS}"
+        try make -j "${JOBS}" install
+        popdir "${LLVM_DIR}.src"
+        popdir "${DEP_DIR}"
+        rm -rf "${LLVM_DIR}.src"
+        rm "llvm-${LLVM_VER}.src.tar.xz"
+    fi
+    export LLVM_DIR="${LLVM_DIR}"
 }
 
 install_boost() {
-   BOOST_DIR=$1
+    BOOST_DIR="$1"
 
-   if [ ! -d "${BOOST_DIR}" ]; then
-      echo "Installing Boost ${BOOST_VER} @ ${BOOST_DIR}"
-      try wget -O boost_${BOOST_VER//\./_}.tar.gz https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VER}/source/boost_${BOOST_VER//\./_}.tar.gz
-      try tar --transform="s:^boost_${BOOST_VER//\./_}:boost_${BOOST_VER//\./_}patched:" -xvzf boost_${BOOST_VER//\./_}.tar.gz -C ${DEP_DIR}
-      pushdir ${BOOST_DIR}
-      patch -p1 < "${SCRIPT_DIR}/0001-beast-fix-moved-from-executor.patch"
-      try ./bootstrap.sh -with-toolset=clang --prefix=${BOOST_DIR}/bin
-      ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I${CLANG_DIR}/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fPIE' linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j${JOBS} install
-      popdir ${DEP_DIR}
-      rm boost_${BOOST_VER//\./_}.tar.gz
-   fi
-   export BOOST_DIR=${BOOST_DIR}
+    if [ ! -d "${BOOST_DIR}" ]; then
+        echo "Installing Boost ${BOOST_VER} @ ${BOOST_DIR}"
+        try wget -O "boost_${BOOST_VER//\./_}.tar.gz" "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VER}/source/boost_${BOOST_VER//\./_}.tar.gz"
+        try tar --transform="s:^boost_${BOOST_VER//\./_}:boost_${BOOST_VER//\./_}patched:" -xvzf "boost_${BOOST_VER//\./_}.tar.gz" -C "${DEP_DIR}"
+        pushdir "${BOOST_DIR}"
+        patch -p1 < "${SCRIPT_DIR}/0001-beast-fix-moved-from-executor.patch"
+        try ./bootstrap.sh -with-toolset=clang --prefix="${BOOST_DIR}/bin"
+        ./b2 toolset=clang cxxflags="-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I\${CLANG_DIR}/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fPIE" linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j "${JOBS}" install
+        popdir "${DEP_DIR}"
+        rm "boost_${BOOST_VER//\./_}.tar.gz"
+    fi
+    export BOOST_DIR="${BOOST_DIR}"
 }
 
-pushdir ${DEP_DIR}
+pushdir "${DEP_DIR}"
 
-install_clang ${DEP_DIR}/clang-${CLANG_VER}
-install_llvm ${DEP_DIR}/llvm-${LLVM_VER}
-install_boost ${DEP_DIR}/boost_${BOOST_VER//\./_}patched
+install_clang "${DEP_DIR}/clang-${CLANG_VER}"
+install_llvm "${DEP_DIR}/llvm-${LLVM_VER}"
+install_boost "${DEP_DIR}/boost_${BOOST_VER//\./_}patched"
 
 # go back to the directory where the script starts
-popdir ${START_DIR}
+popdir "${START_DIR}"
 
-pushdir ${LEAP_DIR}
+pushdir "${LEAP_DIR}"
 
 # build Leap
 echo "Building Leap ${SCRIPT_DIR}"
-try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=${LLVM_DIR}/lib/cmake -DCMAKE_PREFIX_PATH=${BOOST_DIR}/bin ${SCRIPT_DIR}/..
+try cmake -DCMAKE_TOOLCHAIN_FILE="${SCRIPT_DIR}/pinned_toolchain.cmake" -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="${LLVM_DIR}/lib/cmake" -DCMAKE_PREFIX_PATH="${BOOST_DIR}/bin" "${SCRIPT_DIR}/.."
 
-try make -j${JOBS}
+try make -j "${JOBS}"
 try cpack
 
-echo " .----------------.  .----------------.  .----------------.  .----------------. ";
-echo "| .--------------. || .--------------. || .--------------. || .--------------. |";
-echo "| |   _____      | || |  _________   | || |      __      | || |   ______     | |";
-echo "| |  |_   _|     | || | |_   ___  |  | || |     /  \     | || |  |_   __ \   | |";
-echo "| |    | |       | || |   | |_  \_|  | || |    / /\ \    | || |    | |__) |  | |";
-echo "| |    | |   _   | || |   |  _|  _   | || |   / ____ \   | || |    |  ___/   | |";
-echo "| |   _| |__/ |  | || |  _| |___/ |  | || | _/ /    \ \_ | || |   _| |_      | |";
-echo "| |  |________|  | || | |_________|  | || ||____|  |____|| || |  |_____|     | |";
-echo "| |              | || |              | || |              | || |              | |";
-echo "| '--------------' || '--------------' || '--------------' || '--------------' |";
-echo " '----------------'  '----------------'  '----------------'  '----------------' ";
-echo "Leap has successfully built and constructed its packages.  You should be able to find the packages at ${LEAP_DIR}.  Enjoy!!!"
+# art generated with DALL-E (https://openai.com/blog/dall-e), then fed through ASCIIart.club (https://asciiart.club) with permission
+cat <<'TXT' # BASH interpolation must remain disabled for the ASCII art to print correctly
+
+                                ,▄▄A`
+                             _╓▄██`
+                          ╓▄▓▓▀▀`
+                        ╓▓█▀╓▄▓
+                        ▓▌▓▓▓▀
+                      ,▄▓███▓H
+                    _╨╫▀╚▀╠▌`╙¥,
+                  ╓«    _╟▄▄   `½,                              ╓▄▄╦▄≥_
+                 ╙▓╫╬▒R▀▀╙▀▀▓φ_ «_╙Y╥▄mmMM#╦▄,_          ,╓╦mM╩╨╙╙╙\`║═
+                   ``       `▀▄__╫▓▓╨`    _```"""*ⁿⁿ^`````Ω,        `╟∩
+                              ╙▌▓▓"`    ,«ñ`            ╔╬▓▌⌂        ╔▌
+                               ╙█▌,,╔╗M╨,░             `  "╫▓m_      ╟H      _
+                        _,,,,__,╠█▓▒`  .╣▌µ _       _.╓╔▄▄▓█▓▓N_     ╙▀╩KKM╙╟▓N
+                      ,▄▓█▓████▀▀▀╙▀╓╔φ»█▓▓Ñ╦«, :»»µ╦▓▓█▀└╙▀███▓╥__   _,╓▄▓▓▓M▓`,
+                  __╓Φ▓█╫▓▓▓▓▓▓▓▓▓▓▓▓▓▀K▀▀███▓▓▓▓▓▓▀▀╙       `▀▀▀▓▄▄K╨╙└   `▀▌╙█▄*.
+                ,╓Φ▓▓▀▄▓▀`         ╙▀╙                                       ╙▓╙╙▓▄*
+              .▄▓╫▀╦▄▀`                                                       ╙▓╙µ╙▀
+             ▄▓▀╨▓▓╨_                                                          `▀▄▄M
+            _█▌▄▌╙`                                                               `╙
+             ╙└`
+
+
+                Ñ▓▓▓▓          ¢▄▄▄▄▄▄▄▄▄▄         ,           ,,,,,,,,,,,_
+               _╫▓▓█▌          ╟▓████████▀       ╓╣▓▌_         ╠╫▓█▓▓▓▓█▓▓▓▓▄
+               _▓▓▓█▌          ╟╫██▄,,,,_       æ▄███▓▄        ▐║████▀╨╨▀▓███▌
+               :╫▓▓█▌          ╟╢████████▓    ,╬███████▓,      ▐║████▓▄▄▓▓███▀
+               :╫▓▓█▌_        _╟║███▀▀▀▀▀▀   ╓╫███╣╫╬███▓N_    j▐██▀▀▀▀▀▀▀▀▀`
+    ___________]╫▓▓█▓φ╓╓╓╓╓╓,__╟╣█▌▌,,,,,_ _╬▓████████████▓▄_  ▐M█▌
+    _ _________]╫▌▓█████████▓▓▄╟╣████████▓▄╣██▓▀^    _ ╙▓██▓▓╗__M█▓
+      ___   _ _ ▀╣▀╣╩╩╩╩╩╩╩▀▀▀▀╙▀▀▀▀▀▀▀▀▀▀▀▀▀▀`          ╙▀▀▀▀╩═╩▀▀
+     _ __ __ _____ ___  ____ _  __  _  ____ ___
+    ____ ____ __  _ _  _ _ _ _ __   _ __ __
+    __ _ ________  ___ ________   ___ _ _ _      _
+       _ __  __     _ __ ____ _     _ ____ _   _ _       _
+     _  _ _     ____ ____ _ _    _ __  _ _ _ __
+
+---
+Leap has successfully built and constructed its packages. You should be able to
+find the packages at:
+TXT
+echo "${LEAP_DIR}"
+echo
+echo 'Thank you, fam!!'
+echo

tests/CMakeLists.txt

@@ -68,6 +68,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/large-lib-test.py ${CMAKE_CURRENT_BIN
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/launcher.py ${CMAKE_CURRENT_BINARY_DIR}/launcher.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/auto_bp_peering_test.py ${CMAKE_CURRENT_BINARY_DIR}/auto_bp_peering_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/auto_bp_peering_test_shape.json ${CMAKE_CURRENT_BINARY_DIR}/auto_bp_peering_test_shape.json COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/split_blocklog_replay_test.py ${CMAKE_CURRENT_BINARY_DIR}/split_blocklog_replay_test.py COPYONLY)
 
 if(DEFINED ENV{GITHUB_ACTIONS})
   set(UNSHARE "--unshared")
@@ -113,8 +114,8 @@ set_property(TEST ship_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME ship_test_unix COMMAND tests/ship_test.py -v --num-clients 10 --num-requests 5000 --clean-run ${UNSHARE} --unix-socket WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST ship_test_unix PROPERTY LABELS nonparallelizable_tests)
 
-add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --num-blocks 50 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST ship_streamer_test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME ship_streamer_test COMMAND tests/ship_streamer_test.py -v --num-clients 10 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST ship_streamer_test PROPERTY LABELS long_running_tests)
 
 add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST p2p_dawn515_test PROPERTY LABELS nonparallelizable_tests)
@@ -125,8 +126,12 @@ add_test(NAME nodeos_protocol_feature_test COMMAND tests/nodeos_protocol_feature
 set_property(TEST nodeos_protocol_feature_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME compute_transaction_test COMMAND tests/compute_transaction_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST compute_transaction_test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME read_only_trx_test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST read_only_trx_test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME read-only-trx-basic-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 0 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST read-only-trx-basic-test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME read-only-trx-parallel-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST read-only-trx-parallel-test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME read-only-trx-parallel-eos-vm-oc-test COMMAND tests/read_only_trx_test.py -v -p 2 -n 3 --eos-vm-oc-enable --read-only-threads 3 --num-test-runs 3 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST read-only-trx-parallel-eos-vm-oc-test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME subjective_billing_test COMMAND tests/subjective_billing_test.py -v -p 2 -n 4 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST subjective_billing_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME get_account_test COMMAND tests/get_account_test.py -v -p 2 --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
@@ -134,6 +139,8 @@ set_property(TEST get_account_test PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -d 2 -p 4 -n 6 -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST distributed-transactions-test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME distributed-transactions-speculative-test COMMAND tests/distributed-transactions-test.py -d 2 -p 4 -n 6 --speculative -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST distributed-transactions-speculative-test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME restart-scenarios-test-resync COMMAND tests/restart-scenarios-test.py -c resync -p4 -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST restart-scenarios-test-resync PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME restart-scenarios-test-hard_replay COMMAND tests/restart-scenarios-test.py -c hardReplay -p4 -v --clean-run ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
@@ -253,6 +260,9 @@ set_property(TEST light_validation_sync_test PROPERTY LABELS nonparallelizable_t
 add_test(NAME auto_bp_peering_test COMMAND tests/auto_bp_peering_test.py ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST auto_bp_peering_test PROPERTY LABELS long_running_tests)
 
+add_test(NAME split_blocklog_replay_test COMMAND tests/split_blocklog_replay_test.py ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST split_blocklog_replay_test PROPERTY LABELS nonparallelizable_tests)
+
 if(ENABLE_COVERAGE_TESTING)
 
   set(Coverage_NAME ${PROJECT_NAME}_coverage)

tests/TestHarness/Cluster.py

@@ -22,8 +22,11 @@ from .Node import BlockType
 from .Node import Node
 from .WalletMgr import WalletMgr
 from .launch_transaction_generators import TransactionGeneratorsLauncher, TpsTrxGensConfig
-from .libc import unshare, CLONE_NEWNET
-from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
+try:
+    from .libc import unshare, CLONE_NEWNET
+    from .interfaces import getInterfaceFlags, setInterfaceUp, IFF_LOOPBACK
+except:
+    pass
 
 # Protocol Feature Setup Policy
 class PFSetupPolicy:
@@ -1447,7 +1450,7 @@ class Cluster(object):
     def killall(self, kill=True, silent=True, allInstances=False):
         """Kill cluster nodeos instances. allInstances will kill all nodeos instances running on the system."""
         signalNum=9 if kill else 15
-        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"python3 {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
+        cmd="%s -k %d --nogen -p 1 -n 1 --nodeos-log-path %s" % (f"{sys.executable} {str(self.launcherPath)}", signalNum, self.nodeosLogPath)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         if 0 != subprocess.call(cmd.split(), stdout=Utils.FNull):
             if not silent: Utils.Print("Launcher failed to shut down eos cluster.")
@@ -1693,7 +1696,6 @@ class Cluster(object):
                             waitToComplete:bool=False, abiFile=None, actionsData=None, actionsAuths=None):
         Utils.Print("Configure txn generators")
         node=self.getNode(nodeId)
-        p2pListenPort = self.getNodeP2pPort(nodeId)
         info = node.getInfo()
         chainId = info['chain_id']
         lib_id = info['last_irreversible_block_id']
@@ -1702,13 +1704,12 @@ class Cluster(object):
         tpsLimitPerGenerator=tpsPerGenerator
 
         self.preExistingFirstTrxFiles = glob.glob(f"{Utils.DataDir}/first_trx_*.txt")
-
-        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=targetTps, tpsLimitPerGenerator=tpsLimitPerGenerator)
+        connectionPairList = [f"{self.host}:{self.getNodeP2pPort(nodeId)}"]
+        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=targetTps, tpsLimitPerGenerator=tpsLimitPerGenerator, connectionPairList=connectionPairList)
         self.trxGenLauncher = TransactionGeneratorsLauncher(chainId=chainId, lastIrreversibleBlockId=lib_id,
                                                     contractOwnerAccount=contractOwnerAcctName, accts=','.join(map(str, acctNamesList)),
                                                     privateKeys=','.join(map(str, acctPrivKeysList)), trxGenDurationSec=durationSec, logDir=Utils.DataDir,
-                                                    abiFile=abiFile, actionsData=actionsData, actionsAuths=actionsAuths,
-                                                    peerEndpoint=self.host, port=p2pListenPort, tpsTrxGensConfig=tpsTrxGensConfig)
+                                                    abiFile=abiFile, actionsData=actionsData, actionsAuths=actionsAuths, tpsTrxGensConfig=tpsTrxGensConfig)
 
         Utils.Print("Launch txn generators and start generating/sending transactions")
         self.trxGenLauncher.launch(waitToComplete=waitToComplete)
@@ -1729,4 +1730,7 @@ class Cluster(object):
                 for line in f:
                     firstTrxs.append(line.rstrip('\n'))
         Utils.Print(f"first transactions: {firstTrxs}")
-        node.waitForTransactionsInBlock(firstTrxs)
+        status = node.waitForTransactionsInBlock(firstTrxs)
+        if status is None:
+            Utils.Print('ERROR: Failed to spin up transaction generators: never received first transactions')
+        return status

tests/TestHarness/Node.py

@@ -61,7 +61,7 @@ class Node(Transactions):
             self.fetchBlock = lambda blockNum: self.processUrllibRequest("trace_api", "get_block", {"block_num":blockNum}, silentErrors=False, exitOnError=True)
             self.fetchKeyCommand = lambda: "[transaction][transaction_header][ref_block_num]"
             self.fetchRefBlock = lambda trans: trans["transaction_header"]["ref_block_num"]
-            self.cleosLimit = "--time-limit 99999"
+            self.cleosLimit = "--time-limit 999"
             self.fetchHeadBlock = lambda node, headBlock: node.processUrllibRequest("chain", "get_block_info", {"block_num":headBlock}, silentErrors=False, exitOnError=True)
 
     def __str__(self):
@@ -484,6 +484,10 @@ class Node(Transactions):
 
     def scheduleSnapshot(self):
         return self.processUrllibRequest("producer", "schedule_snapshot")
+    
+    def scheduleSnapshotAt(self, sbn):
+        param = { "start_block_num": sbn, "end_block_num": sbn }
+        return self.processUrllibRequest("producer", "schedule_snapshot", param)
 
     # kill all existing nodeos in case lingering from previous test
     @staticmethod

tests/TestHarness/WalletMgr.py

@@ -12,9 +12,9 @@ from .testUtils import Utils
 Wallet=namedtuple("Wallet", "name password host port")
 # pylint: disable=too-many-instance-attributes
 class WalletMgr(object):
-    __walletLogOutFile=f"{Utils.TestLogRoot}/test_keosd_out.log"
-    __walletLogErrFile=f"{Utils.TestLogRoot}/test_keosd_err.log"
     __walletDataDir=f"{Utils.TestLogRoot}/test_wallet_0"
+    __walletLogOutFile=f"{__walletDataDir}/test_keosd_out.log"
+    __walletLogErrFile=f"{__walletDataDir}/test_keosd_err.log"
     __MaxPort=9999
 
     # pylint: disable=too-many-arguments
@@ -83,6 +83,9 @@ class WalletMgr(object):
         cmd="%s --data-dir %s --config-dir %s --unlock-timeout=999999 --http-server-address=%s:%d --http-max-response-time-ms 99999 --verbose-http-errors" % (
             Utils.EosWalletPath, WalletMgr.__walletDataDir, WalletMgr.__walletDataDir, self.host, self.port)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+        if not os.path.isdir(WalletMgr.__walletDataDir):
+            if Utils.Debug: Utils.Print(f"Creating dir {WalletMgr.__walletDataDir} in dir: {os.getcwd()}")
+            os.mkdir(WalletMgr.__walletDataDir)
         with open(WalletMgr.__walletLogOutFile, 'w') as sout, open(WalletMgr.__walletLogErrFile, 'w') as serr:
             popen=subprocess.Popen(cmd.split(), stdout=sout, stderr=serr)
             self.__walletPid=popen.pid
@@ -299,6 +302,5 @@ class WalletMgr(object):
 
     @staticmethod
     def cleanup():
-        dataDir=WalletMgr.__walletDataDir
-        if os.path.isdir(dataDir) and os.path.exists(dataDir):
+        if os.path.isdir(WalletMgr.__walletDataDir) and os.path.exists(WalletMgr.__walletDataDir):
             shutil.rmtree(WalletMgr.__walletDataDir)

tests/TestHarness/launch_transaction_generators.py

@@ -10,16 +10,19 @@ sys.path.append(harnessPath)
 
 from .testUtils import Utils
 from pathlib import Path
+from re import sub
 
 Print = Utils.Print
 
 class TpsTrxGensConfig:
 
-    def __init__(self, targetTps: int, tpsLimitPerGenerator: int):
+    def __init__(self, targetTps: int, tpsLimitPerGenerator: int, connectionPairList: list):
         self.targetTps: int = targetTps
         self.tpsLimitPerGenerator: int = tpsLimitPerGenerator
-
-        self.numGenerators = math.ceil(self.targetTps / self.tpsLimitPerGenerator)
+        self.connectionPairList = connectionPairList
+        self.numConnectionPairs = len(self.connectionPairList)
+        round_to_multiple = lambda num, multiple: math.ceil(num/multiple) * multiple
+        self.numGenerators = round_to_multiple(math.ceil(self.targetTps / self.tpsLimitPerGenerator), self.numConnectionPairs)
         self.initialTpsPerGenerator = math.floor(self.targetTps / self.numGenerators)
         self.modTps = self.targetTps % self.numGenerators
         self.cleanlyDivisible = self.modTps == 0
@@ -35,8 +38,7 @@ class TpsTrxGensConfig:
 class TransactionGeneratorsLauncher:
 
     def __init__(self, chainId: int, lastIrreversibleBlockId: int, contractOwnerAccount: str, accts: str, privateKeys: str, trxGenDurationSec: int, logDir: str,
-                 abiFile: Path, actionsData, actionsAuths,
-                 peerEndpoint: str, port: int, tpsTrxGensConfig: TpsTrxGensConfig):
+                 abiFile: Path, actionsData, actionsAuths, tpsTrxGensConfig: TpsTrxGensConfig):
         self.chainId = chainId
         self.lastIrreversibleBlockId = lastIrreversibleBlockId
         self.contractOwnerAccount  = contractOwnerAccount
@@ -46,14 +48,14 @@ class TransactionGeneratorsLauncher:
         self.tpsTrxGensConfig = tpsTrxGensConfig
         self.logDir = logDir
         self.abiFile = abiFile
-        self.actionsData=actionsData
-        self.actionsAuths=actionsAuths
-        self.peerEndpoint = peerEndpoint
-        self.port = port
+        self.actionsData = actionsData
+        self.actionsAuths = actionsAuths
 
     def launch(self, waitToComplete=True):
         self.subprocess_ret_codes = []
+        connectionPairIter = 0
         for id, targetTps in enumerate(self.tpsTrxGensConfig.targetTpsPerGenList):
+            connectionPair = self.tpsTrxGensConfig.connectionPairList[connectionPairIter].rsplit(":")
             popenStringList = [
                                 './tests/trx_generator/trx_generator',
                                 '--generator-id', f'{id}',
@@ -65,8 +67,8 @@ class TransactionGeneratorsLauncher:
                                 '--trx-gen-duration', f'{self.trxGenDurationSec}',
                                 '--target-tps', f'{targetTps}',
                                 '--log-dir', f'{self.logDir}',
-                                '--peer-endpoint', f'{self.peerEndpoint}',
-                                '--port', f'{self.port}']
+                                '--peer-endpoint', f'{connectionPair[0]}',
+                                '--port', f'{connectionPair[1]}']
             if self.abiFile is not None and self.actionsData is not None and self.actionsAuths is not None:
                 popenStringList.extend(['--abi-file', f'{self.abiFile}',
                                         '--actions-data', f'{self.actionsData}',
@@ -74,6 +76,7 @@ class TransactionGeneratorsLauncher:
             if Utils.Debug:
                 Print(f"Running trx_generator: {' '.join(popenStringList)}")
             self.subprocess_ret_codes.append(subprocess.Popen(popenStringList))
+            connectionPairIter = (connectionPairIter + 1) % len(self.tpsTrxGensConfig.connectionPairList)
         exitCodes=None
         if waitToComplete:
             exitCodes = [ret_code.wait() for ret_code in self.subprocess_ret_codes]
@@ -100,20 +103,20 @@ def parseArgs():
     parser.add_argument("abi_file", type=str, help="The path to the contract abi file to use for the supplied transaction action data")
     parser.add_argument("actions_data", type=str, help="The json actions data file or json actions data description string to use")
     parser.add_argument("actions_auths", type=str, help="The json actions auth file or json actions auths description string to use, containting authAcctName to activePrivateKey pairs.")
-    parser.add_argument("peer_endpoint", type=str, help="set the peer endpoint to send transactions to", default="127.0.0.1")
-    parser.add_argument("port", type=int, help="set the peer endpoint port to send transactions to", default=9876)
+    parser.add_argument("connection_pair_list", type=str, help="Comma separated list of endpoint:port combinations to send transactions to", default="localhost:9876")
     args = parser.parse_args()
     return args
 
 def main():
     args = parseArgs()
+    connectionPairList = sub('[\s+]', '', args.connection_pair_list)
+    connectionPairList = connectionPairList.rsplit(',')
 
     trxGenLauncher = TransactionGeneratorsLauncher(chainId=args.chain_id, lastIrreversibleBlockId=args.last_irreversible_block_id,
                                                    contractOwnerAccount=args.contract_owner_account, accts=args.accounts,
                                                    privateKeys=args.priv_keys, trxGenDurationSec=args.trx_gen_duration, logDir=args.log_dir,
                                                    abiFile=args.abi_file, actionsData=args.actions_data, actionsAuths=args.actions_auths,
-                                                   peerEndpoint=args.peer_endpoint, port=args.port,
-                                                   tpsTrxGensConfig=TpsTrxGensConfig(targetTps=args.target_tps, tpsLimitPerGenerator=args.tps_limit_per_generator))
+                                                   tpsTrxGensConfig=TpsTrxGensConfig(targetTps=args.target_tps, tpsLimitPerGenerator=args.tps_limit_per_generator, connectionPairList=connectionPairList))
 
 
     exit_codes = trxGenLauncher.launch()

tests/TestHarness/queries.py

@@ -118,10 +118,10 @@ class NodeosQueries:
         # could be a transaction response
         if cntxt.hasKey("processed"):
             cntxt.add("processed")
-            cntxt.add("action_traces")
-            cntxt.index(0)
             if not cntxt.isSectionNull("except"):
                 return "no_block"
+            cntxt.add("action_traces")
+            cntxt.index(0)
             return cntxt.add("block_num")
 
         # or what the trace api plugin returns
@@ -242,7 +242,7 @@ class NodeosQueries:
         assert(isinstance(transId, str))
         exitOnErrorForDelayed=not delayedRetry and exitOnError
         timeout=3
-        cmdDesc="get transaction_trace"
+        cmdDesc=self.fetchTransactionCommand()
         cmd="%s %s" % (cmdDesc, transId)
         msg="(transaction id=%s)" % (transId);
         for i in range(0,(int(60/timeout) - 1)):
@@ -295,8 +295,8 @@ class NodeosQueries:
         refBlockNum=None
         key=""
         try:
-            key="[transaction][transaction_header][ref_block_num]"
-            refBlockNum=trans["transaction_header"]["ref_block_num"]
+            key = self.fetchKeyCommand()
+            refBlockNum = self.fetchRefBlock(trans)
             refBlockNum=int(refBlockNum)+1
         except (TypeError, ValueError, KeyError) as _:
             Utils.Print("transaction%s not found. Transaction: %s" % (key, trans))
@@ -347,8 +347,8 @@ class NodeosQueries:
 
     def getTable(self, contract, scope, table, exitOnError=False):
         cmdDesc = "get table"
-        cmd="%s %s %s %s" % (cmdDesc, contract, scope, table)
-        msg="contract=%s, scope=%s, table=%s" % (contract, scope, table);
+        cmd=f"{cmdDesc} {self.cleosLimit} {contract} {scope} {table}"
+        msg=f"contract={contract}, scope={scope}, table={table}"
         return self.processCleosCmd(cmd, cmdDesc, exitOnError=exitOnError, exitMsg=msg)
 
     def getTableAccountBalance(self, contract, scope):
@@ -529,7 +529,7 @@ class NodeosQueries:
             return m.group(1)
         except subprocess.CalledProcessError as ex:
             end=time.perf_counter()
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             Utils.Print("ERROR: Exception during code hash retrieval.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
             return None
 
@@ -580,8 +580,9 @@ class NodeosQueries:
         except subprocess.CalledProcessError as ex:
             if not silentErrors:
                 end=time.perf_counter()
-                msg=ex.output.decode("utf-8")
-                errorMsg="Exception during \"%s\". Exception message: %s.  cmd Duration=%.3f sec. %s" % (cmdDesc, msg, end-start, exitMsg)
+                out=ex.output.decode("utf-8")
+                msg=ex.stderr.decode("utf-8")
+                errorMsg="Exception during \"%s\". Exception message: %s.  stdout: %s.  cmd Duration=%.3f sec. %s" % (cmdDesc, msg, out, end-start, exitMsg)
                 if exitOnError:
                     Utils.cmdError(errorMsg)
                     Utils.errorExit(errorMsg)
@@ -595,15 +596,18 @@ class NodeosQueries:
 
         return trans
 
-    def processUrllibRequest(self, resource, command, payload={}, silentErrors=False, exitOnError=False, exitMsg=None, returnType=ReturnType.json, endpoint=None):
+    def processUrllibRequest(self, resource, command, payload={}, silentErrors=False, exitOnError=False, exitMsg=None, returnType=ReturnType.json, method="POST", endpoint=None):
         if not endpoint:
             endpoint = self.endpointHttp
         cmd = f"{endpoint}/v1/{resource}/{command}"
-        req = urllib.request.Request(cmd, method="POST")
-        req.add_header('Content-Type', 'application/json')
-        data = payload
-        data = json.dumps(data)
-        data = data.encode()
+        req = urllib.request.Request(cmd, method=method)
+        if len(payload):
+            req.add_header('Content-Type', 'application/json')
+            data = payload
+            data = json.dumps(data)
+            data = data.encode()
+        else:
+            data = None
         if Utils.Debug: Utils.Print("cmd: %s %s" % (cmd, payload))
         rtn=None
         start=time.perf_counter()
@@ -639,6 +643,8 @@ class NodeosQueries:
                         rtn = ex.read()
                     else:
                         unhandledEnumType(returnType)
+            elif returnType==ReturnType.raw:
+                return ex.code
             else:
                 return None
 
@@ -771,4 +777,3 @@ class NodeosQueries:
     def getActivatedProtocolFeatures(self):
         latestBlockHeaderState = self.getLatestBlockHeaderState()
         return latestBlockHeaderState["activated_protocol_features"]["protocol_features"]
-

tests/TestHarness/transactions.py

@@ -107,7 +107,7 @@ class Transactions(NodeosQueries):
                 self.trackCmdTransaction(trans, reportStatus=reportStatus)
         except subprocess.CalledProcessError as ex:
             end=time.perf_counter()
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             Utils.Print("ERROR: Exception during funds transfer.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
             if exitOnError:
                 Utils.cmdError("could not transfer \"%s\" from %s to %s" % (amountStr, source, destination))
@@ -131,7 +131,7 @@ class Transactions(NodeosQueries):
                 Utils.Print("cmd Duration: %.3f sec" % (end-start))
         except subprocess.CalledProcessError as ex:
             end=time.perf_counter()
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             Utils.Print("ERROR: Exception during spawn of funds transfer.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
             if exitOnError:
                 Utils.cmdError("could not transfer \"%s\" from %s to %s" % (amountStr, source, destination))
@@ -158,15 +158,15 @@ class Transactions(NodeosQueries):
         except subprocess.CalledProcessError as ex:
             if not shouldFail:
                 end=time.perf_counter()
-                msg=ex.output.decode("utf-8")
-                Utils.Print("ERROR: Exception during set contract.  cmd Duration: %.3f sec.  %s" % (end-start, msg))
+                out=ex.output.decode("utf-8")
+                msg=ex.stderr.decode("utf-8")
+                Utils.Print("ERROR: Exception during set contract. stderr: %s.  stdout: %s.  cmd Duration: %.3f sec." % (msg, out, end-start))
                 return None
             else:
                 retMap={}
                 retMap["returncode"]=ex.returncode
                 retMap["cmd"]=ex.cmd
                 retMap["output"]=ex.output
-                retMap["stdout"]=ex.stdout
                 retMap["stderr"]=ex.stderr
                 return retMap
 
@@ -213,7 +213,7 @@ class Transactions(NodeosQueries):
                 Utils.Print("cmd Duration: %.3f sec" % (end-start))
             return (NodeosQueries.getTransStatus(retTrans) == 'executed', retTrans)
         except subprocess.CalledProcessError as ex:
-            msg=ex.output.decode("utf-8")
+            msg=ex.stderr.decode("utf-8")
             if not silentErrors:
                 end=time.perf_counter()
                 Utils.Print("ERROR: Exception during push transaction.  cmd Duration=%.3f sec.  %s" % (end - start, msg))
@@ -245,7 +245,6 @@ class Transactions(NodeosQueries):
         except subprocess.CalledProcessError as ex:
             msg=ex.stderr.decode("utf-8")
             output=ex.output.decode("utf-8")
-            msg=ex.output.decode("utf-8")
             if not silentErrors:
                 end=time.perf_counter()
                 Utils.Print("ERROR: Exception during push message. stderr: %s. stdout: %s.  cmd Duration=%.3f sec." % (msg, output, end - start))

tests/auto_bp_peering_test.py

@@ -35,8 +35,7 @@ args = TestHelper.parse_args({
 Utils.Debug = args.v
 killAll = args.clean_run
 dumpErrorDetails = args.dump_error_details
-# dontKill = args.leave_running
-dontKill = True
+dontKill = args.leave_running
 killEosInstances = not dontKill
 killWallet = not dontKill
 keepLogs = args.keep_logs

tests/cli_test.py

@@ -11,7 +11,7 @@ import time
 import shutil
 import signal
 
-from TestHarness import Account, Cluster, Node, ReturnType, Utils, WalletMgr
+from TestHarness import Account, Node, ReturnType, Utils, WalletMgr
 
 testSuccessful=False
 
@@ -352,7 +352,7 @@ def abi_file_with_nodeos_test():
         os.makedirs(data_dir, exist_ok=True)
         walletMgr = WalletMgr(True)
         walletMgr.launch()
-        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
+        node = Node('localhost', 8888, nodeId, cmd="./programs/nodeos/nodeos -e -p eosio --plugin eosio::trace_api_plugin --trace-no-abis --plugin eosio::producer_plugin --plugin eosio::producer_api_plugin --plugin eosio::chain_api_plugin --plugin eosio::chain_plugin --plugin eosio::http_plugin --access-control-allow-origin=* --http-validate-host=false --max-transaction-time=-1 --resource-monitor-not-shutdown-on-threshold-exceeded " + "--data-dir " + data_dir + " --config-dir " + data_dir, walletMgr=walletMgr)
         node.verifyAlive() # Setting node state to not alive
         node.relaunch(newChain=True, cachePopen=True)
         node.waitForBlock(1)
@@ -403,7 +403,7 @@ def abi_file_with_nodeos_test():
                     os.kill(node.pid, signal.SIGKILL)
         if testSuccessful:
             Utils.Print("Cleanup nodeos data.")
-            shutil.rmtree(data_dir)
+            shutil.rmtree(Utils.DataPath)
 
         if malicious_token_abi_path:
             if os.path.exists(malicious_token_abi_path):

tests/distributed-transactions-test.py

@@ -3,6 +3,7 @@
 import random
 
 from TestHarness import Cluster, TestHelper, Utils, WalletMgr
+from TestHarness.TestHelper import AppArgs
 
 ###############################################################
 # distributed-transactions-test
@@ -19,8 +20,10 @@ from TestHarness import Cluster, TestHelper, Utils, WalletMgr
 Print=Utils.Print
 errorExit=Utils.errorExit
 
-args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file","--seed"
-                           ,"--dump-error-details","-v","--leave-running","--clean-run","--keep-logs","--unshared"})
+appArgs = AppArgs()
+extraArgs = appArgs.add_bool(flag="--speculative", help="Run nodes in read-mode=speculative")
+args=TestHelper.parse_args({"-p","-n","-d","-s","--nodes-file","--seed", "--speculative"
+                           ,"--dump-error-details","-v","--leave-running","--clean-run","--keep-logs","--unshared"}, applicationSpecificArgs=appArgs)
 
 pnodes=args.p
 topo=args.s
@@ -34,6 +37,7 @@ dontKill=args.leave_running
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
+speculative=args.speculative
 
 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -71,7 +75,11 @@ try:
                (pnodes, total_nodes-pnodes, topo, delay))
 
         Print("Stand up cluster")
-        if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay) is False:
+        extraNodeosArgs = ""
+        if speculative:
+           extraNodeosArgs = " --read-mode speculative "
+
+        if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, extraNodeosArgs=extraNodeosArgs) is False:
             errorExit("Failed to stand up eos cluster.")
 
         Print ("Wait for Cluster stabilization")

tests/http_plugin_test.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 
 from TestHarness import Cluster, Node, ReturnType, TestHelper, Utils
+import urllib.request
+import sys
 
 ###############################################################
 # http_plugin_tests.py
@@ -22,7 +24,7 @@ dumpErrorDetails = dumpErrorDetails=args.dump_error_details
 
 
 Utils.Debug=debug
-cluster=Cluster(walletd=True,unshared=args.unshared)
+cluster=Cluster(host="127.0.0.1",walletd=True,unshared=args.unshared)
 
 testSuccessful=False
 
@@ -34,17 +36,34 @@ try:
     TestHelper.printSystemInfo("BEGIN")
 
     Print("Stand up cluster")
-
-    if cluster.launch(dontBootstrap=True, loadSystemContract=False) is False:
+    node0_extra_config = "--http-validate-host true --http-server-address 127.0.0.1:8888"
+    if cluster.launch(dontBootstrap=True, loadSystemContract=False, specificExtraNodeosArgs = {0: node0_extra_config}) is False:
         cmdError("launcher")
         errorExit("Failed to stand up eos cluster.")
-
     Print("Getting cluster info")
     cluster.getInfos()
+
+    node0 = cluster.nodes[0]
+    ## HTTP plugin listens to 127.0.0.1:8888 by default. With the --http-validate-host=true,
+    ## the HTTP request to "http://localhost:8888" should fail because the HOST header doesn't
+    ## match "127.0.0.1".
+
+    def get_info_status(url):
+        try:
+            req = urllib.request.Request( url, method = "GET")
+            return urllib.request.urlopen(req, data=None).code
+        except urllib.error.HTTPError as response:
+            return response.code
+        except:
+            e = sys.exc_info()[0]
+            return e
+    url = node0.endpointHttp.replace("127.0.0.1", "localhost") + "/v1/chain/get_info"
+    code = get_info_status(url)
+    assert code == 400, f"Expected HTTP returned code 400, got {code}"
     testSuccessful = True
+
 finally:
     TestHelper.shutdown(cluster, None, testSuccessful, killEosInstances, True, keepLogs, killAll, dumpErrorDetails)
 
 exitCode = 0 if testSuccessful else 1
 exit(exitCode)
-        

tests/launcher.py

@@ -174,7 +174,7 @@ class launcher(object):
         parser.add_argument('-i', '--timestamp', help='set the timestamp for the first block.  Use "now" to indicate the current time')
         parser.add_argument('-l', '--launch', choices=['all', 'none', 'local'], help='select a subset of nodes to launch.  If not set, the default is to launch all unless an output file is named, in which case none are started.', default='all')
         parser.add_argument('-o', '--output', help='save a copy of the generated topology in this file and exit without launching', dest='topology_filename')
-        parser.add_argument('-k', '--kill', help='retrieve the list of previously started process ids and issue a kill to each')
+        parser.add_argument('-k', '--kill', type=int, help='kill a network as specified in arguments with given signal')
         parser.add_argument('--down', type=comma_separated, help='comma-separated list of node numbers that will be shut down', default=[])
         parser.add_argument('--bounce', type=comma_separated, help='comma-separated list of node numbers that will be restarted', default=[])
         parser.add_argument('--roll', type=comma_separated, help='comma-separated list of host names where the nodes will be rolled to a new version')
@@ -613,6 +613,17 @@ plugin = eosio::chain_api_plugin
                     if relaunch:
                         self.launch(node)
 
+    def kill(self, signum):
+        errorCode = 0
+        for node in self.network.nodes.values():
+            try:
+                with open(node.data_dir_name / f'{Utils.EosServerName}.pid', 'r') as f:
+                    pid = int(f.readline())
+                    self.terminate_wait_pid(pid, signum, raise_if_missing=False)
+            except FileNotFoundError as err:
+                errorCode = 1
+        return errorCode
+
     def start_all(self):
         if self.args.launch.lower() != 'none':
             for instance in self.network.nodes.values():
@@ -628,8 +639,9 @@ plugin = eosio::chain_api_plugin
                     f.write(f'"{node.dot_label}"->"{pname}" [dir="forward"];\n')
             f.write('}')
 
-    def terminate_wait_pid(self, pid, raise_if_missing=True):
-        '''Terminate a non-child process with SIGTERM and wait for it to exit.'''
+    def terminate_wait_pid(self, pid, signum = signal.SIGTERM, raise_if_missing=True):
+        '''Terminate a non-child process with given signal number or with SIGTERM if not
+           provided and wait for it to exit.'''
         if sys.version_info >= (3, 9) and platform.system() == 'Linux': # on our supported platforms, Python 3.9 accompanies a kernel > 5.3
             try:
                 fd = os.pidfd_open(pid)
@@ -640,7 +652,7 @@ plugin = eosio::chain_api_plugin
                 po = select.poll()
                 po.register(fd, select.POLLIN)
                 try:
-                    os.kill(pid, signal.SIGTERM)
+                    os.kill(pid, signum)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -663,7 +675,7 @@ plugin = eosio::chain_api_plugin
                     return min(delay * 2, 0.04)
                 delay = 0.0001
                 try:
-                    os.kill(pid, signal.SIGTERM)
+                    os.kill(pid, signum)
                 except ProcessLookupError:
                     if raise_if_missing:
                         raise
@@ -676,12 +688,16 @@ plugin = eosio::chain_api_plugin
                         return
 
 if __name__ == '__main__':
+    errorCode = 0
     l = launcher(sys.argv[1:])
     if len(l.args.down):
         l.down(l.args.down)
     elif len(l.args.bounce):
         l.bounce(l.args.bounce)
+    elif l.args.kill:
+        errorCode = l.kill(l.args.kill)
     elif l.args.launch == 'all' or l.args.launch == 'local':
         l.start_all()
     for f in glob.glob(Utils.DataPath):
         shutil.rmtree(f)
+    sys.exit(errorCode)

tests/nodeos_contrl_c_test.py

@@ -115,7 +115,7 @@ try:
         errorExit("Failed to kill the seed node")
 
 finally:
-    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=True, cleanRun=True, dumpErrorDetails=True)
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=True, killWallet=True, keepLogs=False, cleanRun=True, dumpErrorDetails=True)
 
 errorCode = 0 if testSuccessful else 1
 exit(errorCode)

tests/nodeos_forked_chain_test.py

@@ -5,8 +5,10 @@ from datetime import timedelta
 import time
 import json
 import signal
+import os
 
 from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr, CORE_SYMBOL
+from TestHarness.TestHelper import AppArgs
 
 ###############################################################
 # nodeos_forked_chain_test
@@ -27,6 +29,8 @@ from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr, CORE_SYMBOL
 #   Time is allowed to progress so that the "bridge" node can catchup and both producer nodes to come to consensus
 #   The block log is then checked for both producer nodes to verify that the 10 producer fork is selected and that
 #       both nodes are in agreement on the block log.
+#   This test also runs a state_history_plugin (SHiP) on node 0 and uses ship_streamer to verify all blocks are received
+#   across the fork.
 #
 ###############################################################
 
@@ -121,9 +125,10 @@ def getMinHeadAndLib(prodNodes):
     return (headBlockNum, libNum)
 
 
-
+appArgs = AppArgs()
+extraArgs = appArgs.add(flag="--num-ship-clients", type=int, help="How many ship_streamers should be started", default=2)
 args = TestHelper.parse_args({"--prod-count","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run",
-                              "--wallet-port","--unshared"})
+                              "--wallet-port","--unshared"}, applicationSpecificArgs=appArgs)
 Utils.Debug=args.v
 totalProducerNodes=2
 totalNonProducerNodes=1
@@ -137,6 +142,7 @@ dontKill=args.leave_running
 prodCount=args.prod_count
 killAll=args.clean_run
 walletPort=args.wallet_port
+num_clients=args.num_ship_clients
 
 walletMgr=WalletMgr(True, port=walletPort)
 testSuccessful=False
@@ -154,6 +160,9 @@ try:
     cluster.cleanup()
     Print("Stand up cluster")
     specificExtraNodeosArgs={}
+    shipNodeNum = 0
+    specificExtraNodeosArgs[shipNodeNum]="--plugin eosio::state_history_plugin --disable-replay-opts"
+
     # producer nodes will be mapped to 0 through totalProducerNodes-1, so the number totalProducerNodes will be the non-producing node
     specificExtraNodeosArgs[totalProducerNodes]="--plugin eosio::test_control_api_plugin"
 
@@ -286,6 +295,31 @@ try:
     timestampStr=Node.getBlockAttribute(block, "timestamp", blockNum)
     timestamp=datetime.strptime(timestampStr, Utils.TimeFmt)
 
+    shipNode = cluster.getNode(0)
+    block_range = 1000
+    start_block_num = blockNum
+    end_block_num = start_block_num + block_range
+
+    shipClient = "tests/ship_streamer"
+    cmd = f"{shipClient} --start-block-num {start_block_num} --end-block-num {end_block_num} --fetch-block --fetch-traces --fetch-deltas"
+    if Utils.Debug: Utils.Print(f"cmd: {cmd}")
+    clients = []
+    files = []
+    shipTempDir = os.path.join(Utils.DataDir, "ship")
+    os.makedirs(shipTempDir, exist_ok = True)
+    shipClientFilePrefix = os.path.join(shipTempDir, "client")
+
+    starts = []
+    for i in range(0, num_clients):
+        start = time.perf_counter()
+        outFile = open(f"{shipClientFilePrefix}{i}.out", "w")
+        errFile = open(f"{shipClientFilePrefix}{i}.err", "w")
+        Print(f"Start client {i}")
+        popen=Utils.delayedCheckOutput(cmd, stdout=outFile, stderr=errFile)
+        starts.append(time.perf_counter())
+        clients.append((popen, cmd))
+        files.append((outFile, errFile))
+        Utils.Print(f"Client {i} started, Ship node head is: {shipNode.getBlockNum()}")
 
     # ***   Identify what the production cycle is   ***
 
@@ -559,6 +593,25 @@ try:
         Utils.errorExit("Did not find find block %s (the original divergent block) in blockProducers0, test setup is wrong.  blockProducers0: %s" % (killBlockNum, ", ".join(blockProducers0)))
     Print("Fork resolved and determined producer %s for block %s" % (resolvedKillBlockProducer, killBlockNum))
 
+    Print(f"Stopping all {num_clients} clients")
+    for index, (popen, _), (out, err), start in zip(range(len(clients)), clients, files, starts):
+        popen.wait()
+        Print(f"Stopped client {index}.  Ran for {time.perf_counter() - start:.3f} seconds.")
+        out.close()
+        err.close()
+        outFile = open(f"{shipClientFilePrefix}{index}.out", "r")
+        data = json.load(outFile)
+        block_num = start_block_num
+        for i in data:
+            # fork can cause block numbers to be repeated
+            this_block_num = i['get_blocks_result_v0']['this_block']['block_num']
+            if this_block_num < block_num:
+                block_num = this_block_num
+            assert block_num == this_block_num, f"{block_num} != {this_block_num}"
+            assert isinstance(i['get_blocks_result_v0']['block'], str) # verify block in result
+            block_num += 1
+        assert block_num-1 == end_block_num, f"{block_num-1} != {end_block_num}"
+
     blockProducers0=[]
     blockProducers1=[]
 

tests/nodeos_irreversible_mode_test.py

@@ -21,7 +21,7 @@ errorExit = Utils.errorExit
 cmdError = Utils.cmdError
 relaunchTimeout = 30
 numOfProducers = 4
-totalNodes = 10
+totalNodes = 20
 
 # Parse command line arguments
 args = TestHelper.parse_args({"-v","--clean-run","--dump-error-details","--leave-running","--keep-logs","--unshared"})
@@ -32,6 +32,8 @@ dontKill=args.leave_running
 killEosInstances=not dontKill
 killWallet=not dontKill
 keepLogs=args.keep_logs
+speculativeReadMode="head"
+blockLogRetainBlocks="10000"
 
 # Setup cluster and it's wallet manager
 walletMgr=WalletMgr(True)
@@ -174,7 +176,18 @@ try:
          0:"--enable-stale-production",
          4:"--read-mode irreversible",
          6:"--read-mode irreversible",
-         9:"--plugin eosio::producer_api_plugin"})
+         9:"--plugin eosio::producer_api_plugin",
+        10:"--read-mode speculative",
+        11:"--read-mode irreversible",
+        12:"--read-mode speculative",
+        13:"--read-mode irreversible",
+        14:"--read-mode speculative --plugin eosio::producer_api_plugin",
+        15:"--read-mode speculative",
+        16:"--read-mode irreversible",
+        17:"--read-mode speculative",
+        18:"--read-mode irreversible",
+        19:"--read-mode speculative --plugin eosio::producer_api_plugin"
+        })
 
    producingNodeId = 0
    producingNode = cluster.getNode(producingNodeId)
@@ -254,7 +267,7 @@ try:
 
       # Kill and relaunch in irreversible mode
       nodeToTest.kill(signal.SIGTERM)
-      relaunchNode(nodeToTest, chainArg=" --read-mode irreversible")
+      relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible", "--block-log-retain-blocks":blockLogRetainBlocks})
 
       # Ensure the node condition is as expected after relaunch
       confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode)
@@ -267,7 +280,7 @@ try:
 
       # Kill and relaunch in speculative mode
       nodeToTest.kill(signal.SIGTERM)
-      relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "head"})
+      relaunchNode(nodeToTest, addSwapFlags={"--read-mode": speculativeReadMode, "--block-log-retain-blocks":blockLogRetainBlocks})
 
       # Ensure the node condition is as expected after relaunch
       confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest, headLibAndForkDbHeadBeforeSwitchMode)
@@ -283,7 +296,7 @@ try:
          # Kill and relaunch in irreversible mode
          nodeToTest.kill(signal.SIGTERM)
          waitForBlksProducedAndLibAdvanced() # Wait for some blks to be produced and lib advance
-         relaunchNode(nodeToTest, chainArg=" --read-mode irreversible")
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible", "--block-log-retain-blocks":blockLogRetainBlocks})
 
          # Ensure the node condition is as expected after relaunch
          ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest)
@@ -302,7 +315,7 @@ try:
          # Kill and relaunch in irreversible mode
          nodeToTest.kill(signal.SIGTERM)
          waitForBlksProducedAndLibAdvanced() # Wait for some blks to be produced and lib advance)
-         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "head"})
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": speculativeReadMode, "--block-log-retain-blocks":blockLogRetainBlocks})
 
          # Ensure the node condition is as expected after relaunch
          ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest)
@@ -360,7 +373,7 @@ try:
          backupBlksDir(nodeIdOfNodeToTest)
 
          # Relaunch in irreversible mode and create the snapshot
-         relaunchNode(nodeToTest, chainArg=" --read-mode irreversible")
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible"})
          confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest)
          nodeToTest.createSnapshot()
          nodeToTest.kill(signal.SIGTERM)
@@ -368,7 +381,7 @@ try:
          # Start from clean data dir, recover back up blocks, and then relaunch with irreversible snapshot
          removeState(nodeIdOfNodeToTest)
          recoverBackedupBlksDir(nodeIdOfNodeToTest) # this function will delete the existing blocks dir first
-         relaunchNode(nodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": "head"})
+         relaunchNode(nodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": speculativeReadMode})
          confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest)
          # Ensure it automatically replays "reversible blocks", i.e. head lib and fork db should be the same
          headLibAndForkDbHeadAfterRelaunch = getHeadLibAndForkDbHead(nodeToTest)
@@ -394,6 +407,50 @@ try:
       finally:
          stopProdNode()
 
+   # 10th test case: Load an irreversible snapshot into a node running without a block log
+   # Expectation: Node launches successfully
+   #              and the head and lib should be advancing after some blocks produced
+   def switchToNoBlockLogWithIrrModeSnapshot(nodeIdOfNodeToTest, nodeToTest):
+      try:
+         # Kill node and backup blocks directory of speculative mode
+         headLibAndForkDbHeadBeforeShutdown = getHeadLibAndForkDbHead(nodeToTest)
+         nodeToTest.kill(signal.SIGTERM)
+
+         # Relaunch in irreversible mode and create the snapshot
+         relaunchNode(nodeToTest, addSwapFlags={"--read-mode": "irreversible", "--block-log-retain-blocks":"0"})
+         confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest)
+         nodeToTest.createSnapshot()
+         nodeToTest.kill(signal.SIGTERM)
+
+         # Start from clean data dir and then relaunch with irreversible snapshot, no block log means that fork_db will be reset
+         removeState(nodeIdOfNodeToTest)
+         relaunchNode(nodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addSwapFlags={"--read-mode": speculativeReadMode, "--block-log-retain-blocks":"0"})
+         confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest)
+         # Ensure it does not replay "reversible blocks", i.e. head and lib should be different
+         headLibAndForkDbHeadAfterRelaunch = getHeadLibAndForkDbHead(nodeToTest)
+         assert headLibAndForkDbHeadBeforeShutdown != headLibAndForkDbHeadAfterRelaunch, \
+            "1: Head, Lib, and Fork Db same after relaunch {} vs {}".format(headLibAndForkDbHeadBeforeShutdown, headLibAndForkDbHeadAfterRelaunch)
+
+         # Start production and wait until lib advance, ensure everything is alright
+         startProdNode()
+         ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest)
+
+         # Note the head, lib and fork db head
+         stopProdNode()
+         headLibAndForkDbHeadBeforeShutdown = getHeadLibAndForkDbHead(nodeToTest)
+         nodeToTest.kill(signal.SIGTERM)
+
+         # Relaunch the node again (using the same snapshot)
+         # The end result should be the same as before shutdown
+         removeState(nodeIdOfNodeToTest)
+         relaunchNode(nodeToTest)
+         headLibAndForkDbHeadAfterRelaunch2 = getHeadLibAndForkDbHead(nodeToTest)
+         assert headLibAndForkDbHeadAfterRelaunch == headLibAndForkDbHeadAfterRelaunch2, \
+            "2: Head, Lib, and Fork Db after relaunch is different {} vs {}".format(headLibAndForkDbHeadAfterRelaunch, headLibAndForkDbHeadAfterRelaunch2)
+      finally:
+         stopProdNode()
+
+
    # Start executing test cases here
    testSuccessful = executeTest(1, replayInIrrModeWithRevBlks)
    testSuccessful = testSuccessful and executeTest(2, replayInIrrModeWithoutRevBlks)
@@ -405,6 +462,23 @@ try:
    testSuccessful = testSuccessful and executeTest(8, replayInIrrModeWithoutRevBlksAndConnectedToProdNode)
    testSuccessful = testSuccessful and executeTest(9, switchToSpecModeWithIrrModeSnapshot)
 
+   # retest with read-mode speculative instead of head
+   speculativeReadMode="speculative"
+   testSuccessful = testSuccessful and executeTest(10, switchSpecToIrrMode)
+   testSuccessful = testSuccessful and executeTest(11, switchIrrToSpecMode)
+   testSuccessful = testSuccessful and executeTest(12, switchSpecToIrrModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(13, switchIrrToSpecModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(14, switchToSpecModeWithIrrModeSnapshot)
+
+   # retest with read-mode head and no block log
+   speculativeReadMode="head"
+   blockLogRetainBlocks="0"
+   testSuccessful = testSuccessful and executeTest(15, switchSpecToIrrMode)
+   testSuccessful = testSuccessful and executeTest(16, switchIrrToSpecMode)
+   testSuccessful = testSuccessful and executeTest(17, switchSpecToIrrModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(18, switchIrrToSpecModeWithConnectedToProdNode)
+   testSuccessful = testSuccessful and executeTest(19, switchToNoBlockLogWithIrrModeSnapshot)
+
 finally:
    TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails)
    # Print test result

tests/nodeos_read_terminate_at_block_test.py

@@ -191,7 +191,7 @@ try:
         0 : "--enable-stale-production",
         1 : "--read-mode irreversible --terminate-at-block 75",
         2 : "--read-mode head --terminate-at-block 100",
-        3 : "--read-mode head --terminate-at-block 125"
+        3 : "--read-mode speculative --terminate-at-block 125"
     }
 
     # Kill any existing instances and launch cluster

tests/nodeos_snapshot_diff_test.py

@@ -126,13 +126,15 @@ try:
     waitForBlock(node0, blockNum, blockType=BlockType.lib)
 
     Print("Configure and launch txn generators")
+    
     targetTpsPerGenerator = 10
     testTrxGenDurationSec=60*30
     cluster.launchTrxGenerators(contractOwnerAcctName=cluster.eosioAccount.name, acctNamesList=[account1Name, account2Name],
                                 acctPrivKeysList=[account1PrivKey,account2PrivKey], nodeId=snapshotNodeId, tpsPerGenerator=targetTpsPerGenerator,
                                 numGenerators=trxGeneratorCnt, durationSec=testTrxGenDurationSec, waitToComplete=False)
 
-    cluster.waitForTrxGeneratorsSpinup(nodeId=snapshotNodeId, numGenerators=trxGeneratorCnt)
+    status = cluster.waitForTrxGeneratorsSpinup(nodeId=snapshotNodeId, numGenerators=trxGeneratorCnt)
+    assert status is not None, "ERROR: Failed to spinup Transaction Generators"
 
     blockNum=node0.getBlockNum(BlockType.head)
     timePerBlock=500
@@ -156,11 +158,8 @@ try:
     minReqPctLeeway=0.60
     minRequiredTransactions=minReqPctLeeway*transactionsPerBlock
     assert steadyStateAvg>=minRequiredTransactions, "Expected to at least receive %s transactions per block, but only getting %s" % (minRequiredTransactions, steadyStateAvg)
-
-    Print("Create snapshot")
-    ret = nodeProg.scheduleSnapshot()
-    assert ret is not None, "Snapshot creation failed"
     
+    Print("Create snapshot (node 0)")
     ret = nodeSnap.createSnapshot()
     assert ret is not None, "Snapshot creation failed"
     ret_head_block_num = ret["payload"]["head_block_num"]
@@ -177,6 +176,29 @@ try:
     Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(snapshotFile), "snapshot to-json", silentErrors=False)
     snapshotFile = snapshotFile + ".json"
 
+    Print("Trim programmable blocklog to snapshot head block num and relaunch programmable node")
+    nodeProg.kill(signal.SIGTERM)
+    output=cluster.getBlockLog(progNodeId, blockLogAction=BlockLogAction.trim, first=0, last=ret_head_block_num, throwException=True)
+    removeState(progNodeId)
+    Utils.rmFromFile(Utils.getNodeConfigDir(progNodeId, "config.ini"), "p2p-peer-address")
+    isRelaunchSuccess = nodeProg.relaunch(chainArg="--replay", addSwapFlags={}, timeout=relaunchTimeout, cachePopen=True)
+    assert isRelaunchSuccess, "Failed to relaunch programmable node"
+
+    Print("Schedule snapshot (node 2)")
+    ret = nodeProg.scheduleSnapshotAt(ret_head_block_num)
+    assert ret is not None, "Snapshot scheduling failed"
+
+    Print("Wait for programmable node lib to advance")
+    waitForBlock(nodeProg, ret_head_block_num+1, blockType=BlockType.lib)
+
+    Print("Kill programmable node")
+    nodeProg.kill(signal.SIGTERM)
+
+    Print("Convert snapshot to JSON")
+    progSnapshotFile = getLatestSnapshot(progNodeId)
+    Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(progSnapshotFile), "snapshot to-json", silentErrors=False)
+    progSnapshotFile = progSnapshotFile + ".json"
+
     Print("Trim irreversible blocklog to snapshot head block num")
     nodeIrr.kill(signal.SIGTERM)
     output=cluster.getBlockLog(irrNodeId, blockLogAction=BlockLogAction.trim, first=0, last=ret_head_block_num, throwException=True)
@@ -204,15 +226,6 @@ try:
     irrSnapshotFile = irrSnapshotFile + ".json"
 
     assert Utils.compareFiles(snapshotFile, irrSnapshotFile), f"Snapshot files differ {snapshotFile} != {irrSnapshotFile}"
-  
-    Print("Kill programmable node")
-    nodeProg.kill(signal.SIGTERM)
-
-    Print("Convert snapshot to JSON")
-    progSnapshotFile = getLatestSnapshot(progNodeId)
-    Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(progSnapshotFile), "snapshot to-json", silentErrors=False)
-    progSnapshotFile = progSnapshotFile + ".json"
-
     assert Utils.compareFiles(progSnapshotFile, irrSnapshotFile), f"Snapshot files differ {progSnapshotFile} != {irrSnapshotFile}"
 
     testSuccessful=True

tests/nodeos_snapshot_forked_test.py

@@ -171,7 +171,7 @@ try:
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
     Print("Wait for LIB to move, which indicates prodC has forked out the branch")
-    assert prodC.waitForLibToAdvance(), \
+    assert prodC.waitForLibToAdvance(60), \
         "ERROR: Network did not reach consensus after bridge node was restarted."
  
     for prodNode in prodNodes:

tests/nodeos_startup_catchup.py

@@ -118,7 +118,8 @@ try:
                                 tpsPerGenerator=targetTpsPerGenerator, numGenerators=trxGeneratorCnt, durationSec=testTrxGenDurationSec,
                                 waitToComplete=False)
 
-    cluster.waitForTrxGeneratorsSpinup(nodeId=node0.nodeId, numGenerators=trxGeneratorCnt)
+    status = cluster.waitForTrxGeneratorsSpinup(nodeId=node0.nodeId, numGenerators=trxGeneratorCnt)
+    assert status is not None, "ERROR: Failed to spinup Transaction Generators"
 
     blockNum=head(node0)
     timePerBlock=500

tests/performance_tests/README.md

@@ -488,8 +488,7 @@ Performance Test Basic Single Test:
 * `abi_file`                    The path to the contract abi file to use for the supplied transaction action data
 * `actions_data`                The json actions data file or json actions data description string to use
 * `actions_auths`               The json actions auth file or json actions auths description string to use, containting authAcctName to activePrivateKey pairs.
-* `peer_endpoint`               set the peer endpoint to send transactions to, default="127.0.0.1"
-* `port`                        set the peer endpoint port to send transactions to, default=9876
+* `connection_pair_list`        Comma separated list of endpoint:port combinations to send transactions to
 </details>
 
 #### Transaction Generator

tests/performance_tests/performance_test_basic.py

@@ -327,7 +327,9 @@ class PerformanceTestBasic:
     def runTpsTest(self) -> PtbTpsTestResult:
         completedRun = False
         self.producerNode = self.cluster.getNode(self.producerNodeId)
-        self.producerP2pPort = self.cluster.getNodeP2pPort(self.producerNodeId)
+        self.connectionPairList = []
+        for producer in range(0, self.clusterConfig.pnodes):
+            self.connectionPairList.append(f"{self.cluster.getNode(producer).host}:{self.cluster.getNodeP2pPort(producer)}")
         self.validationNode = self.cluster.getNode(self.validationNodeId)
         self.wallet = self.walletMgr.create('default')
         self.setupContract()
@@ -371,13 +373,13 @@ class PerformanceTestBasic:
         self.cluster.biosNode.kill(signal.SIGTERM)
 
         self.data.startBlock = self.waitForEmptyBlocks(self.validationNode, self.emptyBlockGoal)
-        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=self.ptbConfig.targetTps, tpsLimitPerGenerator=self.ptbConfig.tpsLimitPerGenerator)
+        tpsTrxGensConfig = TpsTrxGensConfig(targetTps=self.ptbConfig.targetTps, tpsLimitPerGenerator=self.ptbConfig.tpsLimitPerGenerator, connectionPairList=self.connectionPairList)
 
         self.cluster.trxGenLauncher = TransactionGeneratorsLauncher(chainId=chainId, lastIrreversibleBlockId=lib_id, contractOwnerAccount=self.clusterConfig.specifiedContract.account.name,
                                                        accts=','.join(map(str, self.accountNames)), privateKeys=','.join(map(str, self.accountPrivKeys)),
                                                        trxGenDurationSec=self.ptbConfig.testTrxGenDurationSec, logDir=self.trxGenLogDirPath,
                                                        abiFile=abiFile, actionsData=actionsDataJson, actionsAuths=actionsAuthsJson,
-                                                       peerEndpoint=self.producerNode.host, port=self.producerP2pPort, tpsTrxGensConfig=tpsTrxGensConfig)
+                                                       tpsTrxGensConfig=tpsTrxGensConfig)
 
         trxGenExitCodes = self.cluster.trxGenLauncher.launch()
         print(f"Transaction Generator exit codes: {trxGenExitCodes}")
@@ -495,15 +497,17 @@ class PerformanceTestBasic:
             traceback.print_exc()
 
         finally:
+            # Despite keepLogs being hardcoded to False, logs will still appear on test failure in TestLogs
+            # due to testSuccessful being False
             TestHelper.shutdown(
-                self.cluster,
-                self.walletMgr,
-                testSuccessful,
-                self.testHelperConfig._killEosInstances,
-                self.testHelperConfig._killWallet,
-                self.testHelperConfig.keepLogs,
-                self.testHelperConfig.killAll,
-                self.testHelperConfig.dumpErrorDetails
+                cluster=self.cluster,
+                walletMgr=self.walletMgr,
+                testSuccessful=testSuccessful,
+                killEosInstances=self.testHelperConfig._killEosInstances,
+                killWallet=self.testHelperConfig._killWallet,
+                keepLogs=False,
+                cleanRun=self.testHelperConfig.killAll,
+                dumpErrorDetails=self.testHelperConfig.dumpErrorDetails
                 )
 
             if self.ptbConfig.delPerfLogs:

tests/plugin_http_api_test.py

@@ -40,8 +40,8 @@ class PluginHttpTest(unittest.TestCase):
         self.keosd.killall(True)
         WalletMgr.cleanup()
         Node.killAllNodeos()
-        if os.path.exists(self.data_dir):
-            shutil.rmtree(self.data_dir)
+        if os.path.exists(Utils.DataPath):
+            shutil.rmtree(Utils.DataPath)
         if os.path.exists(self.config_dir):
             shutil.rmtree(self.config_dir)
         time.sleep(self.sleep_s)
@@ -56,7 +56,7 @@ class PluginHttpTest(unittest.TestCase):
                         "http_plugin", "db_size_api_plugin", "prometheus_plugin"]
         nodeos_plugins = "--plugin eosio::" +  " --plugin eosio::".join(plugin_names)
         nodeos_flags = (" --data-dir=%s --config-dir=%s --trace-dir=%s --trace-no-abis --access-control-allow-origin=%s "
-                        "--contracts-console --http-validate-host=%s --verbose-http-errors --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
+                        "--contracts-console --http-validate-host=%s --verbose-http-errors --max-transaction-time -1 --abi-serializer-max-time-ms 30000 --http-max-response-time-ms 30000 "
                         "--p2p-peer-address localhost:9011 --resource-monitor-not-shutdown-on-threshold-exceeded ") % (self.data_dir, self.config_dir, self.data_dir, "\'*\'", "false")
         start_nodeos_cmd = ("%s -e -p eosio %s %s ") % (Utils.EosServerPath, nodeos_plugins, nodeos_flags)
         self.nodeos.launchCmd(start_nodeos_cmd, self.node_id)
@@ -600,6 +600,35 @@ class PluginHttpTest(unittest.TestCase):
         ret_json = self.nodeos.processUrllibRequest(resource, command, self.http_post_invalid_param)
         self.assertEqual(ret_json["code"], 400)
         self.assertEqual(ret_json["error"]["code"], 3200006)
+        # get_transaction_id with missing actions
+        payload_no_actions = {"expiration":"2020-08-01T07:15:49",
+                "ref_block_num": 34881,
+                "ref_block_prefix":2972818865,
+                "max_net_usage_words":0,
+                "max_cpu_usage_ms":0,
+                "delay_sec":0,
+                "context_free_actions":[],
+                "transaction_extensions": [],
+                "signatures": ["SIG_K1_KeqfqiZu1GwUxQb7jzK9Fdks6HFaVBQ9AJtCZZj56eG9qGgvVMVtx8EerBdnzrhFoX437sgwtojf2gfz6S516Ty7c22oEp"],
+                "context_free_data": []}
+        ret_json = self.nodeos.processUrllibRequest(resource, command, payload_no_actions)
+        self.assertEqual(ret_json["code"], 400)
+        self.assertEqual(ret_json["error"]["code"], 3200006)
+        # get_transaction_id with invalid actions
+        payload_invalid_actions = {"expiration":"2020-08-01T07:15:49",
+                "ref_block_num": 34881,
+                "ref_block_prefix":2972818865,
+                "max_net_usage_words":0,
+                "max_cpu_usage_ms":0,
+                "delay_sec":0,
+                "context_free_actions":[],
+                "actions":  "hello_world",
+                "transaction_extensions": [],
+                "signatures": ["SIG_K1_KeqfqiZu1GwUxQb7jzK9Fdks6HFaVBQ9AJtCZZj56eG9qGgvVMVtx8EerBdnzrhFoX437sgwtojf2gfz6S516Ty7c22oEp"],
+                "context_free_data": []}
+        ret_json = self.nodeos.processUrllibRequest(resource, command, payload_invalid_actions)
+        self.assertEqual(ret_json["code"], 400)
+        self.assertEqual(ret_json["error"]["code"], 3200006)
         # get_transaction_id with valid parameter
         payload = {"expiration":"2020-08-01T07:15:49",
                    "ref_block_num": 34881,
@@ -614,6 +643,23 @@ class PluginHttpTest(unittest.TestCase):
                    "context_free_data": []}
         ret_str = self.nodeos.processUrllibRequest(resource, command, payload, returnType=ReturnType.raw).decode('ascii')
         self.assertEqual(ret_str, "\"0be762a6406bab15530e87f21e02d1c58e77944ee55779a76f4112e3b65cac48\"")
+        # transaction that has hex_data
+        payload_hex = {"expiration":"2020-08-01T07:15:49",
+                "ref_block_num": 34881,
+                "ref_block_prefix":2972818865,
+                "max_net_usage_words":0,
+                "max_cpu_usage_ms":0,
+                "delay_sec":0,
+                "context_free_actions":[],
+                "actions":[{"account":"eosio.token","name": "transfer","authorization": [{"actor": "han","permission": "active"}],
+                "data": "{\"entry\":774831,\"miner\":\"eosminer1111\",\"nonce\":139429}\"}",
+                "hex_data": "000000000000a6690000000000ea305501000000000000000453595300000000016d"}],
+                "transaction_extensions": [],
+                "signatures": ["SIG_K1_KeqfqiZu1GwUxQb7jzK9Fdks6HFaVBQ9AJtCZZj56eG9qGgvVMVtx8EerBdnzrhFoX437sgwtojf2gfz6S516Ty7c22oEp"],
+                "context_free_data": []}
+        ret_str = self.nodeos.processUrllibRequest(resource, command, payload_hex, returnType=ReturnType.raw).decode('ascii')
+        self.assertEqual(ret_str, "\"0be762a6406bab15530e87f21e02d1c58e77944ee55779a76f4112e3b65cac48\"")
+
 
         # push_block with empty parameter
         command = "push_block"
@@ -1341,8 +1387,8 @@ class PluginHttpTest(unittest.TestCase):
     def test_prometheusApi(self) :
         resource = "prometheus"
         command = "metrics"
-
-        ret_text = self.nodeos.processUrllibRequest(resource, command, returnType = ReturnType.raw ).decode()
+        endpointPrometheus = f'http://{self.nodeos.host}:9101'
+        ret_text = self.nodeos.processUrllibRequest(resource, command, returnType = ReturnType.raw, method="GET", endpoint=endpointPrometheus).decode()
         # filter out all empty lines or lines starting with '#'
         data_lines = filter(lambda line: len(line) > 0 and line[0]!='#', ret_text.split('\n'))
         # converting each line into a key value pair and then construct a dictionay out of all the pairs
@@ -1352,6 +1398,8 @@ class PluginHttpTest(unittest.TestCase):
         self.assertTrue(int(metrics["blocks_produced"]) > 1)
         self.assertTrue(int(metrics["last_irreversible"]) > 1)
 
+        ret = self.nodeos.processUrllibRequest(resource, "m", returnType = ReturnType.raw, method="GET", silentErrors= True, endpoint=endpointPrometheus)
+        self.assertTrue(ret == 404)
 
     def test_multipleRequests(self):
         """Test keep-alive ability of HTTP plugin.  Handle multiple requests in a single session"""

tests/read_only_trx_test.py

@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
 
 import random
+import time
+import signal
+import threading
 
-from TestHarness import Account, Cluster, TestHelper, Utils, WalletMgr
+from TestHarness import Account, Cluster, ReturnType, TestHelper, Utils, WalletMgr
 from TestHarness.TestHelper import AppArgs
 
 ###############################################################
@@ -18,6 +21,7 @@ errorExit=Utils.errorExit
 
 appArgs=AppArgs()
 appArgs.add(flag="--read-only-threads", type=int, help="number of read-only threads", default=0)
+appArgs.add(flag="--num-test-runs", type=int, help="number of times to run the tests", default=1)
 appArgs.add_bool(flag="--eos-vm-oc-enable", help="enable eos-vm-oc")
 appArgs.add(flag="--wasm-runtime", type=str, help="if set to eos-vm-oc, must compile with EOSIO_EOS_VM_OC_DEVELOPER", default="eos-vm-jit")
 
@@ -29,6 +33,10 @@ pnodes=args.p
 topo=args.s
 delay=args.d
 total_nodes = pnodes if args.n < pnodes else args.n
+# For this test, we need at least 1 non-producer
+if total_nodes <= pnodes:
+    Print ("non-producing nodes %d must be greater than 0. Force it to %d. producing nodes: %d," % (total_nodes - pnodes, pnodes + 1, pnodes))
+    total_nodes = pnodes + 1
 debug=args.v
 nodesFile=args.nodes_file
 dontLaunch=nodesFile is not None
@@ -37,6 +45,7 @@ dontKill=args.leave_running
 dumpErrorDetails=args.dump_error_details
 killAll=args.clean_run
 keepLogs=args.keep_logs
+numTestRuns=args.num_test_runs
 
 killWallet=not dontKill
 killEosInstances=not dontKill
@@ -45,6 +54,7 @@ if nodesFile is not None:
 
 Utils.Debug=debug
 testSuccessful=False
+errorInThread=False
 
 random.seed(seed) # Use a fixed seed for repeatability.
 cluster=Cluster(walletd=True,unshared=args.unshared)
@@ -54,6 +64,9 @@ EOSIO_ACCT_PRIVATE_DEFAULT_KEY = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79z
 EOSIO_ACCT_PUBLIC_DEFAULT_KEY = "EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV"
 
 try:
+    TestHelper.printSystemInfo("BEGIN")
+    cluster.setWalletMgr(walletMgr)
+
     if dontLaunch: # run test against remote cluster
         jsonStr=None
         with open(nodesFile, "r") as f:
@@ -71,11 +84,24 @@ try:
             cluster.killall(allInstances=killAll)
             cluster.cleanup()
 
-    Print ("producing nodes: %s, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" % (pnodes, total_nodes-pnodes, topo, delay))
+    Print ("producing nodes: %d, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" % (pnodes, total_nodes-pnodes, topo, delay))
 
+    cluster.killall(allInstances=killAll)
+    cluster.cleanup()
     Print("Stand up cluster")
+    # set up read-only options for API node
+    specificExtraNodeosArgs={}
+    # producer nodes will be mapped to 0 through pnodes-1, so the number pnodes is the no-producing API node
+    specificExtraNodeosArgs[pnodes]=" --plugin eosio::net_api_plugin"
+    specificExtraNodeosArgs[pnodes]+=" --read-only-threads "
+    specificExtraNodeosArgs[pnodes]+=str(args.read_only_threads)
+    if args.eos_vm_oc_enable:
+        specificExtraNodeosArgs[pnodes]+=" --eos-vm-oc-enable"
+    if args.wasm_runtime:
+        specificExtraNodeosArgs[pnodes]+=" --wasm-runtime "
+        specificExtraNodeosArgs[pnodes]+=args.wasm_runtime
     extraNodeosArgs=" --http-max-response-time-ms 990000 --disable-subjective-api-billing false "
-    if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay,extraNodeosArgs=extraNodeosArgs ) is False:
+    if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, topo=topo, delay=delay, specificExtraNodeosArgs=specificExtraNodeosArgs, extraNodeosArgs=extraNodeosArgs ) is False:
         errorExit("Failed to stand up eos cluster.")
 
     Print ("Wait for Cluster stabilization")
@@ -97,7 +123,7 @@ try:
     userAccount = Account(userAccountName)
     userAccount.ownerPublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
     userAccount.activePublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
-    cluster.createAccountAndVerify(userAccount, cluster.eosioAccount)
+    cluster.createAccountAndVerify(userAccount, cluster.eosioAccount, stakeCPU=2000)
 
     noAuthTableContractDir="unittests/test-contracts/no_auth_table"
     noAuthTableWasmFile="no_auth_table.wasm"
@@ -116,28 +142,161 @@ try:
         }
         return apiNode.pushTransaction(trx, opts)
 
-    Print("Insert a user")
-    results = sendTransaction('insert', {"user": userAccountName, "id": 1, "age": 10}, auth=[{"actor": userAccountName, "permission":"active"}])
-    assert(results[0])
-    apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
+    def sendReadOnlyTrxOnThread(startId, numTrxs):
+        Print("start sendReadOnlyTrxOnThread")
 
-    # verify the return value (age) from read-only is the same as created.
-    Print("Send a read-only Get transaction to verify previous Insert")
-    results = sendTransaction('getage', {"user": userAccountName}, opts='--read')
-    assert(results[0])
-    assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 10)
+        global errorInThread
+        errorInThread = False
+        try:
+           for i in range(numTrxs):
+               results = sendTransaction('age', {"user": userAccountName, "id": startId + i}, opts='--read')
+               assert(results[0])
+               assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 25)
+        except Exception as e:
+            Print("Exception in sendReadOnlyTrxOnThread: ", e)
+            errorInThread = True
 
-    # verify non-read-only modification works
-    Print("Send a non-read-only Modify transaction")
-    results = sendTransaction('modify', {"user": userAccountName, "age": 25},        auth=[{"actor": userAccountName, "permission": "active"}])
-    assert(results[0])
-    apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
+    def sendTrxsOnThread(startId, numTrxs, opts=None):
+        Print("sendTrxsOnThread: ", startId, numTrxs, opts)
 
-    # verify 'cleos push action getage "user": "user" --read' works
-    Print("Send a read-only Get action")
-    results = apiNode.pushMessage(testAccountName, 'getage', "{{\"user\": \"{}\"}}".format(userAccountName), opts='--read');
-    assert(results[0])
-    assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 25)
+        global errorInThread
+        errorInThread = False
+        try:
+            for i in range(numTrxs):
+                results = sendTransaction('age', {"user": userAccountName, "id": startId + i}, auth=[{"actor": userAccountName, "permission":"active"}], opts=opts)
+                assert(results[0])
+        except Exception as e:
+            Print("Exception in sendTrxsOnThread: ", e)
+            errorInThread = True
+
+    def doRpc(resource, command, numRuns, fieldIn, expectedValue, code, payload={}):
+        global errorInThread
+        errorInThread = False
+        try:
+            for i in range(numRuns):
+                ret_json = apiNode.processUrllibRequest(resource, command, payload)
+                if code is None:
+                    assert(ret_json["payload"][fieldIn] is not None)
+                    if expectedValue is not None:
+                        assert(ret_json["payload"][fieldIn] == expectedValue)
+                else:
+                    assert(ret_json["code"] == code)
+        except Exception as e:
+            Print("Exception in doRpc: ", e)
+            errorInThread = True
+
+    def runReadOnlyTrxAndRpcInParallel(resource, command, fieldIn=None, expectedValue=None, code=None, payload={}):
+        Print("runReadOnlyTrxAndRpcInParallel: ", command)
+
+        numRuns = 10
+        trxThread = threading.Thread(target = sendReadOnlyTrxOnThread, args = (0, numRuns ))
+        rpcThread = threading.Thread(target = doRpc, args = (resource, command, numRuns, fieldIn, expectedValue, code, payload))
+        trxThread.start()
+        rpcThread.start()
+
+        trxThread.join()
+        rpcThread.join()
+        assert(not errorInThread)
+
+    def mixedOpsTest(opt=None):
+        Print("mixedOpsTest -- opt = ", opt)
+
+        numRuns = 300
+        readOnlyThread = threading.Thread(target = sendReadOnlyTrxOnThread, args = (0, numRuns ))
+        readOnlyThread.start()
+        sendTrxThread = threading.Thread(target = sendTrxsOnThread, args = (numRuns, numRuns, opt))
+        sendTrxThread.start()
+        pushBlockThread = threading.Thread(target = doRpc, args = ("chain", "push_block", numRuns, None, None, 202, {"block":"signed_block"}))
+        pushBlockThread.start()
+
+        readOnlyThread.join()
+        sendTrxThread.join()
+        pushBlockThread.join()
+        assert(not errorInThread)
+
+    def sendMulReadOnlyTrx(numThreads):
+        threadList = []
+        num_trxs_per_thread = 500
+        for i in range(numThreads):
+            thr = threading.Thread(target = sendReadOnlyTrxOnThread, args = (i * num_trxs_per_thread, num_trxs_per_thread ))
+            thr.start()
+            threadList.append(thr)
+        for thr in threadList:
+            thr.join()
+
+    def basicTests():
+        Print("Insert a user")
+        results = sendTransaction('insert', {"user": userAccountName, "id": 1, "age": 10}, auth=[{"actor": userAccountName, "permission":"active"}])
+        assert(results[0])
+        apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
+
+        # verify the return value (age) from read-only is the same as created.
+        Print("Send a read-only Get transaction to verify previous Insert")
+        results = sendTransaction('getage', {"user": userAccountName}, opts='--read')
+        assert(results[0])
+        assert(results[1]['processed']['action_traces'][0]['return_value_data'] == 10)
+
+        # verify non-read-only modification works
+        Print("Send a non-read-only Modify transaction")
+        results = sendTransaction('modify', {"user": userAccountName, "age": 25}, auth=[{"actor": userAccountName, "permission": "active"}])
+        assert(results[0])
+        apiNode.waitForTransactionInBlock(results[1]['transaction_id'])
+
+    def multiReadOnlyTests():
+        Print("Verify multiple read-only Get actions work after Modify")
+        sendMulReadOnlyTrx(numThreads=5)
+
+    def chainApiTests():
+        # verify chain APIs can run in parallel with read-ony transactions
+        runReadOnlyTrxAndRpcInParallel("chain", "get_info", "server_version")
+        runReadOnlyTrxAndRpcInParallel("chain", "get_consensus_parameters", "chain_config")
+        runReadOnlyTrxAndRpcInParallel("chain", "get_activated_protocol_features", "activated_protocol_features")
+        runReadOnlyTrxAndRpcInParallel("chain", "get_block", "block_num", expectedValue=1, payload={"block_num_or_id":1})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_block_info", "block_num", expectedValue=1, payload={"block_num":1})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_account", "account_name", expectedValue=userAccountName, payload = {"account_name":userAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_code", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_code_hash", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_raw_code_and_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_raw_abi", "account_name", expectedValue=testAccountName, payload = {"account_name":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_producers", "rows", payload = {"json":"true","lower_bound":""})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_table_rows", "rows", payload = {"json":"true","code":"eosio","scope":"eosio","table":"global"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_table_by_scope", fieldIn="rows", payload = {"json":"true","table":"noauth"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_currency_balance", code=200, payload = {"code":"eosio.token", "account":testAccountName})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_currency_stats", fieldIn="SYS", payload = {"code":"eosio.token", "symbol":"SYS"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_required_keys", code=400)
+        runReadOnlyTrxAndRpcInParallel("chain", "get_transaction_id", code=400, payload = {"ref_block_num":"1"})
+        runReadOnlyTrxAndRpcInParallel("chain", "push_block", code=202, payload = {"block":"signed_block"})
+        runReadOnlyTrxAndRpcInParallel("chain", "get_producer_schedule", "active")
+        runReadOnlyTrxAndRpcInParallel("chain", "get_scheduled_transactions", "transactions", payload = {"json":"true","lower_bound":""})
+
+    def netApiTests():
+        # NET APIs
+        runReadOnlyTrxAndRpcInParallel("net", "status", code=201, payload = "localhost")
+        runReadOnlyTrxAndRpcInParallel("net", "connections", code=201)
+        runReadOnlyTrxAndRpcInParallel("net", "connect", code=201, payload = "localhost")
+        runReadOnlyTrxAndRpcInParallel("net", "disconnect", code=201, payload = "localhost")
+
+    def runEverythingParallel():
+        threadList = []
+        threadList.append(threading.Thread(target = multiReadOnlyTests))
+        threadList.append(threading.Thread(target = chainApiTests))
+        threadList.append(threading.Thread(target = netApiTests))
+        threadList.append(threading.Thread(target = mixedOpsTest))
+        for thr in threadList:
+            thr.start()
+        for thr in threadList:
+            thr.join()
+
+    basicTests()
+
+    if args.read_only_threads > 0: # Save test time. No need to run other tests if multi-threaded is not enabled
+        for i in range(numTestRuns):
+            multiReadOnlyTests()
+            chainApiTests()
+            netApiTests()
+            mixedOpsTest()
+            runEverythingParallel()
 
     testSuccessful = True
 finally:

tests/ship_client.cpp

@@ -125,6 +125,8 @@ int main(int argc, char* argv[]) {
             eosio::check(result_doucment[1]["head"].IsObject(),                                 "'head' is not an object");
             eosio::check(result_doucment[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
             eosio::check(result_doucment[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
+            eosio::check(result_doucment[1]["head"].HasMember("block_id"),                      "'head' does not contain 'block_id'");
+            eosio::check(result_doucment[1]["head"]["block_id"].IsString(),                     "'head.block_id' isn't a string");
 
             uint32_t this_block_num = result_doucment[1]["head"]["block_num"].GetUint();
 

tests/ship_streamer.cpp

@@ -10,6 +10,8 @@
 #include <boost/beast.hpp>
 #include <boost/program_options.hpp>
 
+#include <map>
+#include <set>
 #include <iostream>
 #include <string>
 
@@ -111,45 +113,81 @@ int main(int argc, char* argv[]) {
       stream.write(boost::asio::buffer(request_type.json_to_bin(request_sb.GetString(), [](){})));
       stream.read_message_max(0);
 
+      // Each block_num can have multiple block_ids since forks are possible
+      //       block_num,         block_id
+      std::map<uint32_t, std::set<std::string>> block_ids;
       bool is_first = true;
       for(;;) {
          boost::beast::flat_buffer buffer;
          stream.read(buffer);
 
          eosio::input_stream is((const char*)buffer.data().data(), buffer.data().size());
-         rapidjson::Document result_doucment;
-         result_doucment.Parse(result_type.bin_to_json(is).c_str());
+         rapidjson::Document result_document;
+         result_document.Parse(result_type.bin_to_json(is).c_str());
 
-         eosio::check(!result_doucment.HasParseError(),                                      "Failed to parse result JSON from abieos");
-         eosio::check(result_doucment.IsArray(),                                             "result should have been an array (variant) but it's not");
-         eosio::check(result_doucment.Size() == 2,                                           "result was an array but did not contain 2 items like a variant should");
-         eosio::check(std::string(result_doucment[0].GetString()) == "get_blocks_result_v0", "result type doesn't look like get_blocks_result_v0");
-         eosio::check(result_doucment[1].IsObject(),                                         "second item in result array is not an object");
-         eosio::check(result_doucment[1].HasMember("head"),                                  "cannot find 'head' in result");
-         eosio::check(result_doucment[1]["head"].IsObject(),                                 "'head' is not an object");
-         eosio::check(result_doucment[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
-         eosio::check(result_doucment[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
-
-         uint32_t this_block_num = 0;
-         if( result_doucment[1].HasMember("this_block") && result_doucment[1]["this_block"].IsObject() ) {
-            if( result_doucment[1]["this_block"].HasMember("block_num") && result_doucment[1]["this_block"]["block_num"].IsUint() ) {
-               this_block_num = result_doucment[1]["this_block"]["block_num"].GetUint();
-            }
-         }
+         eosio::check(!result_document.HasParseError(),                                      "Failed to parse result JSON from abieos");
+         eosio::check(result_document.IsArray(),                                             "result should have been an array (variant) but it's not");
+         eosio::check(result_document.Size() == 2,                                           "result was an array but did not contain 2 items like a variant should");
+         eosio::check(std::string(result_document[0].GetString()) == "get_blocks_result_v0", "result type doesn't look like get_blocks_result_v0");
+         eosio::check(result_document[1].IsObject(),                                         "second item in result array is not an object");
+         eosio::check(result_document[1].HasMember("head"),                                  "cannot find 'head' in result");
+         eosio::check(result_document[1]["head"].IsObject(),                                 "'head' is not an object");
+         eosio::check(result_document[1]["head"].HasMember("block_num"),                     "'head' does not contain 'block_num'");
+         eosio::check(result_document[1]["head"]["block_num"].IsUint(),                      "'head.block_num' isn't a number");
+         eosio::check(result_document[1]["head"].HasMember("block_id"),                      "'head' does not contain 'block_id'");
+         eosio::check(result_document[1]["head"]["block_id"].IsString(),                     "'head.block_id' isn't a string");
 
+         // stream what was received
          if(is_first) {
-            std::cout << "[" << std::endl;
-            is_first = false;
+           std::cout << "[" << std::endl;
+           is_first = false;
          } else {
-            std::cout << "," << std::endl;
+           std::cout << "," << std::endl;
          }
          std::cout << "{ \"get_blocks_result_v0\":" << std::endl;
 
          rapidjson::StringBuffer result_sb;
          rapidjson::PrettyWriter<rapidjson::StringBuffer> result_writer(result_sb);
-         result_doucment[1].Accept(result_writer);
+         result_document[1].Accept(result_writer);
          std::cout << result_sb.GetString() << std::endl << "}" << std::endl;
 
+         // validate after streaming, so that invalid entry is included in the output
+         uint32_t this_block_num = 0;
+         if( result_document[1].HasMember("this_block") && result_document[1]["this_block"].IsObject() ) {
+            if( result_document[1]["this_block"].HasMember("block_num") && result_document[1]["this_block"]["block_num"].IsUint() ) {
+               this_block_num = result_document[1]["this_block"]["block_num"].GetUint();
+            }
+            std::string this_block_id;
+            if( result_document[1]["this_block"].HasMember("block_id") && result_document[1]["this_block"]["block_id"].IsString() ) {
+               this_block_id = result_document[1]["this_block"]["block_id"].GetString();
+            }
+            std::string prev_block_id;
+            if( result_document[1]["prev_block"].HasMember("block_id") && result_document[1]["prev_block"]["block_id"].IsString() ) {
+               prev_block_id = result_document[1]["prev_block"]["block_id"].GetString();
+            }
+            if( !irreversible_only && !this_block_id.empty() && !prev_block_id.empty() ) {
+               // verify forks were sent
+               if (block_ids.count(this_block_num-1)) {
+                  if (block_ids[this_block_num-1].count(prev_block_id) == 0) {
+                     std::cerr << "Received block: << " << this_block_num << " that does not link to previous: ";
+                     std::copy(block_ids[this_block_num-1].begin(), block_ids[this_block_num-1].end(), std::ostream_iterator<std::string>(std::cerr, " "));
+                     std::cerr << std::endl;
+                     return 1;
+                  }
+               }
+               block_ids[this_block_num].insert(this_block_id);
+
+               if( result_document[1]["last_irreversible"].HasMember("block_num") && result_document[1]["last_irreversible"]["block_num"].IsUint() ) {
+                  uint32_t lib_num = result_document[1]["last_irreversible"]["block_num"].GetUint();
+                  auto i = block_ids.lower_bound(lib_num);
+                  if (i != block_ids.end()) {
+                     block_ids.erase(block_ids.begin(), i);
+                  }
+               }
+            }
+
+         }
+
          if( this_block_num == end_block_num ) break;
       }