Merge pull request #657 from AntelopeIO/bump_leap_to_3_2_1

[3.2] bump Leap version to v3.2.1

.gitignore

@@ -79,6 +79,8 @@ Testing/*
 build.tar.gz
 build/*
 build-debug/*
+deps/*
+dependencies/*
 
 etc/eosio/node_*
 var/lib/node_*

CMakeLists.txt

@@ -15,8 +15,8 @@ set( CXX_STANDARD_REQUIRED ON)
 
 set(VERSION_MAJOR 3)
 set(VERSION_MINOR 2)
-set(VERSION_PATCH 0)
-set(VERSION_SUFFIX dev)
+set(VERSION_PATCH 1)
+#set(VERSION_SUFFIX rc2)
 
 if(VERSION_SUFFIX)
     set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}")

CMakeModules/EosioTesterBuild.cmake.in

@@ -51,7 +51,7 @@ find_package(Boost @Boost_MAJOR_VERSION@.@Boost_MINOR_VERSION@ EXACT REQUIRED CO
 
 find_library(libtester eosio_testing @CMAKE_BINARY_DIR@/libraries/testing NO_DEFAULT_PATH)
 find_library(libchain eosio_chain @CMAKE_BINARY_DIR@/libraries/chain NO_DEFAULT_PATH)
-find_library(libfc fc @CMAKE_BINARY_DIR@/libraries/fc NO_DEFAULT_PATH)
+find_library(libfc fc @CMAKE_BINARY_DIR@/libraries/libfc NO_DEFAULT_PATH)
 find_library(libsecp256k1 secp256k1 @CMAKE_BINARY_DIR@/libraries/libfc/secp256k1 NO_DEFAULT_PATH)
 
 find_library(libff ff @CMAKE_BINARY_DIR@/libraries/libfc/libraries/ff/libff NO_DEFAULT_PATH)

README.md

@@ -1,126 +1,257 @@
 # Leap
+Leap is a C++ implementation of the [Antelope](https://github.com/AntelopeIO) protocol. It contains blockchain node software and supporting tools for developers and node operators.
 
-Leap is blockchain node software and supporting tools that implements the [Antelope](https://github.com/AntelopeIO) protocol.
+## Branches
+The `main` branch is the development branch; do not use it for production. Refer to the [release page](https://github.com/AntelopeIO/leap/releases) for current information on releases, pre-releases, and obsolete releases, as well as the corresponding tags for those releases.
 
-## Repo Organization
+## Supported Operating Systems
+We currently support the following operating systems.
+- Ubuntu 22.04 Jammy
+- Ubuntu 20.04 Focal
+- Ubuntu 18.04 Bionic
 
-`main` branch is the development branch: do not use this for production. Refer to the [release page](https://github.com/AntelopeIO/leap/releases) for current information on releases, pre-releases, and obsolete releases as well as the corresponding tags for those releases.
+Other Unix derivatives such as macOS are tended to on a best-effort basis and may not be full featured. If you aren't using Ubuntu, please visit the "[Build Unsupported OS](./docs/00_install/01_build-from-source/00_build-unsupported-os.md)" page to explore your options.
 
-## Software Installation
-
-Visit the [release page](https://github.com/AntelopeIO/leap/releases) for Ubuntu binaries. This is the fastest way to get started with the software.
-
-### Building From Source
-
-Recent Ubuntu LTS releases are the only Linux distributions that we fully support. Other Linux distros and other POSIX operating systems (such as macOS) are tended to on a best-effort basis and may not be full featured. Notable requirements to build are:
-* C++17 compiler and standard library
-* boost 1.67+
-* CMake 3.8+
-* (for Linux only) LLVM 7 - 11 (newer versions do not work)
-
-A few other common libraries are tools also required such as openssl 1.1+, libcurl, curl, libusb, GMP, Python 3, and zlib.
-
-**A Warning On Parallel Compilation Jobs (`-j` flag)**: When building C/C++ software often the build is performed in parallel via a command such as `make -j $(nproc)` which uses the number of CPU cores as the number of compilation jobs to perform simultaneously. However, be aware that some compilation units (.cpp files) in Leap are extremely complex and will consume nearly 4GB of memory to compile. You may need to reduce the level of parallelization depending on the amount of memory on your build host. e.g. instead of `make -j $(nproc)` run `make -j2`. Failures due to memory exhaustion will typically but not always manifest as compiler crashes.
-
-Generally we recommend performing what we refer to as a "pinned build" which ensures the compiler and boost version remain the same between builds of different Leap versions (Leap requires these versions remain the same otherwise its state needs to be repopulated from a portable snapshot).
-
-#### Building Pinned Build Binary Packages
-In the directory `<leap src>/scripts` you will find the two scripts `install_deps.sh` and `pinned_build.sh`. If you haven't installed build dependencies then run `install_deps.sh`. Then run `pinned_build.sh <dependencies directory> <leap build directory> <number of jobs>`.
-
-The dependencies directory is where the script will pull the C++ dependencies that need to be built with the pinned compiler for building the pinned binaries for binary packaging.
-
-The binary package will be produced in the Leap build directory that was supplied.
-
-#### Manual (non "pinned") Build Instructions
-
-<details>
-  <summary>Ubuntu 20.04 & 22.04 Build Instructions</summary>
-
-Install required dependencies: 
+If you are running an unsupported Ubuntu derivative, such as Linux Mint, you can find the version of Ubuntu your distribution was based on by using this command:
+```bash
+cat /etc/upstream-release/lsb-release
 ```
-apt-get update && apt-get install   \
-        build-essential             \
-        cmake                       \
-        curl                        \
-        git                         \
-        libboost-all-dev            \
-        libcurl4-openssl-dev        \
-        libgmp-dev                  \
-        libssl-dev                  \
-        libusb-1.0-0-dev            \
-        llvm-11-dev                 \
+Your best bet is to follow the instructions for your Ubuntu base, but we make no guarantees.
+
+## Binary Installation
+This is the fastest way to get started. From the [latest release](https://github.com/AntelopeIO/leap/releases/latest) page, download a binary for one of our [supported operating systems](#supported-operating-systems), or visit the [release tags](https://github.com/AntelopeIO/leap/releases) page to download a binary for a specific version of Leap.
+
+Once you have a `*.deb` file downloaded for your version of Ubuntu, you can install it as follows:
+```bash
+sudo apt-get update
+sudo apt-get install -y ~/Downloads/leap*.deb
+```
+Your download path may vary. If you are in an Ubuntu docker container, omit `sudo` because you run as `root` by default.
+
+Finally, verify Leap was installed correctly:
+```bash
+nodeos --full-version
+```
+You should see a [semantic version](https://semver.org) string followed by a `git` commit hash with no errors. For example:
+```
+v3.1.2-0b64f879e3ebe2e4df09d2e62f1fc164cc1125d1
+```
+
+## Build and Install from Source
+You can also build and install Leap from source.
+
+### Prerequisites
+You will need to build on a [supported operating system](#supported-operating-systems).
+
+Requirements to build:
+- C++17 compiler and standard library
+- boost 1.67+
+- CMake 3.8+
+- LLVM 7 - 11 - for Linux only
+  - newer versions do not work
+- openssl 1.1+
+- libcurl
+- curl
+- libusb
+- git
+- GMP
+- Python 3
+- zlib
+
+### Step 1 - Clone
+If you don't have the Leap repo cloned to your computer yet, [open a terminal](https://itsfoss.com/open-terminal-ubuntu) and navigate to the folder where you want to clone the Leap repository:
+```bash
+cd ~/Downloads
+```
+Clone Leap using either HTTPS...
+```bash
+git clone --recursive https://github.com/AntelopeIO/leap.git
+```
+...or SSH:
+```bash
+git clone --recursive git@github.com:AntelopeIO/leap.git
+```
+
+> ℹ️ **HTTPS vs. SSH Clone** ℹ️  
+Both an HTTPS or SSH git clone will yield the same result - a folder named `leap` containing our source code. It doesn't matter which type you use.
+
+Navigate into that folder:
+```bash
+cd leap
+```
+
+### Step 2 - Checkout Release Tag or Branch
+Choose which [release](https://github.com/AntelopeIO/leap/releases) or [branch](#branches) you would like to build, then check it out. If you are not sure, use the [latest release](https://github.com/AntelopeIO/leap/releases/latest). For example, if you want to build release 3.1.2 then you would check it out using its tag, `v3.1.2`. In the example below, replace `v0.0.0` with your selected release tag accordingly:
+```bash
+git fetch --all --tags
+git checkout v0.0.0
+```
+
+Once you are on the branch or release tag you want to build, make sure everything is up-to-date:
+```bash
+git pull
+git submodule update --init --recursive
+```
+
+### Step 3 - Build
+Select build instructions below for a [pinned build](#pinned-build) (preferred) or an [unpinned build](#unpinned-build).
+
+> ℹ️ **Pinned vs. Unpinned Build** ℹ️  
+We have two types of builds for Leap: "pinned" and "unpinned." The only difference is that pinned builds use specific versions for some dependencies hand-picked by the Leap engineers - they are "pinned" to those versions. In contrast, unpinned builds use the default dependency versions available on the build system at the time. We recommend performing a "pinned" build to ensure the compiler and boost versions remain the same between builds of different Leap versions. Leap requires these versions to remain the same, otherwise its state might need to be recovered from a portable snapshot or the chain needs to be replayed.
+
+> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️  
+When building C/C++ software, often the build is performed in parallel via a command such as `make -j "$(nproc)"` which uses all available CPU threads. However, be aware that some compilation units (`*.cpp` files) in Leap will consume nearly 4GB of memory. Failures due to memory exhaustion will typically, but not always, manifest as compiler crashes. Using all available CPU threads may also prevent you from doing other things on your computer during compilation. For these reasons, consider reducing this value.
+
+> 🐋 **Docker and `sudo`** 🐋  
+If you are in an Ubuntu docker container, omit `sudo` from all commands because you run as `root` by default. Most other docker containers also exclude `sudo`, especially Debian-family containers. If your shell prompt is a hash tag (`#`), omit `sudo`.
+
+#### Pinned Build
+Make sure you are in the root of the `leap` repo, then run the `install_depts.sh` script to install dependencies:
+```bash
+sudo scripts/install_deps.sh
+```
+
+Next, run the pinned build script. You have to give it three arguments, in the following order:
+  - A temporary folder, for all dependencies that need to be built from source.
+  - A build folder, where the binaries you need to install will be built to.
+  - The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
+
+The following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads (Note: you don't need `sudo` for this command):
+```bash
+scripts/pinned_build.sh deps build "$(nproc)"
+```
+Now you can optionally [test](#step-4---test) your build, or [install](#step-5---install) the `*.deb` binary packages, which will be in the root of your build directory.
+
+#### Unpinned Build
+The following instructions are valid for this branch. Other release branches may have different requirements, so ensure you follow the directions in the branch or release you intend to build. If you are in an Ubuntu docker container, omit `sudo` because you run as `root` by default.
+
+<details> <summary>Ubuntu 22.04 Jammy & Ubuntu 20.04 Focal</summary>
+
+Install dependencies:
+```bash
+sudo apt-get update
+sudo apt-get install -y \
+        build-essential \
+        cmake \
+        curl \
+        git \
+        libboost-all-dev \
+        libcurl4-openssl-dev \
+        libgmp-dev \
+        libssl-dev \
+        libusb-1.0-0-dev \
+        llvm-11-dev \
         pkg-config
 ```
-and perform the build:
-```
-git submodule update --init --recursive
-mkdir build
+To build, make sure you are in the root of the `leap` repo, then run the following command:
+```bash
+mkdir -p build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/usr/lib/llvm-11 ..
 make -j $(nproc) package
 ```
 </details>
 
-<details>
-  <summary>Ubuntu 18.04 Build Instructions</summary>
+<details> <summary>Ubuntu 18.04 Bionic</summary>
 
-Install required dependencies. You will need to build Boost from source on this distribution. 
-```
-apt-get update && apt-get install   \
-        build-essential             \
-        cmake                       \
-        curl                        \
-        g++-8                       \
-        git                         \
-        libcurl4-openssl-dev        \
-        libgmp-dev                  \
-        libssl-dev                  \
-        libusb-1.0-0-dev            \
-        llvm-7-dev                  \
-        pkg-config                  \
-        python3                     \
+Install dependencies:
+```bash
+sudo apt-get update
+sudo apt-get install -y \
+        build-essential \
+        cmake \
+        curl \
+        g++-8 \
+        git \
+        libcurl4-openssl-dev \
+        libgmp-dev \
+        libssl-dev \
+        libusb-1.0-0-dev \
+        llvm-7-dev \
+        pkg-config \
+        python3 \
         zlib1g-dev
-        
-curl -L https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 | tar jx && \
-   cd boost_1_79_0 &&                                                                                     \
-   ./bootstrap.sh --prefix=$HOME/boost1.79 &&                                                             \
-   ./b2 --with-iostreams --with-date_time --with-filesystem --with-system                                 \
-        --with-program_options --with-chrono --with-test -j$(nproc) install &&                            \
-   cd ..
 ```
-and perform the build:
+You need to build Boost from source on this distribution:
+```bash
+curl -fL https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 -o ~/Downloads/boost_1_79_0.tar.bz2
+tar -jvxf ~/Downloads/boost_1_79_0.tar.bz2 -C ~/Downloads/
+pushd ~/Downloads/boost_1_79_0
+./bootstrap.sh --prefix="$HOME/boost1.79"
+./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -j "$(nproc)" install
+popd
 ```
-git submodule update --init --recursive
-mkdir build
+The Boost `*.tar.bz2` download and `boost_1_79_0` folder can be removed now if you want more space.
+```bash
+rm -r ~/Downloads/boost_1_79_0.tar.bz2 ~/Downloads/boost_1_79_0
+```
+From a terminal in the root of the `leap` repo, build.
+```bash
+mkdir -p build
 cd build
-cmake -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 \
-      -DCMAKE_PREFIX_PATH="$HOME/boost1.79;/usr/lib/llvm-7/"  -DCMAKE_BUILD_TYPE=Release .. \
-make -j $(nproc) package
+cmake -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 -DCMAKE_PREFIX_PATH="$HOME/boost1.79;/usr/lib/llvm-7/" -DCMAKE_BUILD_TYPE=Release ..
+make -j "$(nproc)" package
 ```
-After building you may remove the `$HOME/boost1.79` directory, or you may keep it around until next time building the software.
+After building, you may remove the `~/boost1.79` directory or you may keep it around for your next build.
 </details>
 
-### Running Tests
+Now you can optionally [test](#step-4---test) your build, or [install](#step-5---install) the `*.deb` binary packages, which will be in the root of your build directory.
 
-When building from source it's recommended to run at least what we refer to as the "parallelizable tests". Not included by default in the "parallelizable tests" are the WASM spec tests which can add additional coverage and can also be run in parallel.
+### Step 4 - Test
+Leap supports the following test suites:
 
-```
-cd build
+Test Suite | Test Type | [Test Size](https://testing.googleblog.com/2010/12/test-sizes.html) | Notes
+---|:---:|:---:|---
+[Parallelizable tests](#parallelizable-tests) | Unit tests | Small
+[WASM spec tests](#wasm-spec-tests) | Unit tests | Small | Unit tests for our WASM runtime, each short but _very_ CPU-intensive
+[Serial tests](#serial-tests) | Component/Integration | Medium
+[Long-running tests](#long-running-tests) | Integration | Medium-to-Large | Tests which take an extraordinarily long amount of time to run
 
-# "parallelizable tests": the minimum test set that should be run
-ctest -j $(nproc) -LE _tests
+When building from source, we recommended running at least the [parallelizable tests](#parallelizable-tests).
 
-# Also consider running the WASM spec tests for more coverage
-ctest -j $(nproc) -L wasm_spec_tests
+#### Parallelizable Tests
+This test suite consists of any test that does not require shared resources, such as file descriptors, specific folders, or ports, and can therefore be run concurrently in different threads without side effects (hence, easily parallelized). These are mostly unit tests and [small tests](https://testing.googleblog.com/2010/12/test-sizes.html) which complete in a short amount of time.
+
+You can invoke them by running `ctest` from a terminal in your Leap build directory and specifying the following arguments:
+```bash
+ctest -j "$(nproc)" -LE _tests
 ```
 
-Some other tests are available and recommended but be aware they can be sensitive to other software running on the same host and they may **SIGKILL** other nodeos instances running on the host.
-```
-cd build
+#### WASM Spec Tests
+The WASM spec tests verify that our WASM execution engine is compliant with the web assembly standard. These are very [small](https://testing.googleblog.com/2010/12/test-sizes.html), very fast unit tests. However, there are over a thousand of them so the suite can take a little time to run. These tests are extremely CPU-intensive.
 
-# These tests can't run in parallel but are recommended.
+You can invoke them by running `ctest` from a terminal in your Leap build directory and specifying the following arguments:
+```bash
+ctest -j "$(nproc)" -L wasm_spec_tests
+```
+We have observed severe performance issues when multiple virtual machines are running this test suite on the same physical host at the same time, for example in a CICD system. This can be resolved by disabling hyperthreading on the host.
+
+#### Serial Tests
+The serial test suite consists of [medium](https://testing.googleblog.com/2010/12/test-sizes.html) component or integration tests that use specific paths, ports, rely on process names, or similar, and cannot be run concurrently with other tests. Serial tests can be sensitive to other software running on the same host and they may `SIGKILL` other `nodeos` processes. These tests take a moderate amount of time to complete, but we recommend running them.
+
+You can invoke them by running `ctest` from a terminal in your Leap build directory and specifying the following arguments:
+```bash
 ctest -L "nonparallelizable_tests"
+```
 
-# These tests can't run in parallel. They also take a long time to run.
+#### Long-Running Tests
+The long-running tests are [medium-to-large](https://testing.googleblog.com/2010/12/test-sizes.html) integration tests that rely on shared resources and take a very long time to run.
+
+You can invoke them by running `ctest` from a terminal in your Leap build directory and specifying the following arguments:
+```bash
 ctest -L "long_running_tests"
 ```
+
+### Step 5 - Install
+Once you have [built](#step-3---build-the-source-code) Leap and [tested](#step-4---test) your build, you can install Leap on your system. Don't forget to omit `sudo` if you are running in a docker container.
+
+We recommend installing the binary package you just built. Navigate to your Leap build directory in a terminal and run this command:
+```bash
+sudo apt-get update
+sudo apt-get install -y ./leap[-_][0-9]*.deb
+```
+
+It is also possible to install using `make` instead:
+```bash
+sudo make install
+```

docs.json

@@ -1,65 +0,0 @@
-{
-    "name": "eos",
-    "generators": [
-        {
-            "name": "collate_markdown",
-            "options": {
-                "docs_dir": "docs"
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/chain_api_plugin/chain.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/chain_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/db_size_api_plugin/db_size.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/db_size_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/producer_api_plugin/producer.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/producer_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/net_api_plugin/net.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/net_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/test_control_api_plugin/test_control.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/test_control_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/trace_api_plugin/trace_api.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/trace_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        }
-    ]
-}

docs/00_install/01_build-from-source/index.md

@@ -2,123 +2,19 @@
 content_title: Build Antelope from Source
 ---
 
-The shell scripts previously recommended for building the software have been removed in favor of a build process entirely driven by CMake. Those wishing to build from source are now responsible for installing the necessary dependencies. The list of dependencies and the recommended build procedure are in the README.md file. Instructions are also included for efficiently running the tests.
+The shell scripts previously recommended for building the software have been removed in favor of a build process entirely driven by CMake. Those wishing to build from source are now responsible for installing the necessary dependencies. The list of dependencies and the recommended build procedure are in the [`README.md`](https://github.com/AntelopeIO/leap/blob/release/3.2/README.md) file. Instructions are also included for efficiently running the tests.
 
 ### Using DUNE
-
-As an alternative to building from source try [Docker Utilities for Node Execution](https://github.com/AntelopeIO/DUNE) for the easiest way to get started, and for multi-platform support.  
+As an alternative to building from source, try [Docker Utilities for Node Execution](https://github.com/AntelopeIO/DUNE) for the easiest way to get started and for multi-platform support.
 
 ### Building From Source
-
-Recent Ubuntu LTS releases are the only Linux distributions that we fully support. Other Linux distros and other POSIX operating systems (such as macOS) are tended to on a best-effort basis and may not be full featured. Notable requirements to build are:
-* C++17 compiler and standard library
-* boost 1.67+
-* CMake 3.8+
-* (for Linux only) LLVM 7 - 11 (newer versions do not work)
-
-A few other common libraries are tools also required such as openssl 1.1+, libcurl, curl, libusb, GMP, Python 3, and zlib.
-
-**A Warning On Parallel Compilation Jobs (`-j` flag)**: When building C/C++ software often the build is performed in parallel via a command such as `make -j $(nproc)` which uses the number of CPU cores as the number of compilation jobs to perform simultaneously. However, be aware that some compilation units (.cpp files) in mandel are extremely complex and will consume nearly 4GB of memory to compile. You may need to reduce the level of parallelization depending on the amount of memory on your build host. e.g. instead of `make -j $(nproc)` run `make -j2`. Failures due to memory exhaustion will typically but not always manifest as compiler crashes.
-
-Generally we recommend performing what we refer to as a "pinned build" which ensures the compiler and boost version remain the same between builds of different mandel versions (mandel requires these versions remain the same otherwise its state needs to be repopulated from a portable snapshot).
+You can also build and install Leap from source. Instructions for that currently live [here](https://github.com/AntelopeIO/leap/blob/release/3.2/README.md#build-and-install-from-source).
 
 #### Building Pinned Build Binary Packages
-In the directory `<mandel src>/scripts` you will find the two scripts `install_deps.sh` and `pinned_build.sh`. If you haven't installed build dependencies then run `install_deps.sh`. Then run `pinned_build.sh <dependencies directory> <mandel build directory> <number of jobs>`.
-
-The dependencies directory is where the script will pull the C++ dependencies that need to be built with the pinned compiler for building the pinned binaries for binary packaging.
-
-The binary package will be produced in the mandel build directory that was supplied.
+The pinned build instructions have moved [here](https://github.com/AntelopeIO/leap/blob/release/3.2/README.md#pinned-build). You may want to look at the [prerequisites](https://github.com/AntelopeIO/leap/blob/release/3.2/README.md#prerequisites) and our warning on parallelization using the `-j` jobs flag [here](https://github.com/AntelopeIO/leap/blob/release/3.2/README.md#step-3---build) before you build.
 
 #### Manual (non "pinned") Build Instructions
-
-<details>
-  <summary>Ubuntu 20.04 & 22.04 Build Instructions</summary>
-
-Install required dependencies:
-```
-apt-get update && apt-get install   \
-        build-essential             \
-        cmake                       \
-        curl                        \
-        git                         \
-        libboost-all-dev            \
-        libcurl4-openssl-dev        \
-        libgmp-dev                  \
-        libssl-dev                  \
-        libusb-1.0-0-dev            \
-        llvm-11-dev                 \
-        pkg-config
-```
-and perform the build:
-```
-git submodule update --init --recursive
-mkdir build
-cd build
-cmake -DCMAKE_BUILD_TYPE=Release ..
-make -j $(nproc) package
-```
-</details>
-
-<details>
-  <summary>Ubuntu 18.04 Build Instructions</summary>
-
-Install required dependencies. You will need to build Boost from source on this distribution.
-```
-apt-get update && apt-get install   \
-        build-essential             \
-        cmake                       \
-        curl                        \
-        g++-8                       \
-        git                         \
-        libcurl4-openssl-dev        \
-        libgmp-dev                  \
-        libssl-dev                  \
-        libusb-1.0-0-dev            \
-        llvm-7-dev                  \
-        pkg-config                  \
-        python3                     \
-        zlib1g-dev
-
-curl -L https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 | tar jx && \
-   cd boost_1_79_0 &&                                                                                     \
-   ./bootstrap.sh --prefix=$HOME/boost1.79 &&                                                             \
-   ./b2 --with-iostreams --with-date_time --with-filesystem --with-system                                 \
-        --with-program_options --with-chrono --with-test -j$(nproc) install &&                            \
-   cd ..
-```
-and perform the build:
-```
-git submodule update --init --recursive
-mkdir build
-cd build
-cmake -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 \
-      -DCMAKE_PREFIX_PATH="$HOME/boost1.79;/usr/lib/llvm-7/"  -DCMAKE_BUILD_TYPE=Release .. \
-make -j $(nproc) package
-```
-After building you may remove the `$HOME/boost1.79` directory, or you may keep it around until next time building the software.
-</details>
+The unpinned build instructions have moved [here](https://github.com/AntelopeIO/leap/blob/release/3.2/README.md#unpinned-build). You may want to look at the [prerequisites](https://github.com/AntelopeIO/leap/blob/release/3.2/README.md#prerequisites) and our warning on parallelization using the `-j` jobs flag [here](https://github.com/AntelopeIO/leap/blob/release/3.2/README.md#step-3---build) before you build.
 
 ### Running Tests
-
-When building from source it's recommended to run at least what we refer to as the "parallelizable tests". Not included by default in the "parallelizable tests" are the WASM spec tests which can add additional coverage and can also be run in parallel.
-
-```
-cd build
-
-# "parallelizable tests": the minimum test set that should be run
-ctest -j $(nproc) -LE _tests
-
-# Also consider running the WASM spec tests for more coverage
-ctest -j $(nproc) -L wasm_spec_tests
-```
-
-Some other tests are available and recommended but be aware they can be sensitive to other software running on the same host and they may **SIGKILL** other nodeos instances running on the host.
-```
-cd build
-
-# These tests can't run in parallel but are recommended.
-ctest -L "nonparallelizable_tests"
-
-# These tests can't run in parallel. They also take a long time to run.
-ctest -L "long_running_tests"
-```
+Documentation on available test suites and how to run them has moved [here](https://github.com/AntelopeIO/leap/blob/release/3.2/README.md#test).

docs/00_install/index.md

@@ -10,9 +10,9 @@ The best way to install and use the Antelope software is to build it from source
 
 Antelope currently supports the following operating systems:
 
-1. Ubuntu 18.04
-2. Ubuntu 20.04
-3. Ubuntu 22.04
+1. Ubuntu 22.04 Jammy
+2. Ubuntu 20.04 Focal
+3. Ubuntu 18.04 Bionic
 
 [[info | Note]]
 | It may be possible to build and install Antelope on other Unix-based operating systems. We gathered helpful information on the following page but please keep in mind that it is experimental and not officially supported. 

docs/01_nodeos/03_plugins/chain_plugin/index.md

@@ -21,33 +21,35 @@ These can only be specified from the `nodeos` command-line:
 ```console
 Command Line Options for eosio::chain_plugin:
   --genesis-json arg                    File to read Genesis State from
-  --genesis-timestamp arg               override the initial timestamp in the 
+  --genesis-timestamp arg               override the initial timestamp in the
                                         Genesis State file
-  --print-genesis-json                  extract genesis_state from blocks.log 
+  --print-genesis-json                  extract genesis_state from blocks.log
                                         as JSON, print to console, and exit
-  --extract-genesis-json arg            extract genesis_state from blocks.log 
+  --extract-genesis-json arg            extract genesis_state from blocks.log
                                         as JSON, write into specified file, and
                                         exit
-  --print-build-info                    print build environment information to 
+  --print-build-info                    print build environment information to
                                         console as JSON and exit
-  --extract-build-info arg              extract build environment information 
+  --extract-build-info arg              extract build environment information
                                         as JSON, write into specified file, and
                                         exit
-  --force-all-checks                    do not skip any checks that can be 
-                                        skipped while replaying irreversible 
-                                        blocks
+  --force-all-checks                    do not skip any validation checks while
+                                        replaying blocks (useful for replaying
+                                        blocks from untrusted source)
   --disable-replay-opts                 disable optimizations that specifically
                                         target replay
-  --replay-blockchain                   clear chain state database and replay 
+  --replay-blockchain                   clear chain state database and replay
                                         all blocks
-  --hard-replay-blockchain              clear chain state database, recover as 
-                                        many blocks as possible from the block 
+  --hard-replay-blockchain              clear chain state database, recover as
+                                        many blocks as possible from the block
                                         log, and then replay those blocks
-  --delete-all-blocks                   clear chain state database and block 
+  --delete-all-blocks                   clear chain state database and block
                                         log
-  --truncate-at-block arg (=0)          stop hard replay / block log recovery 
-                                        at this block number (if set to 
+  --truncate-at-block arg (=0)          stop hard replay / block log recovery
+                                        at this block number (if set to
                                         non-zero number)
+  --terminate-at-block arg (=0)         terminate after reaching this block
+                                        number (if set to a non-zero number)
   --snapshot arg                        File to read Snapshot State from
 
 ```
@@ -58,162 +60,170 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::chain_plugin:
-  --blocks-dir arg (="blocks")          the location of the blocks directory 
-                                        (absolute path or relative to 
+  --blocks-dir arg (="blocks")          the location of the blocks directory
+                                        (absolute path or relative to
                                         application data dir)
   --protocol-features-dir arg (="protocol_features")
-                                        the location of the protocol_features 
+                                        the location of the protocol_features
                                         directory (absolute path or relative to
                                         application config dir)
-  --checkpoint arg                      Pairs of [BLOCK_NUM,BLOCK_ID] that 
+  --checkpoint arg                      Pairs of [BLOCK_NUM,BLOCK_ID] that
                                         should be enforced as checkpoints.
-  --wasm-runtime runtime (=eos-vm-jit)  Override default WASM runtime ( 
+  --wasm-runtime runtime (=eos-vm-jit)  Override default WASM runtime (
                                         "eos-vm-jit", "eos-vm")
-                                        "eos-vm-jit" : A WebAssembly runtime 
-                                        that compiles WebAssembly code to 
+                                        "eos-vm-jit" : A WebAssembly runtime
+                                        that compiles WebAssembly code to
                                         native x86 code prior to execution.
                                         "eos-vm" : A WebAssembly interpreter.
-                                        
-  --profile-account arg                 The name of an account whose code will 
+
+  --profile-account arg                 The name of an account whose code will
                                         be profiled
   --abi-serializer-max-time-ms arg (=15)
-                                        Override default maximum ABI 
+                                        Override default maximum ABI
                                         serialization time allowed in ms
-  --chain-state-db-size-mb arg (=1024)  Maximum size (in MiB) of the chain 
+  --chain-state-db-size-mb arg (=1024)  Maximum size (in MiB) of the chain
                                         state database
   --chain-state-db-guard-size-mb arg (=128)
-                                        Safely shut down node when free space 
-                                        remaining in the chain state database 
+                                        Safely shut down node when free space
+                                        remaining in the chain state database
                                         drops below this size (in MiB).
   --signature-cpu-billable-pct arg (=50)
                                         Percentage of actual signature recovery
-                                        cpu to bill. Whole number percentages, 
+                                        cpu to bill. Whole number percentages,
                                         e.g. 50 for 50%
-  --chain-threads arg (=2)              Number of worker threads in controller 
+  --chain-threads arg (=2)              Number of worker threads in controller
                                         thread pool
   --contracts-console                   print contract's output to console
-  --deep-mind                           print deeper information about chain 
+  --deep-mind                           print deeper information about chain
                                         operations
-  --actor-whitelist arg                 Account added to actor whitelist (may 
+  --actor-whitelist arg                 Account added to actor whitelist (may
                                         specify multiple times)
-  --actor-blacklist arg                 Account added to actor blacklist (may 
+  --actor-blacklist arg                 Account added to actor blacklist (may
                                         specify multiple times)
-  --contract-whitelist arg              Contract account added to contract 
+  --contract-whitelist arg              Contract account added to contract
                                         whitelist (may specify multiple times)
-  --contract-blacklist arg              Contract account added to contract 
+  --contract-blacklist arg              Contract account added to contract
                                         blacklist (may specify multiple times)
   --action-blacklist arg                Action (in the form code::action) added
-                                        to action blacklist (may specify 
+                                        to action blacklist (may specify
                                         multiple times)
-  --key-blacklist arg                   Public key added to blacklist of keys 
-                                        that should not be included in 
-                                        authorities (may specify multiple 
+  --key-blacklist arg                   Public key added to blacklist of keys
+                                        that should not be included in
+                                        authorities (may specify multiple
                                         times)
-  --sender-bypass-whiteblacklist arg    Deferred transactions sent by accounts 
-                                        in this list do not have any of the 
-                                        subjective whitelist/blacklist checks 
-                                        applied to them (may specify multiple 
+  --sender-bypass-whiteblacklist arg    Deferred transactions sent by accounts
+                                        in this list do not have any of the
+                                        subjective whitelist/blacklist checks
+                                        applied to them (may specify multiple
                                         times)
-  --read-mode arg (=speculative)        Database read mode ("speculative", 
+  --read-mode arg (=speculative)        Database read mode ("speculative",
                                         "head", "read-only", "irreversible").
-                                        In "speculative" mode: database 
-                                        contains state changes by transactions 
-                                        in the blockchain up to the head block 
-                                        as well as some transactions not yet 
+                                        In "speculative" mode: database
+                                        contains state changes by transactions
+                                        in the blockchain up to the head block
+                                        as well as some transactions not yet
                                         included in the blockchain.
                                         In "head" mode: database contains state
-                                        changes by only transactions in the 
-                                        blockchain up to the head block; 
-                                        transactions received by the node are 
+                                        changes by only transactions in the
+                                        blockchain up to the head block;
+                                        transactions received by the node are
                                         relayed if valid.
-                                        In "read-only" mode: (DEPRECATED: see 
-                                        p2p-accept-transactions & 
-                                        api-accept-transactions) database 
-                                        contains state changes by only 
-                                        transactions in the blockchain up to 
-                                        the head block; transactions received 
+                                        In "read-only" mode: (DEPRECATED: see
+                                        p2p-accept-transactions &
+                                        api-accept-transactions) database
+                                        contains state changes by only
+                                        transactions in the blockchain up to
+                                        the head block; transactions received
                                         via the P2P network are not relayed and
-                                        transactions cannot be pushed via the 
+                                        transactions cannot be pushed via the
                                         chain API.
-                                        In "irreversible" mode: database 
-                                        contains state changes by only 
-                                        transactions in the blockchain up to 
-                                        the last irreversible block; 
-                                        transactions received via the P2P 
-                                        network are not relayed and 
-                                        transactions cannot be pushed via the 
+                                        In "irreversible" mode: database
+                                        contains state changes by only
+                                        transactions in the blockchain up to
+                                        the last irreversible block;
+                                        transactions received via the P2P
+                                        network are not relayed and
+                                        transactions cannot be pushed via the
                                         chain API.
-                                        
-  --api-accept-transactions arg (=1)    Allow API transactions to be evaluated 
+
+  --api-accept-transactions arg (=1)    Allow API transactions to be evaluated
                                         and relayed if valid.
-  --validation-mode arg (=full)         Chain validation mode ("full" or 
+  --validation-mode arg (=full)         Chain validation mode ("full" or
                                         "light").
                                         In "full" mode all incoming blocks will
                                         be fully validated.
-                                        In "light" mode all incoming blocks 
-                                        headers will be fully validated; 
-                                        transactions in those validated blocks 
-                                        will be trusted 
-                                        
-  --disable-ram-billing-notify-checks   Disable the check which subjectively 
+                                        In "light" mode all incoming blocks
+                                        headers will be fully validated;
+                                        transactions in those validated blocks
+                                        will be trusted
+
+  --disable-ram-billing-notify-checks   Disable the check which subjectively
                                         fails a transaction if a contract bills
-                                        more RAM to another account within the 
+                                        more RAM to another account within the
                                         context of a notification handler (i.e.
-                                        when the receiver is not the code of 
+                                        when the receiver is not the code of
                                         the action).
   --maximum-variable-signature-length arg (=16384)
-                                        Subjectively limit the maximum length 
-                                        of variable components in a variable 
+                                        Subjectively limit the maximum length
+                                        of variable components in a variable
                                         legnth signature to this size in bytes
-  --trusted-producer arg                Indicate a producer whose blocks 
-                                        headers signed by it will be fully 
-                                        validated, but transactions in those 
+  --trusted-producer arg                Indicate a producer whose blocks
+                                        headers signed by it will be fully
+                                        validated, but transactions in those
                                         validated blocks will be trusted.
   --database-map-mode arg (=mapped)     Database map mode ("mapped", "heap", or
                                         "locked").
-                                        In "mapped" mode database is memory 
+                                        In "mapped" mode database is memory
                                         mapped as a file.
                                         In "heap" mode database is preloaded in
-                                        to swappable memory and will use huge 
+                                        to swappable memory and will use huge
                                         pages if available.
                                         In "locked" mode database is preloaded,
-                                        locked in to memory, and will use huge 
+                                        locked in to memory, and will use huge
                                         pages if available.
-                                        
-  --enable-account-queries arg (=0)     enable queries to find accounts by 
+
+  --eos-vm-oc-cache-size-mb arg (=1024) Maximum size (in MiB) of the EOS VM OC
+                                        code cache
+  --eos-vm-oc-compile-threads arg (=1)  Number of threads to use for EOS VM OC
+                                        tier-up
+  --eos-vm-oc-enable                    Enable EOS VM OC tier-up runtime
+  --enable-account-queries arg (=0)     enable queries to find accounts by
                                         various metadata.
   --max-nonprivileged-inline-action-size arg (=4096)
-                                        maximum allowed size (in bytes) of an 
-                                        inline action for a nonprivileged 
+                                        maximum allowed size (in bytes) of an
+                                        inline action for a nonprivileged
                                         account
   --transaction-retry-max-storage-size-gb arg
-                                        Maximum size (in GiB) allowed to be 
-                                        allocated for the Transaction Retry 
-                                        feature. Setting above 0 enables this 
+                                        Maximum size (in GiB) allowed to be
+                                        allocated for the Transaction Retry
+                                        feature. Setting above 0 enables this
                                         feature.
   --transaction-retry-interval-sec arg (=20)
-                                        How often, in seconds, to resend an 
-                                        incoming transaction to network if not 
+                                        How often, in seconds, to resend an
+                                        incoming transaction to network if not
                                         seen in a block.
-  --transaction-retry-max-expiration-sec arg (=90)
-                                        Maximum allowed transaction expiration 
-                                        for retry transactions, will retry 
+  --transaction-retry-max-expiration-sec arg (=120)
+                                        Maximum allowed transaction expiration
+                                        for retry transactions, will retry
                                         transactions up to this value.
   --transaction-finality-status-max-storage-size-gb arg
-                                        Maximum size (in GiB) allowed to be 
-                                        allocated for the Transaction Finality 
+                                        Maximum size (in GiB) allowed to be
+                                        allocated for the Transaction Finality
                                         Status feature. Setting above 0 enables
                                         this feature.
   --transaction-finality-status-success-duration-sec arg (=180)
-                                        Duration (in seconds) a successful 
-                                        transaction's Finality Status will 
-                                        remain available from being first 
+                                        Duration (in seconds) a successful
+                                        transaction's Finality Status will
+                                        remain available from being first
                                         identified.
   --transaction-finality-status-failure-duration-sec arg (=180)
-                                        Duration (in seconds) a failed 
-                                        transaction's Finality Status will 
-                                        remain available from being first 
+                                        Duration (in seconds) a failed
+                                        transaction's Finality Status will
+                                        remain available from being first
                                         identified.
+  --block-log-retain-blocks arg         if set, periodically prune the block
+                                        log to store only configured number of
+                                        most recent blocks
 
 ```
 

docs/01_nodeos/03_plugins/history_api_plugin/index.md

@@ -1,59 +0,0 @@
-[[warning | Deprecation Notice]]
-| The `history_plugin` that the `history_api_plugin` depends upon is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead.
-
-## Description
-
-The `history_api_plugin` exposes functionality from the [`history_plugin`](../history_plugin/index.md) to the RPC API interface managed by the [`http_plugin`](../http_plugin/index.md), providing read-only access to blockchain data.
-
-It provides four RPC API endpoints:
-
-* get_actions
-* get_transaction
-* get_key_accounts
-* get_controlled_accounts
-
-The four actions listed above are used by the following `cleos` commands (matching order):
-
-* get actions
-* get transaction
-* get accounts
-* get servants
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::history_api_plugin
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_api_plugin
-```
-
-## Options
-
-None
-
-## Dependencies
-
-* [`history_plugin`](../history_plugin/index.md)
-* [`chain_plugin`](../chain_plugin/index.md)
-* [`http_plugin`](../http_plugin/index.md)
-
-### Load Dependency Examples
-
-```console
-# config.ini
-plugin = eosio::history_plugin
-[options]
-plugin = eosio::chain_plugin
-[options]
-plugin = eosio::http_plugin
-[options]
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_plugin [options]  \
-           --plugin eosio::chain_plugin [operations] [options]  \
-           --plugin eosio::http_plugin [options]
-```

docs/01_nodeos/03_plugins/history_plugin/index.md

@@ -1,41 +0,0 @@
-[[warning | Deprecation Notice]]
-| The `history_plugin` is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead.
-
-## Description
-
-The `history_plugin` provides a cache layer to obtain historical data about the blockchain objects. It depends on [`chain_plugin`](../chain_plugin/index.md) for the data.
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::history_plugin
-[options]
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_plugin [options]
-```
-
-## Options
-
-These can be specified from both the `nodeos` command-line or the `config.ini` file:
-
-```console
-Config Options for eosio::history_plugin:
-  -f [ --filter-on ] arg                Track actions which match 
-                                        receiver:action:actor. Actor may be 
-                                        blank to include all. Action and Actor 
-                                        both blank allows all from Recieiver. 
-                                        Receiver may not be blank.
-  -F [ --filter-out ] arg               Do not track actions which match 
-                                        receiver:action:actor. Action and Actor
-                                        both blank excludes all from Reciever. 
-                                        Actor blank excludes all from 
-                                        reciever:action. Receiver may not be 
-                                        blank.
-```
-
-## Dependencies
-
-* [`chain_plugin`](../chain_plugin/index.md)

docs/01_nodeos/03_plugins/http_plugin/index.md

@@ -22,46 +22,47 @@ These can be specified from both the command-line or the `config.ini` file:
 
 ```console
 Config Options for eosio::http_plugin:
-  --unix-socket-path arg                The filename (relative to data-dir) to 
-                                        create a unix socket for HTTP RPC; set 
-                                        blank to disable (=keosd.sock for keosd)
-  --http-server-address arg (=127.0.0.1:8888 for nodeos)
-                                        The local IP and port to listen for 
+  --unix-socket-path arg                The filename (relative to data-dir) to
+                                        create a unix socket for HTTP RPC; set
+                                        blank to disable.
+  --http-server-address arg (=127.0.0.1:8888)
+                                        The local IP and port to listen for
                                         incoming http connections; set blank to
                                         disable.
-  --https-server-address arg            The local IP and port to listen for 
+  --https-server-address arg            The local IP and port to listen for
                                         incoming https connections; leave blank
                                         to disable.
-  --https-certificate-chain-file arg    Filename with the certificate chain to 
-                                        present on https connections. PEM 
+  --https-certificate-chain-file arg    Filename with the certificate chain to
+                                        present on https connections. PEM
                                         format. Required for https.
-  --https-private-key-file arg          Filename with https private key in PEM 
+  --https-private-key-file arg          Filename with https private key in PEM
                                         format. Required for https
-  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use: 
+  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use:
                                         secp384r1 or prime256v1
   --access-control-allow-origin arg     Specify the Access-Control-Allow-Origin
                                         to be returned on each request.
   --access-control-allow-headers arg    Specify the Access-Control-Allow-Header
                                         s to be returned on each request.
-  --access-control-max-age arg          Specify the Access-Control-Max-Age to 
+  --access-control-max-age arg          Specify the Access-Control-Max-Age to
                                         be returned on each request.
   --access-control-allow-credentials    Specify if Access-Control-Allow-Credent
-                                        ials: true should be returned on each 
+                                        ials: true should be returned on each
                                         request.
-  --max-body-size arg (=1048576)        The maximum body size in bytes allowed 
+  --max-body-size arg (=2097152)        The maximum body size in bytes allowed
                                         for incoming RPC requests
   --http-max-bytes-in-flight-mb arg (=500)
-                                        Maximum size in megabytes http_plugin 
-                                        should use for processing http 
-                                        requests. 503 error response when 
-                                        exceeded.
+                                        Maximum size in megabytes http_plugin
+                                        should use for processing http
+                                        requests. -1 for unlimited. 429 error
+                                        response when exceeded.
+  --http-max-response-time-ms arg (=30) Maximum time for processing a request.
   --verbose-http-errors                 Append the error log to HTTP responses
-  --http-validate-host arg (=1)         If set to false, then any incoming 
+  --http-validate-host arg (=1)         If set to false, then any incoming
                                         "Host" header is considered valid
-  --http-alias arg                      Additionaly acceptable values for the 
-                                        "Host" header of incoming HTTP 
-                                        requests, can be specified multiple 
-                                        times.  Includes http/s_server_address 
+  --http-alias arg                      Additionaly acceptable values for the
+                                        "Host" header of incoming HTTP
+                                        requests, can be specified multiple
+                                        times.  Includes http/s_server_address
                                         by default.
   --http-threads arg (=2)               Number of worker threads in http thread
                                         pool

docs/01_nodeos/03_plugins/net_plugin/index.md

@@ -44,7 +44,7 @@ Config Options for eosio::net_plugin:
   --p2p-accept-transactions arg (=1)    Allow transactions received over p2p
                                         network to be evaluated and relayed if
                                         valid.
-  --agent-name arg (="EOS Test Agent")  The name supplied to identify this node
+  --agent-name arg (=EOS Test Agent)    The name supplied to identify this node
                                         amongst the peers.
   --allowed-connection arg (=any)       Can be 'any' or 'producers' or
                                         'specified' or 'none'. If 'specified',
@@ -73,13 +73,15 @@ Config Options for eosio::net_plugin:
                                         synchronization
   --use-socket-read-watermark arg (=0)  Enable experimental socket read
                                         watermark optimization
-  --peer-log-format arg (=["${_name}" ${_ip}:${_port}])
+  --peer-log-format arg (=["${_name}" - ${_cid} ${_ip}:${_port}] )
                                         The string used to format peers when
                                         logging messages about them.  Variables
                                         are escaped with ${<variable name>}.
                                         Available Variables:
                                            _name  self-reported name
 
+                                           _cid   assigned connection id
+
                                            _id    self-reported ID (64 hex
                                                   characters)
 
@@ -94,7 +96,10 @@ Config Options for eosio::net_plugin:
                                                   peer
 
                                            _lport local port number connected
-                                                  to peer                                        
+                                                  to peer
+  --p2p-keepalive-interval-ms arg (=10000)
+                                        peer heartbeat keepalive message
+                                        interval in milliseconds
 ```
 
 ## Dependencies

docs/01_nodeos/03_plugins/producer_plugin/index.md

@@ -23,7 +23,6 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::producer_plugin:
-
   -e [ --enable-stale-production ]      Enable block production, even if the
                                         chain is stale.
   -x [ --pause-on-startup ]             Start this node in a state where
@@ -49,7 +48,7 @@ Config Options for eosio::producer_plugin:
                                         <public-key>=<provider-spec>
                                         Where:
                                            <public-key>    is a string form of
-                                                           a vaild Antelope public
+                                                           a valid EOS public
                                                            key
 
                                            <provider-spec> is a string in the
@@ -59,7 +58,7 @@ Config Options for eosio::producer_plugin:
                                            <provider-type> is KEY, or KEOSD
 
                                            KEY:<data>      is a string form of
-                                                           a valid Antelope
+                                                           a valid EOS
                                                            private key which
                                                            maps to the provided
                                                            public key
@@ -119,11 +118,6 @@ Config Options for eosio::producer_plugin:
                                         Sets the maximum amount of failures
                                         that are allowed for a given account
                                         per block.
-                                        Disregarded for accounts that have been
-                                        whitelisted by disabling subjective
-                                        billing for the account using the
-                                        disable-subjective-account-billing
-                                        configuration option.
   --subjective-account-decay-time-minutes arg (=1440)
                                         Sets the time to return full subjective
                                         cpu for accounts
@@ -142,9 +136,6 @@ Config Options for eosio::producer_plugin:
   --disable-subjective-account-billing arg
                                         Account which is excluded from
                                         subjective CPU billing
-                                        Account is considered whitelisted and
-                                        will not be subject to enforcement of
-                                        subjective-account-max-failures.
   --disable-subjective-p2p-billing arg (=1)
                                         Disable subjective CPU billing for P2P
                                         transactions

docs/01_nodeos/03_plugins/resource_monitor_plugin/index.md

@@ -1,19 +1,10 @@
 
 ## Overview
 
-The `resource_monitor_plugin` monitors space usage in the computing system where `nodeos` is running. Specifically, every `resource-monitor-interval-seconds` seconds,
-it measures the individual space used by each of the file systems mounted
-by `data-dir`, `state-dir`, `blocks-log-dir`, `snapshots-dir`,
-`state-history-dir`, and `trace-dir`.
-When space usage in any of the monitored file system is within `5%` of the threshold
-specified by `resource-monitor-space-threshold`, a warning containing the file system
-path and percentage of space has used is printed out.
-When space usage exceeds the threshold,
-if `resource-monitor-not-shutdown-on-threshold-exceeded` is not set,
-`nodeos` gracefully shuts down; if `resource-monitor-not-shutdown-on-threshold-exceeded` is set, `nodeos` prints out warnings periodically
-until space usage goes under the threshold.
+The `resource_monitor_plugin` monitors space usage in the computing system where `nodeos` is running. Specifically, every `resource-monitor-interval-seconds` seconds, it measures the individual space used by each of the file systems mounted by `data-dir`, `state-dir`, `blocks-log-dir`, `snapshots-dir`, `state-history-dir`, and `trace-dir`. When space usage in any of the monitored file system is within `5%` of the threshold specified by `resource-monitor-space-threshold`, a warning containing the file system path and percentage of space has used is printed out. When space usage exceeds the threshold, if `resource-monitor-not-shutdown-on-threshold-exceeded` is not set, `nodeos` gracefully shuts down; if `resource-monitor-not-shutdown-on-threshold-exceeded` is set, `nodeos` prints out warnings periodically until space usage goes under the threshold.
 
 `resource_monitor_plugin` is always loaded.
+
 ## Usage
 
 ```console
@@ -32,33 +23,28 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::resource_monitor_plugin:
-
   --resource-monitor-interval-seconds arg (=2)
-                                        Time in seconds between two consecutive checks
-                                        of space usage. Should be between 1 and 300.
+                                        Time in seconds between two consecutive
+                                        checks of resource usage. Should be
+                                        between 1 and 300
   --resource-monitor-space-threshold arg (=90)
-                                        Threshold in terms of percentage of used space
-                                        vs total space. If the used space is within
-                                        `5%` of the threshold, a warning is generated.
-                                        If the used space is above the threshold and
-                                        `resource-monitor-not-shutdown-on-threshold-exceeded`
-                                        is enabled, a shutdown is initiated; otherwise
-                                        a warning will be continuously printed out.
-                                        The value should be between 6 and 99.
+                                        Threshold in terms of percentage of
+                                        used space vs total space. If used
+                                        space is above (threshold - 5%), a
+                                        warning is generated.  Unless
+                                        resource-monitor-not-shutdown-on-thresh
+                                        old-exceeded is enabled, a graceful
+                                        shutdown is initiated if used space is
+                                        above the threshold. The value should
+                                        be between 6 and 99
   --resource-monitor-not-shutdown-on-threshold-exceeded
-                                        A switch used to indicate `nodeos` will "not"
-                                        shutdown when threshold is exceeded. When not
-                                        set, `nodeos` will shutdown.
+                                        Used to indicate nodeos will not
+                                        shutdown when threshold is exceeded.
   --resource-monitor-warning-interval arg (=30)
-                                        Number of monitor intervals between which a
-                                        warning is displayed.  For example, if
-                                        `resource-monitor-warning-interval` is to 10
-                                        and `resource-monitor-interval-seconds` is 2,
-                                        a warning will be displayed every 20 seconds,
-                                        even though the space usage is checked every
-                                        2 seconds.  This is used to throttle the
-                                        number of warnings in the `nodeos` log file.
-                                        Should be between 1 and 450.
+                                        Number of resource monitor intervals
+                                        between two consecutive warnings when
+                                        the threshold is hit. Should be between
+                                        1 and 450
 ```
 
 ## Plugin Dependencies

docs/01_nodeos/03_plugins/state_history_plugin/index.md

@@ -31,19 +31,21 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::state_history_plugin:
-
   --state-history-dir arg (="state-history")
-                                        the location of the state-history 
+                                        the location of the state-history
                                         directory (absolute path or relative to
                                         application data dir)
   --trace-history                       enable trace history
   --chain-state-history                 enable chain state history
   --state-history-endpoint arg (=127.0.0.1:8080)
-                                        the endpoint upon which to listen for 
-                                        incoming connections. Caution: only 
-                                        expose this port to your internal 
+                                        the endpoint upon which to listen for
+                                        incoming connections. Caution: only
+                                        expose this port to your internal
                                         network.
   --trace-history-debug-mode            enable debug mode for trace history
+  --state-history-log-retain-blocks arg if set, periodically prune the state
+                                        history files to store only configured
+                                        number of most recent blocks
 ```
 
 ## Examples

eos.doxygen.in

@@ -4,8 +4,8 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
-PROJECT_NAME           = "EOS.IO"
-PROJECT_NUMBER         = "EOSIO ${DOXY_EOS_VERSION}"
+PROJECT_NAME           = "LEAP"
+PROJECT_NUMBER         = "${DOXY_EOS_VERSION}"
 PROJECT_BRIEF          =
 PROJECT_LOGO           = eos-logo.png
 OUTPUT_DIRECTORY       =
@@ -209,9 +209,9 @@ HTML_DYNAMIC_SECTIONS  = NO
 HTML_INDEX_NUM_ENTRIES = 100
 GENERATE_DOCSET        = NO
 DOCSET_FEEDNAME        = "Doxygen generated docs"
-DOCSET_BUNDLE_ID       = io.eos
-DOCSET_PUBLISHER_ID    = one.block
-DOCSET_PUBLISHER_NAME  = block.one
+DOCSET_BUNDLE_ID       = io.antelope
+DOCSET_PUBLISHER_ID    = com.eosnetwork
+DOCSET_PUBLISHER_NAME  = eosnetwork.com
 GENERATE_HTMLHELP      = NO
 CHM_FILE               =
 HHC_LOCATION           =
@@ -221,14 +221,14 @@ BINARY_TOC             = NO
 TOC_EXPAND             = NO
 GENERATE_QHP           = NO
 QCH_FILE               =
-QHP_NAMESPACE          = io.eos
+QHP_NAMESPACE          = io.antelope
 QHP_VIRTUAL_FOLDER     = doc
 QHP_CUST_FILTER_NAME   =
 QHP_CUST_FILTER_ATTRS  =
 QHP_SECT_FILTER_ATTRS  =
 QHG_LOCATION           =
 GENERATE_ECLIPSEHELP   = NO
-ECLIPSE_DOC_ID         = io.eos
+ECLIPSE_DOC_ID         = io.antelope
 DISABLE_INDEX          = NO
 GENERATE_TREEVIEW      = NO
 ENUM_VALUES_PER_LINE   = 4

libraries/chain/controller.cpp

@@ -157,6 +157,7 @@ struct pending_state {
    block_stage_type                   _block_stage;
    controller::block_status           _block_status = controller::block_status::incomplete;
    std::optional<block_id_type>       _producer_block_id;
+   controller::block_report           _block_report{};
 
    /** @pre _block_stage cannot hold completed_block alternative */
    const pending_block_header_state& get_pending_block_header_state()const {
@@ -229,7 +230,7 @@ struct controller_impl {
    protocol_feature_manager        protocol_features;
    controller::config              conf;
    const chain_id_type             chain_id; // read by thread_pool threads, value will not be changed
-   std::optional<fc::time_point>   replay_head_time;
+   bool                            replaying = false;
    db_read_mode                    read_mode = db_read_mode::SPECULATIVE;
    bool                            in_trx_requiring_checks = false; ///< if true, checks that are normally skipped on replay (e.g. auth checks) cannot be skipped
    std::optional<fc::microseconds> subjective_cpu_leeway;
@@ -482,8 +483,7 @@ struct controller_impl {
       }
 
       auto blog_head = blog.head();
-      auto blog_head_time = blog_head ? blog_head->timestamp.to_time_point() : fork_db.root()->header.timestamp.to_time_point();
-      replay_head_time = blog_head_time;
+      replaying = true;
       auto start_block_num = head->block_num + 1;
       auto start = fc::time_point::now();
 
@@ -551,7 +551,7 @@ struct controller_impl {
       ilog( "replayed ${n} blocks in ${duration} seconds, ${mspb} ms/block",
             ("n", head->block_num + 1 - start_block_num)("duration", (end-start).count()/1000000)
             ("mspb", ((end-start).count()/1000.0)/(head->block_num-start_block_num)) );
-      replay_head_time.reset();
+      replaying = false;
 
       if( except_ptr ) {
          std::rethrow_exception( except_ptr );
@@ -820,7 +820,10 @@ struct controller_impl {
       });
    }
 
-   void add_to_snapshot( const snapshot_writer_ptr& snapshot ) const {
+   void add_to_snapshot( const snapshot_writer_ptr& snapshot ) {
+      // clear in case the previous call to clear did not finish in time of deadline
+      clear_expired_input_transactions( fc::time_point::maximum() );
+
       snapshot->write_section<chain_snapshot_header>([this]( auto &section ){
          section.add_row(chain_snapshot_header(), db);
       });
@@ -993,7 +996,7 @@ struct controller_impl {
       );
    }
 
-   sha256 calculate_integrity_hash() const {
+   sha256 calculate_integrity_hash() {
       sha256::encoder enc;
       auto hash_writer = std::make_shared<integrity_hash_snapshot_writer>(enc);
       add_to_snapshot(hash_writer);
@@ -1252,6 +1255,7 @@ struct controller_impl {
                                                      uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time = false )
    { try {
 
+      auto start = fc::time_point::now();
       const bool validating = !self.is_producing_block();
       EOS_ASSERT( !validating || explicit_billed_cpu_time, transaction_exception, "validating requires explicit billing" );
 
@@ -1291,6 +1295,10 @@ struct controller_impl {
          trace->scheduled = true;
          trace->receipt = push_receipt( gtrx.trx_id, transaction_receipt::expired, billed_cpu_time_us, 0 ); // expire the transaction
          trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta );
+         trace->elapsed = fc::time_point::now() - start;
+         pending->_block_report.total_cpu_usage_us += billed_cpu_time_us;
+         pending->_block_report.total_elapsed_time += trace->elapsed;
+         pending->_block_report.total_time += trace->elapsed;
          emit( self.accepted_transaction, trx );
          dmlog_applied_transaction(trace);
          emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) );
@@ -1321,7 +1329,7 @@ struct controller_impl {
          trace->error_code = controller::convert_exception_to_error_code( e );
          trace->except = e;
          trace->except_ptr = std::current_exception();
-         trace->elapsed = fc::time_point::now() - trx_context.start;
+         trace->elapsed = fc::time_point::now() - start;
 
          if (auto dm_logger = get_deep_mind_logger()) {
             dm_logger->on_fail_deferred();
@@ -1365,6 +1373,11 @@ struct controller_impl {
 
          restore.cancel();
 
+         pending->_block_report.total_net_usage += trace->net_usage;
+         pending->_block_report.total_cpu_usage_us += trace->receipt->cpu_usage_us;
+         pending->_block_report.total_elapsed_time += trace->elapsed;
+         pending->_block_report.total_time += fc::time_point::now() - start;
+
          return trace;
       } catch( const disallowed_transaction_extensions_bad_block_exception& ) {
          throw;
@@ -1395,13 +1408,18 @@ struct controller_impl {
          trace = error_trace;
          if( !trace->except_ptr ) {
             trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta );
+            trace->elapsed = fc::time_point::now() - start;
             emit( self.accepted_transaction, trx );
             dmlog_applied_transaction(trace);
             emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) );
             undo_session.squash();
+            pending->_block_report.total_net_usage += trace->net_usage;
+            if( trace->receipt ) pending->_block_report.total_cpu_usage_us += trace->receipt->cpu_usage_us;
+            pending->_block_report.total_elapsed_time += trace->elapsed;
+            pending->_block_report.total_time += trace->elapsed;
             return trace;
          }
-         trace->elapsed = fc::time_point::now() - trx_context.start;
+         trace->elapsed = fc::time_point::now() - start;
       }
 
       // Only subjective OR hard failure logic below:
@@ -1448,6 +1466,11 @@ struct controller_impl {
          emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) );
       }
 
+      pending->_block_report.total_net_usage += trace->net_usage;
+      if( trace->receipt ) pending->_block_report.total_cpu_usage_us += trace->receipt->cpu_usage_us;
+      pending->_block_report.total_elapsed_time += trace->elapsed;
+      pending->_block_report.total_time += fc::time_point::now() - start;
+
       return trace;
    } FC_CAPTURE_AND_RETHROW() } /// push_scheduled_transaction
 
@@ -1528,10 +1551,8 @@ struct controller_impl {
                trx_context.init_for_implicit_trx();
                trx_context.enforce_whiteblacklist = false;
             } else {
-               bool skip_recording = replay_head_time && (time_point(trn.expiration) < *replay_head_time);
                trx_context.init_for_input_trx( trx->packed_trx()->get_unprunable_size(),
-                                               trx->packed_trx()->get_prunable_size(),
-                                               skip_recording);
+                                               trx->packed_trx()->get_prunable_size() );
             }
 
             trx_context.delay = fc::seconds(trn.delay_sec);
@@ -1590,6 +1611,13 @@ struct controller_impl {
                trx_context.squash();
             }
 
+            if( !trx->read_only ) {
+               pending->_block_report.total_net_usage += trace->net_usage;
+               pending->_block_report.total_cpu_usage_us += trace->receipt->cpu_usage_us;
+               pending->_block_report.total_elapsed_time += trace->elapsed;
+               pending->_block_report.total_time += fc::time_point::now() - start;
+            }
+
             return trace;
          } catch( const disallowed_transaction_extensions_bad_block_exception& ) {
             throw;
@@ -1607,9 +1635,14 @@ struct controller_impl {
          }
 
          if (!trx->read_only) {
-             emit(self.accepted_transaction, trx);
-             dmlog_applied_transaction(trace);
-             emit(self.applied_transaction, std::tie(trace, trx->packed_trx()));
+            emit(self.accepted_transaction, trx);
+            dmlog_applied_transaction(trace);
+            emit(self.applied_transaction, std::tie(trace, trx->packed_trx()));
+
+            pending->_block_report.total_net_usage += trace->net_usage;
+            if( trace->receipt ) pending->_block_report.total_cpu_usage_us += trace->receipt->cpu_usage_us;
+            pending->_block_report.total_elapsed_time += trace->elapsed;
+            pending->_block_report.total_time += fc::time_point::now() - start;
          }
 
          return trace;
@@ -1729,7 +1762,7 @@ struct controller_impl {
          )
          {
             // Promote proposed schedule to pending schedule.
-            if( !replay_head_time ) {
+            if( !replaying ) {
                ilog( "promoting proposed schedule (set in block ${proposed_num}) to pending; current block: ${n} lib: ${lib} schedule: ${schedule} ",
                      ("proposed_num", *gpo.proposed_schedule_block_num)("n", pbhs.block_num)
                      ("lib", pbhs.dpos_irreversible_blocknum)
@@ -2063,12 +2096,6 @@ struct controller_impl {
             EOS_ASSERT( r == static_cast<const transaction_receipt_header&>(receipt),
                         block_validate_exception, "receipt does not match, ${lhs} != ${rhs}",
                         ("lhs", r)("rhs", static_cast<const transaction_receipt_header&>(receipt)) );
-
-            if( trace ) {
-               br.total_net_usage += trace->net_usage;
-               if( trace->receipt ) br.total_cpu_usage_us += trace->receipt->cpu_usage_us;
-               br.total_elapsed_time += trace->elapsed;
-            }
          }
 
          finalize_block();
@@ -2089,6 +2116,7 @@ struct controller_impl {
          // create completed_block with the existing block_state as we just verified it is the same as assembled_block
          pending->_block_stage = completed_block{ bsp };
 
+         br = pending->_block_report; // copy before commit block destroys pending
          commit_block(false);
          br.total_time = fc::time_point::now() - start;
          return;
@@ -2107,44 +2135,65 @@ struct controller_impl {
       }
    } FC_CAPTURE_AND_RETHROW() } /// apply_block
 
+
+   // thread safe, expected to be called from thread other than the main thread
+   block_state_ptr create_block_state_i( const block_id_type& id, const signed_block_ptr& b, const block_header_state& prev ) {
+      auto trx_mroot = calculate_trx_merkle( b->transactions );
+      EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception,
+                  "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) );
+
+      const bool skip_validate_signee = false;
+      auto bsp = std::make_shared<block_state>(
+            prev,
+            b,
+            protocol_features.get_protocol_feature_set(),
+            [this]( block_timestamp_type timestamp,
+                    const flat_set<digest_type>& cur_features,
+                    const vector<digest_type>& new_features )
+            { check_protocol_features( timestamp, cur_features, new_features ); },
+            skip_validate_signee
+      );
+
+      EOS_ASSERT( id == bsp->id, block_validate_exception,
+                  "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id) );
+      return bsp;
+   }
+
    std::future<block_state_ptr> create_block_state_future( const block_id_type& id, const signed_block_ptr& b ) {
       EOS_ASSERT( b, block_validate_exception, "null block" );
 
+      return async_thread_pool( thread_pool.get_executor(), [b, id, control=this]() {
+         // no reason for a block_state if fork_db already knows about block
+         auto existing = control->fork_db.get_block( id );
+         EOS_ASSERT( !existing, fork_database_exception, "we already know about this block: ${id}", ("id", id) );
+
+         auto prev = control->fork_db.get_block_header( b->previous );
+         EOS_ASSERT( prev, unlinkable_block_exception,
+                     "unlinkable block ${id}", ("id", id)("previous", b->previous) );
+
+         return control->create_block_state_i( id, b, *prev );
+      } );
+   }
+
+   // thread safe, expected to be called from thread other than the main thread
+   block_state_ptr create_block_state( const block_id_type& id, const signed_block_ptr& b ) {
+      EOS_ASSERT( b, block_validate_exception, "null block" );
+
       // no reason for a block_state if fork_db already knows about block
       auto existing = fork_db.get_block( id );
       EOS_ASSERT( !existing, fork_database_exception, "we already know about this block: ${id}", ("id", id) );
 
+      // previous not found could mean that previous block not applied yet
       auto prev = fork_db.get_block_header( b->previous );
-      EOS_ASSERT( prev, unlinkable_block_exception,
-                  "unlinkable block ${id}", ("id", id)("previous", b->previous) );
+      if( !prev ) return {};
 
-      return async_thread_pool( thread_pool.get_executor(), [b, prev, id, control=this]() {
-         const bool skip_validate_signee = false;
-
-         auto trx_mroot = calculate_trx_merkle( b->transactions );
-         EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception,
-                     "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) );
-
-         auto bsp = std::make_shared<block_state>(
-                        *prev,
-                        move( b ),
-                        control->protocol_features.get_protocol_feature_set(),
-                        [control]( block_timestamp_type timestamp,
-                                   const flat_set<digest_type>& cur_features,
-                                   const vector<digest_type>& new_features )
-                        { control->check_protocol_features( timestamp, cur_features, new_features ); },
-                        skip_validate_signee
-         );
-
-         EOS_ASSERT( id == bsp->id, block_validate_exception,
-                     "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id) );
-         return bsp;
-      } );
+      return create_block_state_i( id, b, *prev );
    }
 
    void push_block( controller::block_report& br,
-                    std::future<block_state_ptr>& block_state_future,
-                    const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup )
+                    const block_state_ptr& bsp,
+                    const forked_branch_callback& forked_branch_cb,
+                    const trx_meta_cache_lookup& trx_lookup )
    {
       controller::block_status s = controller::block_status::complete;
       EOS_ASSERT(!pending, block_validate_exception, "it is not valid to push a block when there is a pending block");
@@ -2153,7 +2202,7 @@ struct controller_impl {
          trusted_producer_light_validation = old_value;
       });
       try {
-         block_state_ptr bsp = block_state_future.get();
+         EOS_ASSERT( bsp, block_validate_exception, "null block" );
          const auto& b = bsp->block;
 
          if( conf.terminate_at_block > 0 && conf.terminate_at_block < self.head_block_num()) {
@@ -2394,7 +2443,7 @@ struct controller_impl {
       //Look for expired transactions in the deduplication list, and remove them.
       auto& transaction_idx = db.get_mutable_index<transaction_multi_index>();
       const auto& dedupe_index = transaction_idx.indices().get<by_expiration>();
-      auto now = self.pending_block_time();
+      auto now = self.is_building_block() ? self.pending_block_time() : self.head_block_time();
       const auto total = dedupe_index.size();
       uint32_t num_removed = 0;
       while( (!dedupe_index.empty()) && ( now > fc::time_point(dedupe_index.begin()->expiration) ) ) {
@@ -2829,7 +2878,7 @@ void controller::start_block( block_timestamp_type when,
                     block_status::incomplete, std::optional<block_id_type>(), deadline );
 }
 
-block_state_ptr controller::finalize_block( const signer_callback_type& signer_callback ) {
+block_state_ptr controller::finalize_block( block_report& br, const signer_callback_type& signer_callback ) {
    validate_db_available_size();
 
    my->finalize_block();
@@ -2850,6 +2899,8 @@ block_state_ptr controller::finalize_block( const signer_callback_type& signer_c
 
    my->pending->_block_stage = completed_block{ bsp };
 
+   br = my->pending->_block_report;
+
    return bsp;
 }
 
@@ -2870,12 +2921,17 @@ std::future<block_state_ptr> controller::create_block_state_future( const block_
    return my->create_block_state_future( id, b );
 }
 
+block_state_ptr controller::create_block_state( const block_id_type& id, const signed_block_ptr& b ) const {
+   return my->create_block_state( id, b );
+}
+
 void controller::push_block( controller::block_report& br,
-                             std::future<block_state_ptr>& block_state_future,
-                             const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup )
+                             const block_state_ptr& bsp,
+                             const forked_branch_callback& forked_branch_cb,
+                             const trx_meta_cache_lookup& trx_lookup )
 {
    validate_db_available_size();
-   my->push_block( br, block_state_future, forked_branch_cb, trx_lookup );
+   my->push_block( br, bsp, forked_branch_cb, trx_lookup );
 }
 
 transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx,
@@ -3104,11 +3160,11 @@ block_id_type controller::get_block_id_for_num( uint32_t block_num )const { try
    return id;
 } FC_CAPTURE_AND_RETHROW( (block_num) ) }
 
-sha256 controller::calculate_integrity_hash()const { try {
+sha256 controller::calculate_integrity_hash() { try {
    return my->calculate_integrity_hash();
 } FC_LOG_AND_RETHROW() }
 
-void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) const {
+void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) {
    EOS_ASSERT( !my->pending, block_validate_exception, "cannot take a consistent snapshot with a pending block" );
    return my->add_to_snapshot(snapshot);
 }

libraries/chain/fork_database.cpp

@@ -9,6 +9,7 @@
 #include <boost/multi_index/composite_key.hpp>
 #include <fc/io/fstream.hpp>
 #include <fstream>
+#include <shared_mutex>
 
 namespace eosio { namespace chain {
    using boost::multi_index_container;
@@ -60,27 +61,44 @@ namespace eosio { namespace chain {
    }
 
    struct fork_database_impl {
-      fork_database_impl( fork_database& self, const fc::path& data_dir )
-      :self(self)
-      ,datadir(data_dir)
+      explicit fork_database_impl( const fc::path& data_dir )
+      :datadir(data_dir)
       {}
 
-      fork_database&        self;
+      std::shared_mutex     mtx;
       fork_multi_index_type index;
       block_state_ptr       root; // Only uses the block_header_state portion
       block_state_ptr       head;
       fc::path              datadir;
 
-      void add( const block_state_ptr& n,
-                bool ignore_duplicate, bool validate,
-                const std::function<void( block_timestamp_type,
-                                          const flat_set<digest_type>&,
-                                          const vector<digest_type>& )>& validator );
+      void open_impl( const std::function<void( block_timestamp_type,
+                                                const flat_set<digest_type>&,
+                                                const vector<digest_type>& )>& validator );
+      void close_impl();
+
+
+      block_header_state_ptr  get_block_header_impl( const block_id_type& id )const;
+      block_state_ptr         get_block_impl( const block_id_type& id )const;
+      void            reset_impl( const block_header_state& root_bhs );
+      void            rollback_head_to_root_impl();
+      void            advance_root_impl( const block_id_type& id );
+      void            remove_impl( const block_id_type& id );
+      branch_type     fetch_branch_impl( const block_id_type& h, uint32_t trim_after_block_num )const;
+      block_state_ptr search_on_branch_impl( const block_id_type& h, uint32_t block_num )const;
+      pair<branch_type, branch_type> fetch_branch_from_impl( const block_id_type& first,
+                                                             const block_id_type& second )const;
+      void mark_valid_impl( const block_state_ptr& h );
+
+      void add_impl( const block_state_ptr& n,
+                     bool ignore_duplicate, bool validate,
+                     const std::function<void( block_timestamp_type,
+                                               const flat_set<digest_type>&,
+                                               const vector<digest_type>& )>& validator );
    };
 
 
    fork_database::fork_database( const fc::path& data_dir )
-   :my( new fork_database_impl( *this, data_dir ) )
+   :my( new fork_database_impl( data_dir ) )
    {}
 
 
@@ -88,10 +106,18 @@ namespace eosio { namespace chain {
                                                        const flat_set<digest_type>&,
                                                        const vector<digest_type>& )>& validator )
    {
-      if (!fc::is_directory(my->datadir))
-         fc::create_directories(my->datadir);
+      std::lock_guard g( my->mtx );
+      my->open_impl( validator );
+   }
 
-      auto fork_db_dat = my->datadir / config::forkdb_filename;
+   void fork_database_impl::open_impl( const std::function<void( block_timestamp_type,
+                                                                 const flat_set<digest_type>&,
+                                                                 const vector<digest_type>& )>& validator )
+   {
+      if (!fc::is_directory(datadir))
+         fc::create_directories(datadir);
+
+      auto fork_db_dat = datadir / config::forkdb_filename;
       if( fc::exists( fork_db_dat ) ) {
          try {
             string content;
@@ -102,29 +128,29 @@ namespace eosio { namespace chain {
             // validate totem
             uint32_t totem = 0;
             fc::raw::unpack( ds, totem );
-            EOS_ASSERT( totem == magic_number, fork_database_exception,
+            EOS_ASSERT( totem == fork_database::magic_number, fork_database_exception,
                         "Fork database file '${filename}' has unexpected magic number: ${actual_totem}. Expected ${expected_totem}",
                         ("filename", fork_db_dat.generic_string())
                         ("actual_totem", totem)
-                        ("expected_totem", magic_number)
+                        ("expected_totem", fork_database::magic_number)
             );
 
             // validate version
             uint32_t version = 0;
             fc::raw::unpack( ds, version );
-            EOS_ASSERT( version >= min_supported_version && version <= max_supported_version,
+            EOS_ASSERT( version >= fork_database::min_supported_version && version <= fork_database::max_supported_version,
                         fork_database_exception,
                        "Unsupported version of fork database file '${filename}'. "
                        "Fork database version is ${version} while code supports version(s) [${min},${max}]",
                        ("filename", fork_db_dat.generic_string())
                        ("version", version)
-                       ("min", min_supported_version)
-                       ("max", max_supported_version)
+                       ("min", fork_database::min_supported_version)
+                       ("max", fork_database::max_supported_version)
             );
 
             block_header_state bhs;
             fc::raw::unpack( ds, bhs );
-            reset( bhs );
+            reset_impl( bhs );
 
             unsigned_int size; fc::raw::unpack( ds, size );
             for( uint32_t i = 0, n = size.value; i < n; ++i ) {
@@ -132,27 +158,27 @@ namespace eosio { namespace chain {
                fc::raw::unpack( ds, s );
                // do not populate transaction_metadatas, they will be created as needed in apply_block with appropriate key recovery
                s.header_exts = s.block->validate_and_extract_header_extensions();
-               my->add( std::make_shared<block_state>( move( s ) ), false, true, validator );
+               add_impl( std::make_shared<block_state>( std::move( s ) ), false, true, validator );
             }
             block_id_type head_id;
             fc::raw::unpack( ds, head_id );
 
-            if( my->root->id == head_id ) {
-               my->head = my->root;
+            if( root->id == head_id ) {
+               head = root;
             } else {
-               my->head = get_block( head_id );
-               EOS_ASSERT( my->head, fork_database_exception,
+               head = get_block_impl( head_id );
+               EOS_ASSERT( head, fork_database_exception,
                            "could not find head while reconstructing fork database from file; '${filename}' is likely corrupted",
                            ("filename", fork_db_dat.generic_string()) );
             }
 
-            auto candidate = my->index.get<by_lib_block_num>().begin();
-            if( candidate == my->index.get<by_lib_block_num>().end() || !(*candidate)->is_valid() ) {
-               EOS_ASSERT( my->head->id == my->root->id, fork_database_exception,
+            auto candidate = index.get<by_lib_block_num>().begin();
+            if( candidate == index.get<by_lib_block_num>().end() || !(*candidate)->is_valid() ) {
+               EOS_ASSERT( head->id == root->id, fork_database_exception,
                            "head not set to root despite no better option available; '${filename}' is likely corrupted",
                            ("filename", fork_db_dat.generic_string()) );
             } else {
-               EOS_ASSERT( !first_preferred( **candidate, *my->head ), fork_database_exception,
+               EOS_ASSERT( !first_preferred( **candidate, *head ), fork_database_exception,
                            "head not set to best available option available; '${filename}' is likely corrupted",
                            ("filename", fork_db_dat.generic_string()) );
             }
@@ -163,10 +189,15 @@ namespace eosio { namespace chain {
    }
 
    void fork_database::close() {
-      auto fork_db_dat = my->datadir / config::forkdb_filename;
+      std::lock_guard g( my->mtx );
+      my->close_impl();
+   }
 
-      if( !my->root ) {
-         if( my->index.size() > 0 ) {
+   void fork_database_impl::close_impl() {
+      auto fork_db_dat = datadir / config::forkdb_filename;
+
+      if( !root ) {
+         if( index.size() > 0 ) {
             elog( "fork_database is in a bad state when closing; not writing out '${filename}'",
                   ("filename", fork_db_dat.generic_string()) );
          }
@@ -174,13 +205,13 @@ namespace eosio { namespace chain {
       }
 
       std::ofstream out( fork_db_dat.generic_string().c_str(), std::ios::out | std::ios::binary | std::ofstream::trunc );
-      fc::raw::pack( out, magic_number );
-      fc::raw::pack( out, max_supported_version ); // write out current version which is always max_supported_version
-      fc::raw::pack( out, *static_cast<block_header_state*>(&*my->root) );
-      uint32_t num_blocks_in_fork_db = my->index.size();
+      fc::raw::pack( out, fork_database::magic_number );
+      fc::raw::pack( out, fork_database::max_supported_version ); // write out current version which is always max_supported_version
+      fc::raw::pack( out, *static_cast<block_header_state*>(&*root) );
+      uint32_t num_blocks_in_fork_db = index.size();
       fc::raw::pack( out, unsigned_int{num_blocks_in_fork_db} );
 
-      const auto& indx = my->index.get<by_lib_block_num>();
+      const auto& indx = index.get<by_lib_block_num>();
 
       auto unvalidated_itr = indx.rbegin();
       auto unvalidated_end = boost::make_reverse_iterator( indx.lower_bound( false ) );
@@ -215,30 +246,40 @@ namespace eosio { namespace chain {
          fc::raw::pack( out, *(*itr) );
       }
 
-      if( my->head ) {
-         fc::raw::pack( out, my->head->id );
+      if( head ) {
+         fc::raw::pack( out, head->id );
       } else {
          elog( "head not set in fork database; '${filename}' will be corrupted",
                ("filename", fork_db_dat.generic_string()) );
       }
 
-      my->index.clear();
+      index.clear();
    }
 
    fork_database::~fork_database() {
-      close();
+      my->close_impl();
    }
 
    void fork_database::reset( const block_header_state& root_bhs ) {
-      my->index.clear();
-      my->root = std::make_shared<block_state>();
-      static_cast<block_header_state&>(*my->root) = root_bhs;
-      my->root->validated = true;
-      my->head = my->root;
+      std::lock_guard g( my->mtx );
+      my->reset_impl(root_bhs);
+   }
+
+   void fork_database_impl::reset_impl( const block_header_state& root_bhs ) {
+      index.clear();
+      root = std::make_shared<block_state>();
+      static_cast<block_header_state&>(*root) = root_bhs;
+      root->validated = true;
+      head = root;
    }
 
    void fork_database::rollback_head_to_root() {
-      auto& by_id_idx = my->index.get<by_block_id>();
+      std::lock_guard g( my->mtx );
+      my->rollback_head_to_root_impl();
+   }
+
+   void fork_database_impl::rollback_head_to_root_impl() {
+      auto& by_id_idx = index.get<by_block_id>();
       auto itr = by_id_idx.begin();
       while (itr != by_id_idx.end()) {
          by_id_idx.modify( itr, [&]( block_state_ptr& bsp ) {
@@ -246,13 +287,18 @@ namespace eosio { namespace chain {
          } );
          ++itr;
       }
-      my->head = my->root;
+      head = root;
    }
 
    void fork_database::advance_root( const block_id_type& id ) {
-      EOS_ASSERT( my->root, fork_database_exception, "root not yet set" );
+      std::lock_guard g( my->mtx );
+      my->advance_root_impl( id );
+   }
 
-      auto new_root = get_block( id );
+   void fork_database_impl::advance_root_impl( const block_id_type& id ) {
+      EOS_ASSERT( root, fork_database_exception, "root not yet set" );
+
+      auto new_root = get_block_impl( id );
       EOS_ASSERT( new_root, fork_database_exception,
                   "cannot advance root to a block that does not exist in the fork database" );
       EOS_ASSERT( new_root->is_valid(), fork_database_exception,
@@ -262,48 +308,53 @@ namespace eosio { namespace chain {
       deque<block_id_type> blocks_to_remove;
       for( auto b = new_root; b; ) {
          blocks_to_remove.emplace_back( b->header.previous );
-         b = get_block( blocks_to_remove.back() );
-         EOS_ASSERT( b || blocks_to_remove.back() == my->root->id, fork_database_exception, "invariant violation: orphaned branch was present in forked database" );
+         b = get_block_impl( blocks_to_remove.back() );
+         EOS_ASSERT( b || blocks_to_remove.back() == root->id, fork_database_exception, "invariant violation: orphaned branch was present in forked database" );
       }
 
       // The new root block should be erased from the fork database index individually rather than with the remove method,
       // because we do not want the blocks branching off of it to be removed from the fork database.
-      my->index.erase( my->index.find( id ) );
+      index.erase( index.find( id ) );
 
       // The other blocks to be removed are removed using the remove method so that orphaned branches do not remain in the fork database.
       for( const auto& block_id : blocks_to_remove ) {
-         remove( block_id );
+         remove_impl( block_id );
       }
 
       // Even though fork database no longer needs block or trxs when a block state becomes a root of the tree,
       // avoid mutating the block state at all, for example clearing the block shared pointer, because other
       // parts of the code which run asynchronously may later expect it remain unmodified.
 
-      my->root = new_root;
+      root = new_root;
    }
 
    block_header_state_ptr fork_database::get_block_header( const block_id_type& id )const {
-      if( my->root->id == id ) {
-         return my->root;
+      std::shared_lock g( my->mtx );
+      return my->get_block_header_impl( id );
+   }
+
+   block_header_state_ptr fork_database_impl::get_block_header_impl( const block_id_type& id )const {
+      if( root->id == id ) {
+         return root;
       }
 
-      auto itr = my->index.find( id );
-      if( itr != my->index.end() )
+      auto itr = index.find( id );
+      if( itr != index.end() )
          return *itr;
 
       return block_header_state_ptr();
    }
 
-   void fork_database_impl::add( const block_state_ptr& n,
-                                 bool ignore_duplicate, bool validate,
-                                 const std::function<void( block_timestamp_type,
-                                                           const flat_set<digest_type>&,
-                                                           const vector<digest_type>& )>& validator )
+   void fork_database_impl::add_impl( const block_state_ptr& n,
+                                      bool ignore_duplicate, bool validate,
+                                      const std::function<void( block_timestamp_type,
+                                                                const flat_set<digest_type>&,
+                                                                const vector<digest_type>& )>& validator )
    {
       EOS_ASSERT( root, fork_database_exception, "root not yet set" );
       EOS_ASSERT( n, fork_database_exception, "attempt to add null block state" );
 
-      auto prev_bh = self.get_block_header( n->header.previous );
+      auto prev_bh = get_block_header_impl( n->header.previous );
 
       EOS_ASSERT( prev_bh, unlinkable_block_exception,
                   "unlinkable block", ("id", n->id)("previous", n->header.previous) );
@@ -332,19 +383,27 @@ namespace eosio { namespace chain {
    }
 
    void fork_database::add( const block_state_ptr& n, bool ignore_duplicate ) {
-      my->add( n, ignore_duplicate, false,
-               []( block_timestamp_type timestamp,
-                   const flat_set<digest_type>& cur_features,
-                   const vector<digest_type>& new_features )
-               {}
+      std::lock_guard g( my->mtx );
+      my->add_impl( n, ignore_duplicate, false,
+                    []( block_timestamp_type timestamp,
+                        const flat_set<digest_type>& cur_features,
+                        const vector<digest_type>& new_features )
+                    {}
       );
    }
 
-   const block_state_ptr& fork_database::root()const { return my->root; }
+   block_state_ptr fork_database::root()const {
+      std::shared_lock g( my->mtx );
+      return my->root;
+   }
 
-   const block_state_ptr& fork_database::head()const { return my->head; }
+   block_state_ptr fork_database::head()const {
+      std::shared_lock g( my->mtx );
+      return my->head;
+   }
 
    block_state_ptr fork_database::pending_head()const {
+      std::shared_lock g( my->mtx );
       const auto& indx = my->index.get<by_lib_block_num>();
 
       auto itr = indx.lower_bound( false );
@@ -357,8 +416,13 @@ namespace eosio { namespace chain {
    }
 
    branch_type fork_database::fetch_branch( const block_id_type& h, uint32_t trim_after_block_num )const {
+      std::shared_lock g( my->mtx );
+      return my->fetch_branch_impl( h, trim_after_block_num );
+   }
+
+   branch_type fork_database_impl::fetch_branch_impl( const block_id_type& h, uint32_t trim_after_block_num )const {
       branch_type result;
-      for( auto s = get_block(h); s; s = get_block( s->header.previous ) ) {
+      for( auto s = get_block_impl(h); s; s = get_block_impl( s->header.previous ) ) {
          if( s->block_num <= trim_after_block_num )
              result.push_back( s );
       }
@@ -367,7 +431,12 @@ namespace eosio { namespace chain {
    }
 
    block_state_ptr fork_database::search_on_branch( const block_id_type& h, uint32_t block_num )const {
-      for( auto s = get_block(h); s; s = get_block( s->header.previous ) ) {
+      std::shared_lock g( my->mtx );
+      return my->search_on_branch_impl( h, block_num );
+   }
+
+   block_state_ptr fork_database_impl::search_on_branch_impl( const block_id_type& h, uint32_t block_num )const {
+      for( auto s = get_block_impl(h); s; s = get_block_impl( s->header.previous ) ) {
          if( s->block_num == block_num )
              return s;
       }
@@ -381,9 +450,15 @@ namespace eosio { namespace chain {
     */
    pair< branch_type, branch_type >  fork_database::fetch_branch_from( const block_id_type& first,
                                                                        const block_id_type& second )const {
+      std::shared_lock g( my->mtx );
+      return my->fetch_branch_from_impl( first, second );
+   }
+
+   pair< branch_type, branch_type >  fork_database_impl::fetch_branch_from_impl( const block_id_type& first,
+                                                                                 const block_id_type& second )const {
       pair<branch_type,branch_type> result;
-      auto first_branch = (first == my->root->id) ? my->root : get_block(first);
-      auto second_branch = (second == my->root->id) ? my->root : get_block(second);
+      auto first_branch = (first == root->id) ? root : get_block_impl(first);
+      auto second_branch = (second == root->id) ? root : get_block_impl(second);
 
       EOS_ASSERT(first_branch, fork_db_block_not_found, "block ${id} does not exist", ("id", first));
       EOS_ASSERT(second_branch, fork_db_block_not_found, "block ${id} does not exist", ("id", second));
@@ -392,7 +467,7 @@ namespace eosio { namespace chain {
       {
          result.first.push_back(first_branch);
          const auto& prev = first_branch->header.previous;
-         first_branch = (prev == my->root->id) ? my->root : get_block( prev );
+         first_branch = (prev == root->id) ? root : get_block_impl( prev );
          EOS_ASSERT( first_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", prev)
@@ -403,7 +478,7 @@ namespace eosio { namespace chain {
       {
          result.second.push_back( second_branch );
          const auto& prev = second_branch->header.previous;
-         second_branch = (prev == my->root->id) ? my->root : get_block( prev );
+         second_branch = (prev == root->id) ? root : get_block_impl( prev );
          EOS_ASSERT( second_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", prev)
@@ -417,9 +492,9 @@ namespace eosio { namespace chain {
          result.first.push_back(first_branch);
          result.second.push_back(second_branch);
          const auto &first_prev = first_branch->header.previous;
-         first_branch = get_block( first_prev );
+         first_branch = get_block_impl( first_prev );
          const auto &second_prev = second_branch->header.previous;
-         second_branch = get_block( second_prev );
+         second_branch = get_block_impl( second_prev );
          EOS_ASSERT( first_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", first_prev)
@@ -436,13 +511,18 @@ namespace eosio { namespace chain {
          result.second.push_back(second_branch);
       }
       return result;
-   } /// fetch_branch_from
+   } /// fetch_branch_from_impl
 
    /// remove all of the invalid forks built off of this id including this id
    void fork_database::remove( const block_id_type& id ) {
+      std::lock_guard g( my->mtx );
+      return my->remove_impl( id );
+   }
+
+   void fork_database_impl::remove_impl( const block_id_type& id ) {
       deque<block_id_type> remove_queue{id};
-      const auto& previdx = my->index.get<by_prev>();
-      const auto& head_id = my->head->id;
+      const auto& previdx = index.get<by_prev>();
+      const auto& head_id = head->id;
 
       for( uint32_t i = 0; i < remove_queue.size(); ++i ) {
          EOS_ASSERT( remove_queue[i] != head_id, fork_database_exception,
@@ -456,16 +536,19 @@ namespace eosio { namespace chain {
       }
 
       for( const auto& block_id : remove_queue ) {
-         auto itr = my->index.find( block_id );
-         if( itr != my->index.end() )
-            my->index.erase(itr);
+         index.erase( block_id );
       }
    }
 
    void fork_database::mark_valid( const block_state_ptr& h ) {
+      std::lock_guard g( my->mtx );
+      my->mark_valid_impl( h );
+   }
+
+   void fork_database_impl::mark_valid_impl( const block_state_ptr& h ) {
       if( h->validated ) return;
 
-      auto& by_id_idx = my->index.get<by_block_id>();
+      auto& by_id_idx = index.get<by_block_id>();
 
       auto itr = by_id_idx.find( h->id );
       EOS_ASSERT( itr != by_id_idx.end(), fork_database_exception,
@@ -476,15 +559,20 @@ namespace eosio { namespace chain {
          bsp->validated = true;
       } );
 
-      auto candidate = my->index.get<by_lib_block_num>().begin();
-      if( first_preferred( **candidate, *my->head ) ) {
-         my->head = *candidate;
+      auto candidate = index.get<by_lib_block_num>().begin();
+      if( first_preferred( **candidate, *head ) ) {
+         head = *candidate;
       }
    }
 
-   block_state_ptr   fork_database::get_block(const block_id_type& id)const {
-      auto itr = my->index.find( id );
-      if( itr != my->index.end() )
+   block_state_ptr fork_database::get_block(const block_id_type& id)const {
+      std::shared_lock g( my->mtx );
+      return my->get_block_impl(id);
+   }
+
+   block_state_ptr fork_database_impl::get_block_impl(const block_id_type& id)const {
+      auto itr = index.find( id );
+      if( itr != index.end() )
          return *itr;
       return block_state_ptr();
    }

libraries/chain/include/eosio/chain/controller.hpp

@@ -164,26 +164,30 @@ namespace eosio { namespace chain {
                                                            fc::time_point block_deadline, fc::microseconds max_transaction_time,
                                                            uint32_t billed_cpu_time_us, bool explicit_billed_cpu_time );
 
-         block_state_ptr finalize_block( const signer_callback_type& signer_callback );
-         void sign_block( const signer_callback_type& signer_callback );
-         void commit_block();
-
-         std::future<block_state_ptr> create_block_state_future( const block_id_type& id, const signed_block_ptr& b );
-
          struct block_report {
             size_t             total_net_usage = 0;
             size_t             total_cpu_usage_us = 0;
             fc::microseconds   total_elapsed_time{};
             fc::microseconds   total_time{};
          };
+
+         block_state_ptr finalize_block( block_report& br, const signer_callback_type& signer_callback );
+         void sign_block( const signer_callback_type& signer_callback );
+         void commit_block();
+
+         // thread-safe
+         std::future<block_state_ptr> create_block_state_future( const block_id_type& id, const signed_block_ptr& b );
+         // thread-safe
+         block_state_ptr create_block_state( const block_id_type& id, const signed_block_ptr& b ) const;
+
          /**
           * @param br returns statistics for block
-          * @param block_state_future provide from call to create_block_state_future
+          * @param bsp block to push
           * @param cb calls cb with forked applied transactions for each forked block
           * @param trx_lookup user provided lookup function for externally cached transaction_metadata
           */
          void push_block( block_report& br,
-                          std::future<block_state_ptr>& block_state_future,
+                          const block_state_ptr& bsp,
                           const forked_branch_callback& cb,
                           const trx_meta_cache_lookup& trx_lookup );
 
@@ -252,12 +256,13 @@ namespace eosio { namespace chain {
          signed_block_ptr fetch_block_by_id( block_id_type id )const;
 
          block_state_ptr fetch_block_state_by_number( uint32_t block_num )const;
+         // return block_state from forkdb, thread-safe
          block_state_ptr fetch_block_state_by_id( block_id_type id )const;
 
          block_id_type get_block_id_for_num( uint32_t block_num )const;
 
-         sha256 calculate_integrity_hash()const;
-         void write_snapshot( const snapshot_writer_ptr& snapshot )const;
+         sha256 calculate_integrity_hash();
+         void write_snapshot( const snapshot_writer_ptr& snapshot );
 
          bool sender_avoids_whitelist_blacklist_enforcement( account_name sender )const;
          void check_actor_list( const flat_set<account_name>& actors )const;

libraries/chain/include/eosio/chain/fork_database.hpp

@@ -16,6 +16,8 @@ namespace eosio { namespace chain {
     * database tracks the longest chain and the last irreversible block number. All
     * blocks older than the last irreversible block are freed after emitting the
     * irreversible signal.
+    *
+    * An internal mutex is used to provide thread-safety.
     */
    class fork_database {
       public:
@@ -55,9 +57,9 @@ namespace eosio { namespace chain {
 
          void            remove( const block_id_type& id );
 
-         const block_state_ptr& root()const;
-         const block_state_ptr& head()const;
-         block_state_ptr        pending_head()const;
+         block_state_ptr root()const;
+         block_state_ptr head()const;
+         block_state_ptr pending_head()const;
 
          /**
           *  Returns the sequence of block states resulting from trimming the branch from the

libraries/chain/include/eosio/chain/thread_utils.hpp

@@ -1,11 +1,11 @@
 #pragma once
 
 #include <boost/asio/io_context.hpp>
-#include <boost/asio/thread_pool.hpp>
 #include <boost/asio/post.hpp>
 #include <future>
 #include <memory>
 #include <optional>
+#include <thread>
 
 namespace eosio { namespace chain {
 
@@ -17,20 +17,28 @@ namespace eosio { namespace chain {
    public:
       // name_prefix is name appended with -## of thread.
       // short name_prefix (6 chars or under) is recommended as console_appender uses 9 chars for thread name
-      named_thread_pool( std::string name_prefix, size_t num_threads );
+      /// @param delay_start do not spawn threads in constructor, wait for start() call
+      named_thread_pool( std::string name_prefix, size_t num_threads, bool delay_start = false );
 
       // calls stop()
       ~named_thread_pool();
 
       boost::asio::io_context& get_executor() { return _ioc; }
 
-      // destroy work guard, stop io_context, join thread_pool, and stop thread_pool
+      /// Spawn threads, can be re-started after stop().
+      /// Assumes start()/stop() called from the same thread or externally protected.
+      /// @throw assert_exception if already started and not stopped.
+      void start();
+
+      /// destroy work guard, stop io_context, join thread_pool
       void stop();
 
    private:
       using ioc_work_t = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
 
-      boost::asio::thread_pool       _thread_pool;
+      std::string                    _name_prefix;
+      size_t                         _num_threads;
+      std::vector<std::thread>       _thread_pool;
       boost::asio::io_context        _ioc;
       std::optional<ioc_work_t>      _ioc_work;
    };

libraries/chain/include/eosio/chain/transaction_context.hpp

@@ -45,8 +45,7 @@ namespace eosio { namespace chain {
          void init_for_implicit_trx( uint64_t initial_net_usage = 0 );
 
          void init_for_input_trx( uint64_t packed_trx_unprunable_size,
-                                  uint64_t packed_trx_prunable_size,
-                                  bool skip_recording);
+                                  uint64_t packed_trx_prunable_size );
 
          void init_for_deferred_trx( fc::time_point published );
 

libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp

@@ -96,15 +96,15 @@ public:
       return itr->trx_meta;
    }
 
-   template <typename Func>
-   bool clear_expired( const time_point& pending_block_time, const time_point& deadline, Func&& callback ) {
+   template <typename Yield, typename Callback>
+   bool clear_expired( const time_point& pending_block_time, Yield&& yield, Callback&& callback ) {
       auto& persisted_by_expiry = queue.get<by_expiry>();
       while( !persisted_by_expiry.empty() ) {
          const auto& itr = persisted_by_expiry.begin();
          if( itr->expiration() > pending_block_time ) {
             break;
          }
-         if( deadline <= fc::time_point::now() ) {
+         if( yield() ) {
             return false;
          }
          callback( itr->trx_meta->packed_trx(), itr->trx_type );

libraries/chain/thread_utils.cpp

@@ -1,23 +1,16 @@
 #include <eosio/chain/thread_utils.hpp>
 #include <fc/log/logger_config.hpp>
+#include <fc/exception/exception.hpp>
 
 namespace eosio { namespace chain {
 
-
-//
-// named_thread_pool
-//
-named_thread_pool::named_thread_pool( std::string name_prefix, size_t num_threads )
-: _thread_pool( num_threads )
+named_thread_pool::named_thread_pool( std::string name_prefix, size_t num_threads, bool delay_start )
+: _name_prefix( std::move(name_prefix) )
+, _num_threads( num_threads )
 , _ioc( num_threads )
 {
-   _ioc_work.emplace( boost::asio::make_work_guard( _ioc ) );
-   for( size_t i = 0; i < num_threads; ++i ) {
-      boost::asio::post( _thread_pool, [&ioc = _ioc, name_prefix, i]() {
-         std::string tn = name_prefix + "-" + std::to_string( i );
-         fc::set_os_thread_name( tn );
-         ioc.run();
-      } );
+   if( !delay_start ) {
+      start();
    }
 }
 
@@ -25,11 +18,26 @@ named_thread_pool::~named_thread_pool() {
    stop();
 }
 
+void named_thread_pool::start() {
+   FC_ASSERT( !_ioc_work, "Thread pool already started" );
+   _ioc_work.emplace( boost::asio::make_work_guard( _ioc ) );
+   _ioc.restart();
+   for( size_t i = 0; i < _num_threads; ++i ) {
+      _thread_pool.emplace_back( [&ioc = _ioc, &name_prefix = _name_prefix, i]() {
+         std::string tn = name_prefix + "-" + std::to_string( i );
+         fc::set_os_thread_name( tn );
+         ioc.run();
+      } );
+   }
+}
+
 void named_thread_pool::stop() {
    _ioc_work.reset();
    _ioc.stop();
-   _thread_pool.join();
-   _thread_pool.stop();
+   for( auto& t : _thread_pool ) {
+      t.join();
+   }
+   _thread_pool.clear();
 }
 
 

libraries/chain/transaction_context.cpp

@@ -234,8 +234,7 @@ namespace eosio { namespace chain {
    }
 
    void transaction_context::init_for_input_trx( uint64_t packed_trx_unprunable_size,
-                                                 uint64_t packed_trx_prunable_size,
-                                                 bool skip_recording )
+                                                 uint64_t packed_trx_prunable_size )
    {
       const transaction& trx = packed_trx.get_transaction();
       if( trx.transaction_extensions.size() > 0 ) {
@@ -272,8 +271,7 @@ namespace eosio { namespace chain {
          validate_referenced_accounts( trx, enforce_whiteblacklist && control.is_producing_block() );
       }
       init( initial_net_usage);
-      if (!skip_recording)
-         record_transaction( packed_trx.id(), trx.expiration ); /// checks for dupes
+      record_transaction( packed_trx.id(), trx.expiration ); /// checks for dupes
    }
 
    void transaction_context::init_for_deferred_trx( fc::time_point p )

libraries/cli11/include/cli11/CLI11.hpp

@@ -3,7 +3,7 @@
 // https://github.com/CLIUtils/CLI11
 //
 // This is a standalone header file generated by MakeSingleHeader.py in CLI11/scripts
-// from: 1d3b0a7
+// from: b19d133
 //
 // CLI11 2.2.0 Copyright (c) 2017-2022 University of Cincinnati, developed by Henry
 // Schreiner under NSF AWARD 1414736. All rights reserved.
@@ -34,28 +34,28 @@
 #pragma once
 
 // Standard combined includes:
-#include <string>
-#include <memory>
-#include <utility>
-#include <functional>
-#include <iterator>
-#include <type_traits>
-#include <vector>
-#include <exception>
-#include <cstdint>
-#include <cmath>
-#include <numeric>
-#include <tuple>
-#include <algorithm>
-#include <stdexcept>
-#include <set>
-#include <iomanip>
-#include <locale>
-#include <limits>
-#include <sstream>
-#include <fstream>
-#include <map>
 #include <iostream>
+#include <sstream>
+#include <iterator>
+#include <memory>
+#include <string>
+#include <exception>
+#include <map>
+#include <limits>
+#include <numeric>
+#include <type_traits>
+#include <fstream>
+#include <functional>
+#include <cstdint>
+#include <vector>
+#include <tuple>
+#include <locale>
+#include <algorithm>
+#include <cmath>
+#include <set>
+#include <stdexcept>
+#include <iomanip>
+#include <utility>
 
 
 #define CLI11_VERSION_MAJOR 2
@@ -9420,4 +9420,134 @@ inline std::string Formatter::make_option_usage(const Option *opt) const {
 
 
 
+
+class LeapFormatter : public Formatter {
+   // pseudographics - to draw subcommand tree
+   const char* tree_line = u8"\u2502";
+   const char* tree_angle = u8"\u2514";
+   const char* tree_fork = u8"\u251C";
+
+public:
+   LeapFormatter() : Formatter() {
+      // this gives better, more compact display
+      column_width(25);
+   }
+   LeapFormatter(const LeapFormatter&) = default;
+   LeapFormatter(LeapFormatter&&) = default;
+
+   /// This prints out all the subcommands
+   virtual std::string make_subcommands(const App* app, AppFormatMode mode) const {
+      std::stringstream out;
+
+      std::vector<const App*> subcommands = app->get_subcommands({});
+
+      // Make a list in definition order of the groups seen
+      std::vector<std::string> subcmd_groups_seen;
+      for(const App* com: subcommands) {
+         if(com->get_name().empty()) {
+            if(!com->get_group().empty()) {
+               out << make_expanded(com);
+            }
+            continue;
+         }
+         std::string group_key = com->get_group();
+         if(!group_key.empty() &&
+            std::find_if(subcmd_groups_seen.begin(), subcmd_groups_seen.end(), [&group_key](std::string a) {
+               return detail::to_lower(a) == detail::to_lower(group_key);
+            }) == subcmd_groups_seen.end())
+            subcmd_groups_seen.push_back(group_key);
+      }
+
+      // For each group, filter out and print subcommands
+      for(const std::string& group: subcmd_groups_seen) {
+         if(mode != AppFormatMode::SubCompact) {// do not show "Subcommands" header for nested tems in compact mode
+            out << "\n"
+                << group << ":\n";
+         }
+         std::vector<const App*> subcommands_group = app->get_subcommands([&group](const App* sub_app) {
+            return detail::to_lower(sub_app->get_group()) == detail::to_lower(group);
+         });
+         for(const App* new_com: subcommands_group) {
+            if(new_com->get_name().empty())
+               continue;
+
+            std::string tree_symbol = (subcommands_group.back() == new_com ? tree_angle : tree_fork);
+            std::string line_symbol = (subcommands_group.back() == new_com ? "" : tree_line);
+            std::string subc_symbol = "";
+
+            const App* parent = app->get_parent();
+            if(parent != nullptr) {
+               std::vector<const App*> sc_group = parent->get_subcommands([&group](const App* sub_app) {
+                  return detail::to_lower(sub_app->get_group()) == detail::to_lower(group);
+               });
+               if(sc_group.back() != app) {
+                  subc_symbol = tree_line;
+               }
+            }
+
+            switch(mode) {
+               case AppFormatMode::All:
+                  out << tree_symbol << new_com->help(new_com->get_name(), AppFormatMode::Sub);
+                  out << "\n";
+                  break;
+               case AppFormatMode::AllCompact:
+
+                  out << tree_symbol << new_com->help(new_com->get_name(), AppFormatMode::SubCompact);
+                  out << line_symbol;
+                  out << "\n";
+                  break;
+               case AppFormatMode::Normal:
+               case AppFormatMode::Sub:
+                  out << make_subcommand(new_com);
+                  break;
+               case AppFormatMode::SubCompact:
+                  out << tree_symbol << make_expanded(new_com, mode);
+                  break;
+               default:
+                  throw HorribleError("Internal error: unknown help type requested");
+            }
+         }
+      }
+
+      return out.str();
+   }
+
+   /// This prints out a subcommand in help-all
+   virtual std::string make_expanded(const App* sub, AppFormatMode mode = AppFormatMode::Sub) const {
+      std::stringstream out;
+      std::string tmp;
+      std::string subc_symbol = " ";
+      if(mode == AppFormatMode::SubCompact) {
+         detail::format_help(out, sub->get_display_name(true), sub->get_description(), column_width_);
+         out << make_subcommands(sub, mode);
+      } else {
+         out << sub->get_display_name(true) << "\n";
+         out << make_description(sub);
+         if(sub->get_name().empty() && !sub->get_aliases().empty()) {
+            detail::format_aliases(out, sub->get_aliases(), column_width_ + 2);
+         }
+         out << make_positionals(sub);
+         out << make_groups(sub, mode);
+         out << make_subcommands(sub, mode);
+      }
+
+      // Drop blank spaces
+      tmp = detail::find_and_replace(out.str(), "\n\n", "\n");
+      tmp = tmp.substr(0, tmp.size() - 1);// Remove the final '\n'
+
+      //
+      auto group = sub->get_parent()->get_group();
+      std::vector<const App*> sc_group = sub->get_parent()->get_subcommands(
+            [&group](const App* sub_app) { return detail::to_lower(sub_app->get_group()) == detail::to_lower(group); });
+
+      if(sc_group.back() != sub) {
+         subc_symbol = tree_line;
+      }
+
+      // Indent all but the first line (the name)
+      return detail::find_and_replace(tmp, "\n", "\n" + subc_symbol + "  ") + "\n";
+   }
+};
+
+
 } // namespace CLI

libraries/libfc/secp256k1/CMakeLists.txt

@@ -29,9 +29,9 @@ target_include_directories(secp256k1
 target_link_libraries(secp256k1 PRIVATE secp256k1-internal)
 
 install( TARGETS secp256k1 
-   RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} ${INSTALL_COMPONENT_ARGS}
-   LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} ${INSTALL_COMPONENT_ARGS}
-   ARCHIVE DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} ${INSTALL_COMPONENT_ARGS}
+   RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} COMPONENT dev EXCLUDE_FROM_ALL
+   LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} COMPONENT dev EXCLUDE_FROM_ALL
+   ARCHIVE DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} COMPONENT dev EXCLUDE_FROM_ALL
 )
 
 add_executable(secp256k1-bench secp256k1/src/bench.c)

libraries/state_history/include/eosio/state_history/log.hpp

@@ -439,7 +439,7 @@ class state_history_log {
             index.skip(-sizeof(uint64_t));
 
             if (!(remaining % 10000))
-               ilog("${remaining} blocks remaining, log pos = ${pos}", ("num_found", remaining)("pos", pos));
+               ilog("${r} blocks remaining, log pos = ${pos}", ("r", remaining)("pos", pos));
          }
       }
 

libraries/testing/include/eosio/testing/tester.hpp

@@ -597,7 +597,7 @@ namespace eosio { namespace testing {
          auto sb = _produce_block(skip_time, false);
          auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb );
          controller::block_report br;
-         validating_node->push_block( br, bsf, forked_branch_callback{}, trx_meta_cache_lookup{} );
+         validating_node->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} );
 
          return sb;
       }
@@ -607,9 +607,9 @@ namespace eosio { namespace testing {
       }
 
       void validate_push_block(const signed_block_ptr& sb) {
-         auto bs = validating_node->create_block_state_future( sb->calculate_id(), sb );
+         auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb );
          controller::block_report br;
-         validating_node->push_block( br, bs, forked_branch_callback{}, trx_meta_cache_lookup{} );
+         validating_node->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} );
       }
 
       signed_block_ptr produce_empty_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override {
@@ -617,7 +617,7 @@ namespace eosio { namespace testing {
          auto sb = _produce_block(skip_time, true);
          auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb );
          controller::block_report br;
-         validating_node->push_block( br, bsf, forked_branch_callback{}, trx_meta_cache_lookup{} );
+         validating_node->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} );
 
          return sb;
       }

libraries/testing/tester.cpp

@@ -322,7 +322,7 @@ namespace eosio { namespace testing {
       auto bsf = control->create_block_state_future(b->calculate_id(), b);
       unapplied_transactions.add_aborted( control->abort_block() );
       controller::block_report br;
-      control->push_block( br, bsf, [this]( const branch_type& forked_branch ) {
+      control->push_block( br, bsf.get(), [this]( const branch_type& forked_branch ) {
          unapplied_transactions.add_forked( forked_branch );
       }, [this]( const transaction_id_type& id ) {
          return unapplied_transactions.get_trx( id );
@@ -428,7 +428,8 @@ namespace eosio { namespace testing {
          }
       });
 
-      control->finalize_block( [&]( digest_type d ) {
+      controller::block_report br;
+      control->finalize_block( br, [&]( digest_type d ) {
          std::vector<signature_type> result;
          result.reserve(signing_keys.size());
          for (const auto& k: signing_keys)
@@ -1051,7 +1052,7 @@ namespace eosio { namespace testing {
                auto bsf = b.control->create_block_state_future( block->calculate_id(), block );
                b.control->abort_block();
                controller::block_report br;
-               b.control->push_block(br, bsf, forked_branch_callback{}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block);
+               b.control->push_block(br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block);
             }
          }
       };

package.cmake

@@ -61,13 +61,13 @@ set(CPACK_DEBIAN_BASE_FILE_NAME "${CPACK_DEBIAN_FILE_NAME}.deb")
 string(REGEX REPLACE "^(${CMAKE_PROJECT_NAME})" "\\1-dev" CPACK_DEBIAN_DEV_FILE_NAME "${CPACK_DEBIAN_BASE_FILE_NAME}")
 
 #deb package tooling will be unable to detect deps for the dev package. llvm is tricky since we don't know what package could have been used; try to figure it out
-set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "libboost-all-dev;libssl-dev;libgmp-dev")
+set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "libboost-all-dev, libssl-dev, libgmp-dev")
 find_program(DPKG_QUERY "dpkg-query")
 if(DPKG_QUERY AND OS_RELEASE MATCHES "\n?ID=\"?ubuntu" AND LLVM_CMAKE_DIR)
    execute_process(COMMAND "${DPKG_QUERY}" -S "${LLVM_CMAKE_DIR}" COMMAND cut -d: -f1 RESULT_VARIABLE LLVM_PKG_FIND_RESULT OUTPUT_VARIABLE LLVM_PKG_FIND_OUTPUT)
    if(LLVM_PKG_FIND_OUTPUT)
       string(STRIP "${LLVM_PKG_FIND_OUTPUT}" LLVM_PKG_FIND_OUTPUT)
-      list(APPEND CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "${LLVM_PKG_FIND_OUTPUT}")
+      string(APPEND CPACK_DEBIAN_DEV_PACKAGE_DEPENDS ", ${LLVM_PKG_FIND_OUTPUT}")
    endif()
 endif()
 

plugins/chain_api_plugin/chain.swagger.yaml

@@ -1,13 +1,13 @@
 openapi: 3.0.0
 info:
   title: Chain API
-  description: "OAS 3.0 Nodeos Chain API Specification\r"
+  description: Nodeos Chain API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eos.io
+    url: https://eosnetwork.com
 servers:
   - url: "{protocol}://{host}:{port}/v1/chain"
     variables:

plugins/chain_api_plugin/chain_api_plugin.cpp

@@ -105,7 +105,7 @@ void chain_api_plugin::plugin_startup() {
    auto rw_api = chain.get_read_write_api(max_response_time);
 
    auto& _http_plugin = app().get_plugin<http_plugin>();
-   ro_api.set_shorten_abi_errors( !_http_plugin.verbose_errors() );
+   ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() );
 
    _http_plugin.add_api( {
       CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::priority::medium_high);

plugins/chain_interface/include/eosio/chain/plugin_interface.hpp

@@ -37,14 +37,9 @@ namespace eosio { namespace chain { namespace plugin_interface {
    }
 
    namespace incoming {
-      namespace channels {
-         using block                 = channel_decl<struct block_tag, signed_block_ptr>;
-         using transaction           = channel_decl<struct transaction_tag, packed_transaction_ptr>;
-      }
-
       namespace methods {
-         // synchronously push a block/trx to a single provider
-         using block_sync            = method_decl<chain_plugin_interface, bool(const signed_block_ptr&, const std::optional<block_id_type>&), first_provider_policy>;
+         // synchronously push a block/trx to a single provider, block_state_ptr may be null
+         using block_sync            = method_decl<chain_plugin_interface, bool(const signed_block_ptr&, const std::optional<block_id_type>&, const block_state_ptr&), first_provider_policy>;
          using transaction_async     = method_decl<chain_plugin_interface, void(const packed_transaction_ptr&, bool, bool, bool, next_function<transaction_trace_ptr>), first_provider_policy>;
       }
    }

plugins/chain_plugin/chain_plugin.cpp

@@ -142,7 +142,6 @@ public:
    ,irreversible_block_channel(app().get_channel<channels::irreversible_block>())
    ,accepted_transaction_channel(app().get_channel<channels::accepted_transaction>())
    ,applied_transaction_channel(app().get_channel<channels::applied_transaction>())
-   ,incoming_block_channel(app().get_channel<incoming::channels::block>())
    ,incoming_block_sync_method(app().get_method<incoming::methods::block_sync>())
    ,incoming_transaction_async_method(app().get_method<incoming::methods::transaction_async>())
    {}
@@ -170,7 +169,6 @@ public:
    channels::irreversible_block::channel_type&     irreversible_block_channel;
    channels::accepted_transaction::channel_type&   accepted_transaction_channel;
    channels::applied_transaction::channel_type&    applied_transaction_channel;
-   incoming::channels::block::channel_type&        incoming_block_channel;
 
    // retained references to methods for easy calling
    incoming::methods::block_sync::method_type&        incoming_block_sync_method;
@@ -1138,6 +1136,13 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
          // For the time being, when `deep-mind = true` is activated, we set `stdout` here to
          // be an unbuffered I/O stream.
          setbuf(stdout, NULL);
+         
+         //verify configuration is correct
+         EOS_ASSERT( options.at("api-accept-transactions").as<bool>() == false, plugin_config_exception,
+            "api-accept-transactions must be set to false in order to enable deep-mind logging.");
+
+         EOS_ASSERT( options.at("p2p-accept-transactions").as<bool>() == false, plugin_config_exception,
+            "p2p-accept-transactions must be set to false in order to enable deep-mind logging.");
 
          my->chain->enable_deep_mind( &_deep_mind_log );
       }
@@ -1345,8 +1350,8 @@ chain_apis::read_only chain_plugin::get_read_only_api(const fc::microseconds& ht
 }
 
 
-bool chain_plugin::accept_block(const signed_block_ptr& block, const block_id_type& id ) {
-   return my->incoming_block_sync_method(block, id);
+bool chain_plugin::accept_block(const signed_block_ptr& block, const block_id_type& id, const block_state_ptr& bsp ) {
+   return my->incoming_block_sync_method(block, id, bsp);
 }
 
 void chain_plugin::accept_transaction(const chain::packed_transaction_ptr& trx, next_function<chain::transaction_trace_ptr> next) {
@@ -2165,7 +2170,7 @@ fc::variant read_only::get_block_header_state(const get_block_header_state_param
 
 void read_write::push_block(read_write::push_block_params&& params, next_function<read_write::push_block_results> next) {
    try {
-      app().get_method<incoming::methods::block_sync>()(std::make_shared<signed_block>( std::move( params ) ), std::optional<block_id_type>{});
+      app().get_method<incoming::methods::block_sync>()(std::make_shared<signed_block>( std::move(params) ), std::optional<block_id_type>{}, block_state_ptr{});
    } catch ( boost::interprocess::bad_alloc& ) {
       chain_plugin::handle_db_exhaustion();
    } catch ( const std::bad_alloc& ) {

plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp

@@ -849,7 +849,7 @@ public:
    chain_apis::read_write get_read_write_api(const fc::microseconds& http_max_response_time);
    chain_apis::read_only get_read_only_api(const fc::microseconds& http_max_response_time) const;
 
-   bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id );
+   bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id, const chain::block_state_ptr& bsp );
    void accept_transaction(const chain::packed_transaction_ptr& trx, chain::plugin_interface::next_function<chain::transaction_trace_ptr> next);
 
    // Only call this after plugin_initialize()!

plugins/db_size_api_plugin/db_size.swagger.yaml

@@ -1,12 +1,13 @@
 openapi: 3.0.0
 info:
   title: DB Size API
+  description: Nodeos DB Size API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eos.io
+    url: https://eosnetwork.com
 servers:
   - url: '{protocol}://{host}:{port}/v1/'
     variables:

plugins/http_plugin/http_plugin.cpp

@@ -359,6 +359,8 @@ class http_plugin_impl : public std::enable_shared_from_this<http_plugin_impl> {
             }
          }
 
+         my->plugin_state->server_header = current_http_plugin_defaults.server_header;
+
          
          //watch out for the returns above when adding new code here
       } FC_LOG_AND_RETHROW()
@@ -474,6 +476,7 @@ class http_plugin_impl : public std::enable_shared_from_this<http_plugin_impl> {
       my->plugin_state->url_handlers.clear();
 
       app().post( 0, [me = my](){} ); // keep my pointer alive until queue is drained
+      fc_ilog( logger(), "exit shutdown");
    }
 
    void http_plugin::add_handler(const string& url, const url_handler& handler, int priority) {
@@ -537,7 +540,7 @@ class http_plugin_impl : public std::enable_shared_from_this<http_plugin_impl> {
       return (!my->listen_endpoint || my->listen_endpoint->address().is_loopback());
    }
 
-   bool http_plugin::verbose_errors()const {
+   bool http_plugin::verbose_errors() {
       return verbose_http_errors;
    }
 

plugins/http_plugin/include/eosio/http_plugin/beast_http_session.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <eosio/http_plugin/common.hpp>
+#include <fc/io/json.hpp>
 
 #include <memory>
 #include <string>
@@ -105,27 +106,16 @@ protected:
       res_->version(req.version());
       res_->set(http::field::content_type, "application/json");
       res_->keep_alive(req.keep_alive());
-      res_->set(http::field::server, BOOST_BEAST_VERSION_STRING);
-
-      // Returns a bad request response
-      auto const bad_request =
-            [](const beast::string_view& why, detail::abstract_conn& conn) {
-               conn.send_response(std::string(why),
-                                  static_cast<int>(http::status::bad_request));
-            };
-
-      // Returns a not found response
-      auto const not_found =
-            [](const std::string& target, detail::abstract_conn& conn) {
-               conn.send_response("The resource '" + target + "' was not found.",
-                                  static_cast<int>(http::status::not_found));
-            };
+      if(plugin_state_->server_header.size())
+         res_->set(http::field::server, plugin_state_->server_header);
 
       // Request path must be absolute and not contain "..".
-      if(req.target().empty() ||
-         req.target()[0] != '/' ||
-         req.target().find("..") != beast::string_view::npos)
-         return bad_request("Illegal request-target", *this);
+      if(req.target().empty() || req.target()[0] != '/' || req.target().find("..") != beast::string_view::npos) {
+         error_results results{static_cast<uint16_t>(http::status::bad_request), "Illegal request-target"};
+         send_response( fc::json::to_string( results, fc::time_point::maximum() ),
+                        static_cast<unsigned int>(http::status::bad_request) );
+         return;
+      }
 
       try {
          if(!derived().allow_host(req))
@@ -146,7 +136,7 @@ protected:
 
          // Respond to options request
          if(req.method() == http::verb::options) {
-            send_response("", static_cast<int>(http::status::ok));
+            send_response("{}", static_cast<unsigned int>(http::status::ok));
             return;
          }
 
@@ -154,7 +144,7 @@ protected:
          if(!verify_max_bytes_in_flight()) return;
 
          std::string resource = std::string(req.target());
-         // look for the URL handler to handle this reosouce
+         // look for the URL handler to handle this resource
          auto handler_itr = plugin_state_->url_handlers.find(resource);
          if(handler_itr != plugin_state_->url_handlers.end()) {
             if(plugin_state_->logger.is_enabled(fc::log_level::all))
@@ -165,27 +155,29 @@ protected:
                                 std::move(body),
                                 make_http_response_handler(plugin_state_, derived().shared_from_this()));
          } else {
-            fc_dlog(plugin_state_->logger, "404 - not found: ${ep}", ("ep", resource));
-            not_found(resource, *this);
+            fc_dlog( plugin_state_->logger, "404 - not found: ${ep}", ("ep", resource) );
+            error_results results{static_cast<uint16_t>(http::status::not_found), "Not Found",
+                                  error_results::error_info( fc::exception( FC_LOG_MESSAGE( error, "Unknown Endpoint" ) ),
+                                                             http_plugin::verbose_errors() )};
+            send_response( fc::json::to_string( results, fc::time_point::now() + plugin_state_->max_response_time ),
+                           static_cast<unsigned int>(http::status::not_found) );
          }
       } catch(...) {
          handle_exception();
       }
    }
 
-   void report_429_error(std::string what) {
-      send_response(std::move(what),
-                    static_cast<int>(http::status::too_many_requests));
-   }
-
 public:
    virtual bool verify_max_bytes_in_flight() override {
       auto bytes_in_flight_size = plugin_state_->bytes_in_flight.load();
       if(bytes_in_flight_size > plugin_state_->max_bytes_in_flight) {
          fc_dlog(plugin_state_->logger, "429 - too many bytes in flight: ${bytes}", ("bytes", bytes_in_flight_size));
-         std::string what = "Too many bytes in flight: " + std::to_string(bytes_in_flight_size) + ". Try again later.";
-         ;
-         report_429_error(std::move(what));
+         error_results::error_info ei;
+         ei.code = static_cast<int64_t>(http::status::too_many_requests);
+         ei.name = "Busy";
+         ei.what = "Too many bytes in flight: " + std::to_string( bytes_in_flight_size );
+         error_results results{static_cast<uint16_t>(http::status::too_many_requests), "Busy", ei};
+         send_response( fc::json::to_string( results, fc::time_point::maximum() ), static_cast<unsigned int>(http::status::too_many_requests) );
          return false;
       }
       return true;
@@ -198,8 +190,12 @@ public:
       auto requests_in_flight_num = plugin_state_->requests_in_flight.load();
       if(requests_in_flight_num > plugin_state_->max_requests_in_flight) {
          fc_dlog(plugin_state_->logger, "429 - too many requests in flight: ${requests}", ("requests", requests_in_flight_num));
-         std::string what = "Too many requests in flight: " + std::to_string(requests_in_flight_num) + ". Try again later.";
-         report_429_error(std::move(what));
+         error_results::error_info ei;
+         ei.code = static_cast<int64_t>(http::status::too_many_requests);
+         ei.name = "Busy";
+         ei.what = "Too many requests in flight: " + std::to_string( requests_in_flight_num );
+         error_results results{static_cast<uint16_t>(http::status::too_many_requests), "Busy", ei};
+         send_response( fc::json::to_string( results, fc::time_point::maximum() ), static_cast<unsigned int>(http::status::too_many_requests) );
          return false;
       }
       return true;
@@ -310,38 +306,59 @@ public:
    virtual void handle_exception() override {
       std::string err_str;
       try {
-         throw;
-      } catch(const fc::exception& e) {
-         err_str = e.to_detail_string();
-         fc_elog(plugin_state_->logger, "fc::exception: ${w}", ("w", err_str));
-      } catch(std::exception& e) {
-         err_str = e.what();
-         fc_elog(plugin_state_->logger, "std::exception: ${w}", ("w", err_str));
-      } catch(...) {
-         err_str = "unknown";
-         fc_elog(plugin_state_->logger, "unkonwn exception");
+         try {
+            throw;
+         } catch(const fc::exception& e) {
+            err_str = e.to_detail_string();
+            fc_elog(plugin_state_->logger, "fc::exception: ${w}", ("w", err_str));
+            error_results results{static_cast<uint16_t>(http::status::internal_server_error),
+                                  "Internal Service Error",
+                                  error_results::error_info( e, http_plugin::verbose_errors() )};
+            err_str = fc::json::to_string( results, fc::time_point::now() + plugin_state_->max_response_time );
+         } catch(std::exception& e) {
+            err_str = e.what();
+            fc_elog(plugin_state_->logger, "std::exception: ${w}", ("w", err_str));
+            error_results results{static_cast<uint16_t>(http::status::internal_server_error),
+                                  "Internal Service Error",
+                                  error_results::error_info( fc::exception( FC_LOG_MESSAGE( error, err_str )),
+                                                             http_plugin::verbose_errors() )};
+            err_str = fc::json::to_string( results, fc::time_point::now() + plugin_state_->max_response_time );
+         } catch(...) {
+            err_str = "Unknown exception";
+            fc_elog(plugin_state_->logger, err_str);
+            error_results results{static_cast<uint16_t>(http::status::internal_server_error),
+                                  "Internal Service Error",
+                                  error_results::error_info(
+                                        fc::exception( FC_LOG_MESSAGE( error, err_str )),
+                                        http_plugin::verbose_errors() )};
+            err_str = fc::json::to_string( results, fc::time_point::now() + plugin_state_->max_response_time );
+         }
+      } catch (fc::timeout_exception& e) {
+         fc_elog( plugin_state_->logger, "Timeout exception ${te} attempting to handle exception: ${e}", ("te", e.to_detail_string())("e", err_str) );
+         err_str = R"xxx({"message": "Internal Server Error"})xxx";
+      } catch (...) {
+         fc_elog( plugin_state_->logger, "Exception attempting to handle exception: ${e}", ("e", err_str) );
+         err_str = R"xxx({"message": "Internal Server Error"})xxx";
       }
 
+
       if(is_send_exception_response_) {
-         res_->set(http::field::content_type, "text/plain");
+         res_->set(http::field::content_type, "application/json");
          res_->keep_alive(false);
          res_->set(http::field::server, BOOST_BEAST_VERSION_STRING);
 
-         http::status stat = http::status::internal_server_error;
-         auto resp_str = "Internal Server Error\n\nUnhandled Exception: " + err_str;
-         send_response(resp_str, static_cast<int>(stat));
+         send_response(std::move(err_str), static_cast<unsigned int>(http::status::internal_server_error));
          derived().do_eof();
       }
    }
 
-   virtual void send_response(std::optional<std::string> body, int code) override {
+   virtual void send_response(std::string json_body, unsigned int code) override {
       write_begin_ = steady_clock::now();
       auto dt = write_begin_ - handle_begin_;
       handle_time_us_ += std::chrono::duration_cast<std::chrono::microseconds>(dt).count();
 
       res_->result(code);
-      if(body.has_value())
-         res_->body() = *body;
+      res_->body() = std::move(json_body);
 
       res_->prepare_payload();
 

plugins/http_plugin/include/eosio/http_plugin/common.hpp

@@ -60,7 +60,7 @@ struct abstract_conn {
    virtual bool verify_max_requests_in_flight() = 0;
    virtual void handle_exception() = 0;
 
-   virtual void send_response(std::optional<std::string> body, int code) = 0;
+   virtual void send_response(std::string json_body, unsigned int code) = 0;
 };
 
 using abstract_conn_ptr = std::shared_ptr<abstract_conn>;
@@ -130,6 +130,8 @@ struct http_plugin_state {
    bool validate_host = true;
    set<string> valid_hosts;
 
+   string server_header;
+
    url_handlers_type url_handlers;
    bool keep_alive = false;
 
@@ -233,7 +235,7 @@ auto make_http_response_handler(std::shared_ptr<http_plugin_state> plugin_state,
                                  auto tracked_json = make_in_flight(std::move(json), plugin_state);
                                  session_ptr->send_response(std::move(tracked_json->obj()), code);
                               } else {
-                                 session_ptr->send_response({}, code);
+                                 session_ptr->send_response("{}", code);
                               }
                            } catch(...) {
                               session_ptr->handle_exception();

plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp

@@ -45,6 +45,8 @@ namespace eosio {
       //If non 0, HTTP will be enabled by default on the given port number. If
       // 0, HTTP will not be enabled by default
       uint16_t default_http_port{0};
+      //If set, a Server header will be added to the HTTP reply with this value
+      string server_header;
    };
 
    /**
@@ -96,7 +98,7 @@ namespace eosio {
         bool is_on_loopback() const;
         bool is_secure() const;
 
-        bool verbose_errors()const;
+        static bool verbose_errors();
 
         struct get_supported_apis_result {
            vector<string> apis;

plugins/net_api_plugin/net.swagger.yaml

@@ -1,12 +1,13 @@
 openapi: 3.0.0
 info:
   title: Net API
+  description: Nodeos Net API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eos.io
+    url: https://eosnetwork.com
 servers:
   - url: '{protocol}://{host}:{port}/v1/'
     variables:

plugins/net_plugin/net_plugin.cpp

@@ -634,6 +634,7 @@ namespace eosio {
       string                  local_endpoint_port;
 
       std::atomic<uint32_t>   trx_in_progress_size{0};
+      fc::time_point          last_dropped_trx_msg_time;
       const uint32_t          connection_id;
       int16_t                 sent_handshake_count = 0;
       std::atomic<bool>       connecting{true};
@@ -784,7 +785,7 @@ namespace eosio {
       void handle_message( const packed_transaction& msg ) = delete; // packed_transaction_ptr overload used instead
       void handle_message( packed_transaction_ptr msg );
 
-      void process_signed_block( const block_id_type& id, signed_block_ptr msg );
+      void process_signed_block( const block_id_type& id, signed_block_ptr msg, block_state_ptr bsp );
 
       fc::variant_object get_logger_variant() const {
          fc::mutable_variant_object mvo;
@@ -2340,14 +2341,6 @@ namespace eosio {
 
    // called from connection strand
    void connection::connect( const std::shared_ptr<tcp::resolver>& resolver, tcp::resolver::results_type endpoints ) {
-      switch ( no_retry ) {
-         case no_reason:
-         case wrong_version:
-         case benign_other:
-            break;
-         default:
-            return;
-      }
       connecting = true;
       pending_message_buffer.reset();
       buffer_queue.clear_out_queue();
@@ -2683,6 +2676,10 @@ namespace eosio {
          char reason[72];
          snprintf(reason, 72, "Dropping trx, too many trx in progress %lu bytes", trx_in_progress_sz);
          my_impl->producer_plug->log_failed_transaction(ptr->id(), ptr, reason);
+         if (fc::time_point::now() - fc::seconds(1) >= last_dropped_trx_msg_time) {
+            last_dropped_trx_msg_time = fc::time_point::now();
+            peer_wlog(this, reason);
+         }
          return true;
       }
       bool have_trx = my_impl->dispatcher->have_txn( ptr->id() );
@@ -3138,13 +3135,44 @@ namespace eosio {
    // called from connection strand
    void connection::handle_message( const block_id_type& id, signed_block_ptr ptr ) {
       peer_dlog( this, "received signed_block ${num}, id ${id}", ("num", ptr->block_num())("id", id) );
-      app().post(priority::medium, [ptr{std::move(ptr)}, id, c = shared_from_this()]() mutable {
-         c->process_signed_block( id, std::move( ptr ) );
+
+      controller& cc = my_impl->chain_plug->chain();
+      block_state_ptr bsp;
+      bool exception = false;
+      try {
+         if( cc.fetch_block_state_by_id( id ) ) {
+            my_impl->dispatcher->add_peer_block( id, connection_id );
+            my_impl->sync_master->sync_recv_block( shared_from_this(), id, ptr->block_num(), false );
+            return;
+         }
+         // this may return null if block is not immediately ready to be processed
+         bsp = cc.create_block_state( id, ptr );
+      } catch( const fc::exception& ex ) {
+         exception = true;
+         peer_elog(this, "bad block exception: #${n} ${id}...: ${m}",
+                   ("n", ptr->block_num())("id", id.str().substr(8,16))("m",ex.to_string()));
+      } catch( ... ) {
+         exception = true;
+         peer_elog(this, "bad block: #${n} ${id}...: unknown exception",
+                   ("n", ptr->block_num())("id", id.str().substr(8,16)));
+      }
+      if( exception ) {
+         my_impl->sync_master->rejected_block( shared_from_this(), ptr->block_num() );
+         my_impl->dispatcher->rejected_block( id );
+         return;
+      }
+
+      bool signal_producer = !!bsp; // ready to process immediately, so signal producer to interrupt start_block
+      app().post(priority::medium, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c = shared_from_this()]() mutable {
+         c->process_signed_block( id, std::move(ptr), std::move(bsp) );
       });
+
+      if( signal_producer )
+         my_impl->producer_plug->received_block();
    }
 
    // called from application thread
-   void connection::process_signed_block( const block_id_type& blk_id, signed_block_ptr msg ) {
+   void connection::process_signed_block( const block_id_type& blk_id, signed_block_ptr msg, block_state_ptr bsp ) {
       controller& cc = my_impl->chain_plug->chain();
       uint32_t blk_num = msg->block_num();
       // use c in this method instead of this to highlight that all methods called on c-> must be thread safe
@@ -3174,7 +3202,7 @@ namespace eosio {
 
       go_away_reason reason = fatal_other;
       try {
-         bool accepted = my_impl->chain_plug->accept_block(msg, blk_id);
+         bool accepted = my_impl->chain_plug->accept_block(msg, blk_id, bsp);
          my_impl->update_chain_info();
          if( !accepted ) return;
          reason = no_reason;

plugins/producer_api_plugin/producer.swagger.yaml

@@ -1,16 +1,18 @@
 openapi: 3.0.0
 info:
   title: Producer API
+  description: Nodeos Producer API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eosnetwork.com
+    url: https://antelope.io
 tags:
-  - name: eosio
+  - name: Protocol Version 3.2
+    description: The release tag for Leap binaries is also the protocol version
 servers:
-  - url: "{protocol}://{host}:{port}/v1/"
+  - url: "{protocol}://{host}:{port}/v1"
     variables:
       protocol:
         enum:
@@ -21,16 +23,13 @@ servers:
         default: localhost
       port:
         default: "8080"
-components:
-  securitySchemes: {}
-  schemas: {}
 security:
   - {}
 paths:
   /producer/pause:
     post:
       summary: pause
-      description: Pause producer node. Takes no arguments returns no values.
+      description: Pause producer node. Takes no arguments and returns no values.
       operationId: pause
       responses:
         "201":
@@ -38,11 +37,18 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/resume:
     post:
       summary: resume
-      description: Resume producer node. Takes no arguments returns no values.
+      description: Resume producer node. Takes no arguments and returns no values.
       operationId: resume
       responses:
         "201":
@@ -50,23 +56,39 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/paused:
     post:
       summary: paused
-      description: Retreives paused status for producer node. Takes no arguments returns no values.
+      description: Retrieves paused status for producer node. Takes no arguments and returns no values.
       operationId: paused
       responses:
         "201":
           description: OK
+          content:
+            text/plain:
+              schema:
+                description: true or false
+                type: string
+                example: false
+        "400":
+          description: client error
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: '#/components/schemas/Error'
+
   /producer/get_runtime_options:
     post:
       summary: get_runtime_options
-      description: Retreives run time options for producer node. Only `incoming_defer_ratio` is a floating point number the other return values are signed integers.
+      description: Retrieves runtime options for producer node.
       operationId: get_runtime_options
       responses:
         "201":
@@ -74,30 +96,42 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Runtime_Options'
+                $ref: '#/components/schemas/Runtime_Options'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/update_runtime_options:
     post:
       summary: update_runtime_options
-      description: Update run time options for producer node. May post any of the runtime options in combination or along. Only `incoming_defer_ratio` is a floating point number the other values are signed integers.
+      description: Update runtime options for producer node. May post any of the runtime options in combination or alone.
       operationId: update_runtime_options
       requestBody:
         content:
           application/json:
             schema:
-              $ref: '#/component/schema/Runtime_Options'
-
+              $ref: '#/components/schemas/Runtime_Options'
       responses:
         "201":
           description: OK
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/get_greylist:
     post:
       summary: get_greylist
-      description: Retreives the greylist for producer node.
+      description: Retrieves the greylist for producer node.
       operationId: get_greylist
       responses:
         "201":
@@ -112,11 +146,17 @@ paths:
                     description: Array of account names stored in the greylist
                     items:
                       $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/add_greylist_accounts:
     post:
       summary: add_greylist_accounts
-      description: Adds accounts to grey list for producer node. At least one account is required.
+      description: Adds accounts to greylist for producer node. At least one account is required.
       operationId: add_greylist_accounts
       requestBody:
         content:
@@ -129,20 +169,20 @@ paths:
                     description: List of account names to add
                     items:
                       $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
-
       responses:
         "201":
           description: OK
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: '#/components/schemas/OK'
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: '#/components/schemas/Error'
+
   /producer/remove_greylist_accounts:
     post:
       summary: remove_greylist_accounts
@@ -159,26 +199,25 @@ paths:
                   description: List of account names to remove
                   items:
                     $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
-
       responses:
         "201":
           description: OK
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: '#/components/schemas/OK'
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: '#/components/schemas/Error'
+
   /producer/get_whitelist_blacklist:
     post:
       summary: get_whitelist_blacklist
-      description: Retreives the white list and black list for producer node. A json object containing whitelist and blacklist information. `actor_whitelist` , `actor_blacklist`, `contract_whitelist`, `contract_blacklist` are represented by an array of Names. `action_blacklist` is an array of tuples consisting of Name and Action. `key_blacklist` is an array of Public Keys. Name is a string represeting a NamePrivileged or NameBasic or NameBid or NameCatchAll
+      description: Retrieves the whitelist and blacklist for producer node. A JSON object containing whitelist and blacklist information.
       operationId: get_whitelist_blacklist
-
       responses:
         "201":
           description: OK
@@ -215,14 +254,19 @@ paths:
                   key_blacklist:
                     type: array
                     items:
-                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/KeyType.yaml"
+                      - $ref: "https://docs.eosnetwork.com/openapi/v2.0/KeyType.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/set_whitelist_blacklist:
     post:
       summary: set_whitelist_blacklist
-      description: Defines the whitelist and blacklist for a producer node. Takes a json object containing whitelist and blacklist information. At least one of actor_whitelist, actor_blacklist, contract_whitelist, contract_blacklist, action_blacklist, and key_blacklist is required. `actor_whitelist` , `actor_blacklist`, `contract_whitelist`, `contract_blacklist` take an array of Names. `action_blacklist` is an array of tuples consisting of Name and Action. `key_blacklist` is an array of Public Keys. Name is a string represeting a NamePrivileged or NameBasic or NameBid or NameCatchAll
+      description: Defines the whitelist and blacklist for a producer node. Takes a JSON object containing whitelist and blacklist information.
       operationId: set_whitelist_blacklist
-      required: true
       requestBody:
         content:
           application/json:
@@ -264,19 +308,18 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
-
+                $ref: '#/components/schemas/OK'
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: '#/components/schemas/Error'
 
   /producer/create_snapshot:
     post:
       summary: create_snapshot
-      description: Creates a snapshot for producer node. Returns error when unable to create snapshot.
+      description: Creates a snapshot for producer node. Returns error when unable to create a snapshot.
       operationId: create_snapshot
       responses:
         "201":
@@ -290,7 +333,7 @@ paths:
                     $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
                   head_block_num:
                     type: integer
-                    descripiton: Highest block number on the chain
+                    description: Highest block number on the chain
                     example: 5102
                   head_block_time:
                     type: string
@@ -309,14 +352,13 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: '#/components/schemas/Error'
 
   /producer/get_integrity_hash:
     post:
       summary: get_integrity_hash
-      description: Retreives the integrity hash for producer node
+      description: Retrieves the integrity hash for producer node
       operationId: get_integrity_hash
-
       responses:
         "201":
           description: OK
@@ -330,6 +372,12 @@ paths:
                     $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
                   integrity_hash:
                     $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/schedule_protocol_feature_activations:
     post:
@@ -353,18 +401,18 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: '#/components/schemas/OK'
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: '#/components/schemas/Error'
 
   /producer/get_supported_protocol_features:
     post:
       summary: get_supported_protocol_features
-      description: Retreives supported protocol features for producer node. Pass filters in as part of the request body.
+      description: Retrieves supported protocol features for producer node. Pass filters in as part of the request body.
       operationId: get_supported_protocol_features
       requestBody:
         content:
@@ -375,6 +423,7 @@ paths:
                 exclude_disabled:
                   type: boolean
                   description: Exclude disabled protocol features
+                  example: false
                 exclude_unactivatable:
                   type: boolean
                   description: Exclude unactivatable protocol features
@@ -406,7 +455,7 @@ paths:
                           example: "1970-01-01T00:00:00.000"
                         description_digest:
                           $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
-                        dependancies:
+                        dependencies:
                           type: array
                           items:
                             $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
@@ -422,12 +471,18 @@ paths:
                                 type: string
                               value:
                                 type: string
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
 
   /producer/get_account_ram_corrections:
     post:
       summary: get_account_ram_corrections
-      description: Retreives accounts with ram corrections.
+      description: Retrieves accounts with ram corrections.
       operationId: get_account_ram_corrections
       requestBody:
         content:
@@ -436,14 +491,14 @@ paths:
               type: object
               properties:
                 lower_bound:
-                  type: int
+                  type: integer
                   description: lowest account key
                 upper_bound:
-                  type: int
+                  type: integer
                   description: highest account key
                 limit:
-                  type: int
-                  description: number of rows to scans
+                  type: integer
+                  description: number of rows to scan
                   example: 10
                 reverse:
                   type: boolean
@@ -456,13 +511,25 @@ paths:
             application/json:
               schema:
                 type: object
+                required:
+                  - rows
                 properties:
                   rows:
                     type: array
                     items:
                       type: string
+                  more:
+                    type: array
+                    items:
+                      - $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
-  /producer/producer/get_unapplied_transactions:
+  /producer/get_unapplied_transactions:
     post:
       summary: get_unapplied_transactions
       description: Get Unapplied Transactions. No required parameters.
@@ -474,13 +541,13 @@ paths:
               type: object
               properties:
                 limit:
-                  type: int
-                  description: number of transactions to return, defaults to 4,294,967,295
+                  type: integer
+                  description: limit number of transactions to return
                   example: 2
                 lower_bound:
                   $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
                 time_limit_ms:
-                  type: int
+                  type: integer
                   description: defaults to 10ms
                   example: 10
       responses:
@@ -535,10 +602,11 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: '#/components/schemas/Error'
 
-component:
-  schema:
+components:
+  securitySchemes: {}
+  schemas:
     Error:
       type: object
       properties:
@@ -586,7 +654,7 @@ component:
                     example: 246
                   method:
                     type: string
-                    description: function executed when error occured
+                    description: function executed when error occurred
                     example: parse_params
     OK:
       type: object
@@ -618,9 +686,13 @@ component:
           type: integer
           description: Max scheduled transaction time per block in ms
           example: 100
+        subjective_cpu_leeway_us:
+          type: integer
+          description: in micro seconds
+          example: 10
         incoming_defer_ratio:
           type: string
-          description: Incoming defer ration, parsed to double
+          description: Incoming defer ratio, parsed to double
           example: "1.00000000000000000"
         greylist_limit:
           type: integer

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -144,6 +144,9 @@ public:
 
    void log_failed_transaction(const transaction_id_type& trx_id, const chain::packed_transaction_ptr& packed_trx_ptr, const char* reason) const;
 
+   // thread-safe, called when a new block is received
+   void received_block();
+
  private:
    std::shared_ptr<class producer_plugin_impl> my;
 };

plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp

@@ -180,7 +180,8 @@ public:
       fc_dlog( log, "Subjective billed accounts ${n} removed ${r}", ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
    }
 
-   bool remove_expired( fc::logger& log, const fc::time_point& pending_block_time, const fc::time_point& now, const fc::time_point& deadline ) {
+   template <typename Yield>
+   bool remove_expired( fc::logger& log, const fc::time_point& pending_block_time, const fc::time_point& now, Yield&& yield ) {
       bool exhausted = false;
       auto& idx = _trx_cache_index.get<by_expiry>();
       if( !idx.empty() ) {
@@ -189,7 +190,7 @@ public:
          uint32_t num_expired = 0;
 
          while( !idx.empty() ) {
-            if( deadline <= fc::time_point::now() ) {
+            if( yield() ) {
                exhausted = true;
                break;
             }

plugins/producer_plugin/producer_plugin.cpp

@@ -310,6 +310,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       std::optional<named_thread_pool>                          _thread_pool;
 
       std::atomic<int32_t>                                      _max_transaction_time_ms; // modified by app thread, read by net_plugin thread pool
+      std::atomic<bool>                                         _received_block{false}; // modified by net_plugin thread pool and app thread
       fc::microseconds                                          _max_irreversible_block_age_us;
       int32_t                                                   _produce_time_offset_us = 0;
       int32_t                                                   _last_block_time_offset_us = 0;
@@ -327,9 +328,6 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
       chain_plugin* chain_plug = nullptr;
 
-      incoming::channels::block::channel_type::handle         _incoming_block_subscription;
-      incoming::channels::transaction::channel_type::handle   _incoming_transaction_subscription;
-
       compat::channels::transaction_ack::channel_type&        _transaction_ack_channel;
 
       incoming::methods::block_sync::method_type::handle        _incoming_block_sync_provider;
@@ -424,7 +422,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          _idle_trx_time = fc::time_point::now();
       }
 
-      bool on_incoming_block(const signed_block_ptr& block, const std::optional<block_id_type>& block_id) {
+      bool on_incoming_block(const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
          auto& chain = chain_plug->chain();
          if ( _pending_block_mode == pending_block_mode::producing ) {
             fc_wlog( _log, "dropped incoming block #${num} id: ${id}",
@@ -432,6 +430,11 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
             return false;
          }
 
+         // start a new speculative block, speculative start_block may have been interrupted
+         auto ensure = fc::make_scoped_exit([this](){
+            schedule_production_loop();
+         });
+
          const auto& id = block_id ? *block_id : block->calculate_id();
          auto blk_num = block->block_num();
 
@@ -445,16 +448,14 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          if( existing ) { return false; }
 
          // start processing of block
-         auto bsf = chain.create_block_state_future( id, block );
+         std::future<block_state_ptr> bsf;
+         if( !bsp ) {
+            bsf = chain.create_block_state_future( id, block );
+         }
 
          // abort the pending block
          abort_block();
 
-         // exceptions throw out, make sure we restart our loop
-         auto ensure = fc::make_scoped_exit([this](){
-            schedule_production_loop();
-         });
-
          // push the new block
          auto handle_error = [&](const auto& e)
          {
@@ -465,7 +466,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
          controller::block_report br;
          try {
-            chain.push_block( br, bsf, [this]( const branch_type& forked_branch ) {
+            const block_state_ptr& bspr = bsp ? bsp : bsf.get();
+            chain.push_block( br, bspr, [this]( const branch_type& forked_branch ) {
                _unapplied_transactions.add_forked( forked_branch );
             }, [this]( const transaction_id_type& id ) {
                return _unapplied_transactions.get_trx( id );
@@ -494,7 +496,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
          if( now - block->timestamp < fc::minutes(5) || (blk_num % 1000 == 0) ) {
             ilog("Received block ${id}... #${n} @ ${t} signed by ${p} "
-                 "[trxs: ${count}, lib: ${lib}, conf: ${confs}, net: ${net}, cpu: ${cpu}, elapsed: ${elapsed}, time: ${time}, latency: ${latency} ms]",
+                 "[trxs: ${count}, lib: ${lib}, confirmed: ${confs}, net: ${net}, cpu: ${cpu}, elapsed: ${elapsed}, time: ${time}, latency: ${latency} ms]",
                  ("p",block->producer)("id",id.str().substr(8,16))("n",blk_num)("t",block->timestamp)
                  ("count",block->transactions.size())("lib",chain.last_irreversible_block_num())
                  ("confs", block->confirmed)("net", br.total_net_usage)("cpu", br.total_cpu_usage_us)
@@ -502,7 +504,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
                  ("latency", (now - block->timestamp).count()/1000 ) );
             if( chain.get_read_mode() != db_read_mode::IRREVERSIBLE && hbs->id != id && hbs->block != nullptr ) { // not applied to head
                ilog("Block not applied to head ${id}... #${n} @ ${t} signed by ${p} "
-                    "[trxs: ${count}, dpos: ${dpos}, conf: ${confs}, net: ${net}, cpu: ${cpu}, elapsed: ${elapsed}, time: ${time}, latency: ${latency} ms]",
+                    "[trxs: ${count}, dpos: ${dpos}, confirmed: ${confs}, net: ${net}, cpu: ${cpu}, elapsed: ${elapsed}, time: ${time}, latency: ${latency} ms]",
                     ("p",hbs->block->producer)("id",hbs->id.str().substr(8,16))("n",hbs->block_num)("t",hbs->block->timestamp)
                     ("count",hbs->block->transactions.size())("dpos", hbs->dpos_irreversible_blocknum)
                     ("confs", hbs->block->confirmed)("net", br.total_net_usage)("cpu", br.total_cpu_usage_us)
@@ -668,6 +670,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          exhausted
       };
 
+      inline bool should_interrupt_start_block( const fc::time_point& deadline ) const;
       start_block_result start_block();
 
       fc::time_point calculate_pending_block_time() const;
@@ -957,23 +960,9 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
-   my->_incoming_block_subscription = app().get_channel<incoming::channels::block>().subscribe(
-         [this](const signed_block_ptr& block) {
-      try {
-         my->on_incoming_block(block, {});
-      } LOG_AND_DROP();
-   });
-
-   my->_incoming_transaction_subscription = app().get_channel<incoming::channels::transaction>().subscribe(
-         [this](const packed_transaction_ptr& trx) {
-      try {
-         my->on_incoming_transaction_async(trx, false, false, false, [](const auto&){});
-      } LOG_AND_DROP();
-   });
-
    my->_incoming_block_sync_provider = app().get_method<incoming::methods::block_sync>().register_provider(
-         [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id) {
-      return my->on_incoming_block(block, block_id);
+         [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
+      return my->on_incoming_block(block, block_id, bsp);
    });
 
    my->_incoming_transaction_async_provider = app().get_method<incoming::methods::transaction_async>().register_provider(
@@ -1076,6 +1065,7 @@ void producer_plugin::plugin_shutdown() {
    my->_unapplied_transactions.clear();
 
    app().post( 0, [me = my](){} ); // keep my pointer alive until queue is drained
+   fc_ilog(_log, "exit shutdown");
 }
 
 void producer_plugin::handle_sighup() {
@@ -1587,6 +1577,14 @@ fc::time_point producer_plugin_impl::calculate_block_deadline( const fc::time_po
    }
 }
 
+bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline ) const {
+   if( _pending_block_mode == pending_block_mode::producing ) {
+      return deadline <= fc::time_point::now();
+   }
+   // if we can produce then honor deadline so production starts on time
+   return (!_producers.empty() && deadline <= fc::time_point::now()) || _received_block;
+}
+
 producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    chain::controller& chain = chain_plug->chain();
 
@@ -1762,12 +1760,16 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
       }
 
       try {
+         _account_fails.clear();
+
          if( !remove_expired_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
          if( !remove_expired_blacklisted_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
-         if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(), preprocess_deadline ) )
+         if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(),
+                                                  [&](){ return should_interrupt_start_block( preprocess_deadline ); } ) ) {
             return start_block_result::exhausted;
+         }
 
          // limit execution of pending incoming to once per block
          auto incoming_itr = _unapplied_transactions.incoming_begin();
@@ -1789,7 +1791,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
          if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit()
             return start_block_result::failed;
-         if (preprocess_deadline <= fc::time_point::now() || block_is_exhausted()) {
+         if ( should_interrupt_start_block( preprocess_deadline ) || block_is_exhausted() ) {
             return start_block_result::exhausted;
          }
 
@@ -1821,7 +1823,7 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
    size_t num_expired_persistent = 0;
    size_t num_expired_other = 0;
    size_t orig_count = _unapplied_transactions.size();
-   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, deadline,
+   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block(deadline); },
          [&num_expired_persistent, &num_expired_other]( const packed_transaction_ptr& packed_trx_ptr, trx_enum_type trx_type ) {
             // expired exception is logged as part of next() call
             if( trx_type == trx_enum_type::persisted ) {
@@ -1831,7 +1833,7 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
             }
    });
 
-   if( exhausted ) {
+   if( exhausted && _pending_block_mode == pending_block_mode::producing ) {
       fc_wlog( _log, "Unable to process all expired transactions in unapplied queue before deadline, "
                      "Persistent expired ${persistent_expired}, Other expired ${other_expired}",
                ("persistent_expired", num_expired_persistent)("other_expired", num_expired_other) );
@@ -1857,7 +1859,7 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point
       int orig_count = _blacklisted_transactions.size();
 
       while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= lib_time) {
-         if (deadline <= fc::time_point::now()) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2098,7 +2100,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
       auto end_itr = (_pending_block_mode == pending_block_mode::producing) ?
                      _unapplied_transactions.unapplied_end()   : _unapplied_transactions.persisted_end();
       while( itr != end_itr ) {
-         if( deadline <= fc::time_point::now() ) {
+         if( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2280,7 +2282,7 @@ bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline
       size_t processed = 0;
       fc_dlog( _log, "Processing ${n} pending transactions", ("n", _unapplied_transactions.incoming_size()) );
       while( itr != end ) {
-         if (deadline <= fc::time_point::now()) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2326,6 +2328,7 @@ bool producer_plugin_impl::block_is_exhausted() const {
 // -> Idle
 // --> Start block B (block time y.000) at time x.500
 void producer_plugin_impl::schedule_production_loop() {
+   _received_block = false;
    _timer.cancel();
 
    auto result = start_block();
@@ -2475,6 +2478,7 @@ static auto maybe_make_debug_time_logger() -> std::optional<decltype(make_debug_
 
 void producer_plugin_impl::produce_block() {
    //ilog("produce_block ${t}", ("t", fc::time_point::now())); // for testing _produce_time_offset_us
+   auto start = fc::time_point::now();
    EOS_ASSERT(_pending_block_mode == pending_block_mode::producing, producer_exception, "called produce_block while not actually producing");
    chain::controller& chain = chain_plug->chain();
    EOS_ASSERT(chain.is_building_block(), missing_pending_block_state, "pending_block_state does not exist but it should, another plugin may have corrupted it");
@@ -2499,7 +2503,8 @@ void producer_plugin_impl::produce_block() {
    }
 
    //idump( (fc::time_point::now() - chain.pending_block_time()) );
-   chain.finalize_block( [&]( const digest_type& d ) {
+   controller::block_report br;
+   chain.finalize_block( br, [&]( const digest_type& d ) {
       auto debug_logger = maybe_make_debug_time_logger();
       vector<signature_type> sigs;
       sigs.reserve(relevant_providers.size());
@@ -2518,20 +2523,21 @@ void producer_plugin_impl::produce_block() {
    _account_fails.report(_idle_trx_time);
    _account_fails.clear();
 
-   controller::block_report br;
-   for( const auto& r : new_bs->block->transactions ) {
-      br.total_cpu_usage_us += r.cpu_usage_us;
-      br.total_net_usage += r.net_usage_words * 8;
-   }
+   br.total_time += fc::time_point::now() - start;
+
    ilog("Produced block ${id}... #${n} @ ${t} signed by ${p} "
-        "[trxs: ${count}, lib: ${lib}, confirmed: ${confs}, net: ${net}, cpu: ${cpu}]",
+        "[trxs: ${count}, lib: ${lib}, confirmed: ${confs}, net: ${net}, cpu: ${cpu}, elapsed: ${et}, time: ${tt}]",
         ("p",new_bs->header.producer)("id",new_bs->id.str().substr(8,16))
         ("n",new_bs->block_num)("t",new_bs->header.timestamp)
         ("count",new_bs->block->transactions.size())("lib",chain.last_irreversible_block_num())
-        ("net", br.total_net_usage)("cpu", br.total_cpu_usage_us)
+        ("net", br.total_net_usage)("cpu", br.total_cpu_usage_us)("et", br.total_elapsed_time)("tt", br.total_time)
         ("confs", new_bs->header.confirmed));
 }
 
+void producer_plugin::received_block() {
+   my->_received_block = true;
+}
+
 void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id, const packed_transaction_ptr& packed_trx_ptr, const char* reason) const {
    fc_dlog(_trx_log, "[TRX_TRACE] Speculative execution is REJECTING tx: ${trx}",
            ("entire_trx", packed_trx_ptr ? my->chain_plug->get_log_trx(packed_trx_ptr->get_transaction()) : fc::variant{trx_id}));

plugins/producer_plugin/test/test_subjective_billing.cpp

@@ -50,7 +50,7 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) {
       BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) );
 
       // expires transactions but leaves them in the decay at full value
-      sub_bill.remove_expired( log, now + fc::microseconds(1), now, fc::time_point::maximum() );
+      sub_bill.remove_expired( log, now + fc::microseconds(1), now, [](){ return false; } );
 
       BOOST_CHECK_EQUAL( 13+11, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) );
@@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) {
       BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) );
 
-      sub_bill.remove_expired( log, now, now, fc::time_point::maximum() );
+      sub_bill.remove_expired( log, now, now, [](){ return false; } );
       BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) );
 
@@ -148,7 +148,7 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) {
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(a, endtime) );
       BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(b, endtime) );
 
-      sub_bill.remove_expired( log, now + fc::microseconds(1), now, fc::time_point::maximum() );
+      sub_bill.remove_expired( log, now + fc::microseconds(1), now, [](){ return false; } );
       BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) );
 

plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/file_space_handler.hpp

@@ -59,7 +59,8 @@ namespace eosio::resource_monitor {
 
             if ( info.available < fs.shutdown_available ) {
                if (output_threshold_warning) {
-                  wlog("Space usage warning: ${path}'s file system exceeded threshold ${threshold}%, available: ${available}, Capacity: ${capacity}, shutdown_available: ${shutdown_available}", ("path", fs.path_name.string()) ("threshold", shutdown_threshold) ("available", info.available) ("capacity", info.capacity) ("shutdown_available", fs.shutdown_available));
+                  elog("Space usage warning: ${path}'s file system exceeded threshold ${threshold}%, available: ${available}, Capacity: ${capacity}, shutdown_available: ${shutdown_available}",
+                       ("path", fs.path_name.string())("threshold", shutdown_threshold)("available", info.available)("capacity", info.capacity)("shutdown_available", fs.shutdown_available));
                }
                return true;
             } else if ( info.available < fs.warning_available && output_threshold_warning ) {
@@ -114,7 +115,7 @@ namespace eosio::resource_monitor {
    
    void space_monitor_loop() {
       if ( is_threshold_exceeded() && shutdown_on_exceeded ) {
-         wlog("Shutting down");
+         elog("Shutting down, file system exceeded threshold");
          appbase::app().quit(); // This will gracefully stop Nodeos
          return;
       }

plugins/state_history_plugin/state_history_plugin.cpp

@@ -6,12 +6,12 @@
 #include <eosio/state_history/serialization.hpp>
 #include <eosio/state_history/trace_converter.hpp>
 #include <eosio/state_history_plugin/state_history_plugin.hpp>
+#include <eosio/chain/thread_utils.hpp>
 
 #include <boost/asio/bind_executor.hpp>
 #include <boost/asio/ip/host_name.hpp>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/local/stream_protocol.hpp>
-#include <boost/asio/strand.hpp>
 #include <boost/beast/core.hpp>
 #include <boost/beast/websocket.hpp>
 #include <boost/signals2/connection.hpp>
@@ -38,10 +38,6 @@ void async_teardown(role_type, unixs::socket& sock, TeardownHandler&& handler) {
 }
 #endif
 
-// overload pattern for variant visitation
-template<class... Ts> struct overload : Ts... { using Ts::operator()...; };
-template<class... Ts> overload(Ts...) -> overload<Ts...>;
-
 namespace eosio {
 using namespace chain;
 using namespace state_history;
@@ -66,6 +62,14 @@ auto catch_and_log(F f) {
 }
 
 struct state_history_plugin_impl : std::enable_shared_from_this<state_history_plugin_impl> {
+
+   struct session_base {
+      virtual void send_update(const block_state_ptr& block_state) = 0;
+      virtual void close()                                         = 0;
+      virtual ~session_base() = default;
+      std::optional<get_blocks_request_v0>       current_request;
+   };
+
    chain_plugin*                    chain_plug = nullptr;
    std::optional<state_history_log> trace_log;
    std::optional<state_history_log> chain_state_log;
@@ -78,17 +82,14 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
    uint16_t                         endpoint_port = 8080;
    string                           unix_path;
    state_history::trace_converter   trace_converter;
+   std::set<std::shared_ptr<session_base>> session_set;
 
    using acceptor_type = std::variant<std::unique_ptr<tcp::acceptor>, std::unique_ptr<unixs::acceptor>>;
-   std::set<acceptor_type>          acceptor;
+   std::set<acceptor_type>          acceptors;
 
-   std::thread                                                              thr;
-   boost::asio::io_context                                                  ctx;
-   boost::asio::io_context::strand                                          work_strand{ctx};
-   boost::asio::executor_work_guard<boost::asio::io_context::executor_type> work_guard =
-       boost::asio::make_work_guard(ctx);
+   named_thread_pool                thread_pool{"SHiP", 1, true}; // use of executor assumes only one thread
 
-   void get_log_entry(state_history_log& log, uint32_t block_num, std::optional<bytes>& result) {
+   static void get_log_entry(state_history_log& log, uint32_t block_num, std::optional<bytes>& result) {
       if (block_num < log.begin_block() || block_num >= log.end_block())
          return;
       state_history_log_header header;
@@ -105,7 +106,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
       result = state_history::zlib_decompress(compressed);
    }
 
-   void get_block(uint32_t block_num, const block_state_ptr& block_state, std::optional<bytes>& result) {
+   void get_block(uint32_t block_num, const block_state_ptr& block_state, std::optional<bytes>& result) const {
       chain::signed_block_ptr p;
       try {
          if( block_state && block_num == block_state->block_num ) {
@@ -131,40 +132,41 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
       return {};
    }
 
-   struct session_base {
-      virtual void send_update(const block_state_ptr& block_state) = 0;
-      virtual void close()                                         = 0;
-      virtual ~session_base() = default;
-      std::optional<get_blocks_request_v0>       current_request;
-   };
-
-
    template <typename SocketType>
    struct session : session_base, std::enable_shared_from_this<session<SocketType>> {
-     std::shared_ptr<state_history_plugin_impl>  plugin;
-      ws::stream<SocketType>                     socket_stream;
-      bool                                       sending  = false;
-      bool                                       sent_abi = false;
-      std::vector<std::vector<char>>             send_queue;
-      bool                                       need_to_send_update = false;
+      std::shared_ptr<state_history_plugin_impl> plugin;
+      std::optional<ws::stream<SocketType>>      socket_stream; // ship thread only after creation
+      bool                                       sending  = false; // ship thread only
+      std::vector<std::vector<char>>             send_queue; // ship thread only
+      bool                                       need_to_send_update = false; // main thread only
+
+      uint32_t                                   to_send_block_num = 0; // main thread only
+      std::optional<std::vector<block_position>::const_iterator> position_it; // main thread only
 
       session(std::shared_ptr<state_history_plugin_impl> plugin, SocketType socket)
           : plugin(std::move(plugin)), socket_stream(std::move(socket)) {}
 
       void start() {
          fc_ilog(_log, "incoming connection");
-         socket_stream.auto_fragment(false);
-         socket_stream.binary(true);
+         socket_stream->auto_fragment(false);
+         socket_stream->binary(true);
          if constexpr (std::is_same_v<SocketType, tcp::socket>) {
-            socket_stream.next_layer().set_option(boost::asio::ip::tcp::no_delay(true));
+            socket_stream->next_layer().set_option(boost::asio::ip::tcp::no_delay(true));
          }
-         socket_stream.next_layer().set_option(boost::asio::socket_base::send_buffer_size(1024 * 1024));
-         socket_stream.next_layer().set_option(boost::asio::socket_base::receive_buffer_size(1024 * 1024));
-        
-         socket_stream.async_accept([self = this->shared_from_this()](boost::system::error_code ec) {
+         socket_stream->next_layer().set_option(boost::asio::socket_base::send_buffer_size(1024 * 1024));
+         socket_stream->next_layer().set_option(boost::asio::socket_base::receive_buffer_size(1024 * 1024));
+
+         socket_stream->async_accept([self = this->shared_from_this()](boost::system::error_code ec) {
             self->callback(ec, "async_accept", [self] {
-               self->start_read();
-               self->send(state_history_plugin_abi);
+               self->socket_stream->binary(false);
+               self->socket_stream->async_write(
+                     boost::asio::buffer(state_history_plugin_abi, strlen(state_history_plugin_abi)),
+                     [self](boost::system::error_code ec, size_t) {
+                        self->callback(ec, "async_write", [self] {
+                           self->socket_stream->binary(true);
+                           self->start_read();
+                        });
+                     });
             });
          });
       }
@@ -172,7 +174,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
 
       void start_read() {
          auto in_buffer = std::make_shared<boost::beast::flat_buffer>();
-         socket_stream.async_read(
+         socket_stream->async_read(
              *in_buffer, [self = this->shared_from_this(), in_buffer](boost::system::error_code ec, size_t) {
                 self->callback(ec, "async_read", [self, in_buffer] {
                    auto d = boost::asio::buffer_cast<char const*>(boost::beast::buffers_front(in_buffer->data()));
@@ -180,7 +182,8 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
                    fc::datastream<const char*> ds(d, s);
                    state_request               req;
                    fc::raw::unpack(ds, req);
-                   app().post(priority::medium, [self, req = std::move(req)]() mutable { std::visit(*self, req); });
+                   app().post(priority::medium,
+                              [self, req = std::move(req)]() mutable { std::visit(*self, std::move(req)); });
                    self->start_read();
                 });
              });
@@ -190,39 +193,33 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
       void send() {
          if (sending)
             return;
-         if (send_queue.empty())
-            return send_update();
+         if (send_queue.empty()) {
+            app().post(priority::medium, [self = this->shared_from_this()]() {
+               self->send_update();
+            });
+            return;
+         }
          sending = true;
-         socket_stream.binary(sent_abi);
-         sent_abi = true;
-         socket_stream.async_write(             //
-             boost::asio::buffer(send_queue[0]), //
-             [self = this->shared_from_this()](boost::system::error_code ec, size_t) {
-                self->callback(ec, "async_write", [self] {
-                   self->send_queue.erase(self->send_queue.begin());
-                   self->sending = false;
-                   self->send();
-                });
-             });
-      }
 
-      void send(const char* s) {
-          boost::asio::post(this->plugin->work_strand, [self = this->shared_from_this(), str = s ]() {
-            self->send_queue.push_back({str, str + strlen(str)});
-            self->send();
-         });
+         socket_stream->async_write(
+             boost::asio::buffer(send_queue[0]),
+             [self = this->shared_from_this()](boost::system::error_code ec, size_t) {
+                self->send_queue.erase( self->send_queue.begin() );
+                self->sending = false;
+                self->callback(ec, "async_write", [self] { self->send(); });
+             });
       }
 
       template <typename T>
       void send(T obj) {
-         boost::asio::post(this->plugin->work_strand, [self = this->shared_from_this(), obj = std::move(obj) ]() {
+         boost::asio::post(this->plugin->thread_pool.get_executor(), [self = this->shared_from_this(), obj = std::move(obj) ]() mutable {
             self->send_queue.emplace_back(fc::raw::pack(state_result{std::move(obj)}));
             self->send();
          });
       }
-     
-      using result_type = void;
-      void operator()(get_status_request_v0&) {
+
+      // called from main thread
+      void operator()(get_status_request_v0&&) {
          fc_ilog(_log, "got get_status_request_v0");
          auto&                chain = plugin->chain_plug->chain();
          get_status_result_v0 result;
@@ -241,29 +238,36 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
          send(std::move(result));
       }
 
-      void operator()(get_blocks_request_v0& req) {
-         fc_ilog(_log, "received get_blocks_request_v0 = ${req}", ("req",req) );
+      // called from main thread
+      void operator()(get_blocks_request_v0&& req) {
+         fc_dlog(_log, "received get_blocks_request_v0 = ${req}", ("req",req) );
+         to_send_block_num = req.start_block_num;
          for (auto& cp : req.have_positions) {
             if (req.start_block_num <= cp.block_num)
                continue;
             auto id = plugin->get_block_id(cp.block_num);
-            if (!id || *id != cp.block_id)
-               req.start_block_num = std::min(req.start_block_num, cp.block_num);
-
             if (!id) {
+               to_send_block_num = std::min(to_send_block_num, cp.block_num);
                fc_dlog(_log, "block ${block_num} is not available", ("block_num", cp.block_num));
             } else if (*id != cp.block_id) {
+               to_send_block_num = std::min(to_send_block_num, cp.block_num);
                fc_dlog(_log, "the id for block ${block_num} in block request have_positions does not match the existing", ("block_num", cp.block_num));
-            }         
+            }
          }
-         req.have_positions.clear();
-         fc_dlog(_log, "  get_blocks_request_v0 start_block_num set to ${num}", ("num", req.start_block_num));
-         current_request = req;
+
+         fc_dlog(_log, "  get_blocks_request_v0 start_block_num set to ${num}", ("num", to_send_block_num));
+
+         if( !req.have_positions.empty() ) {
+            position_it = req.have_positions.begin();
+         }
+
+         current_request = std::move(req);
          send_update(true);
       }
 
-      void operator()(get_blocks_ack_request_v0& req) {
-         fc_ilog(_log, "received get_blocks_ack_request_v0 = ${req}", ("req",req));
+      // called from main thread
+      void operator()(get_blocks_ack_request_v0&& req) {
+         fc_dlog(_log, "received get_blocks_ack_request_v0 = ${req}", ("req",req));
          if (!current_request) {
             fc_dlog(_log, " no current get_blocks_request_v0, discarding the get_blocks_ack_request_v0");
             return;
@@ -272,9 +276,10 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
          send_update();
       }
 
-      void send_update(get_blocks_result_v0 result, const block_state_ptr& block_state) {
+      // must run on main thread
+      void send_update(get_blocks_result_v0&& result, const block_state_ptr& head_block_state) {
          need_to_send_update = true;
-         if (!send_queue.empty() || !current_request || !current_request->max_messages_in_flight)
+         if (!current_request || !current_request->max_messages_in_flight)
             return;
 
          auto& chain              = plugin->chain_plug->chain();
@@ -282,151 +287,130 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
          uint32_t current =
              current_request->irreversible_only ? result.last_irreversible.block_num : result.head.block_num;
 
-         if (current_request->start_block_num <= current &&
-             current_request->start_block_num < current_request->end_block_num) {
-            auto block_id = plugin->get_block_id(current_request->start_block_num);
-
-            if (block_id) {
-               result.this_block  = block_position{current_request->start_block_num, *block_id};
-               auto prev_block_id = plugin->get_block_id(current_request->start_block_num - 1);
-               if (prev_block_id)
-                  result.prev_block = block_position{current_request->start_block_num - 1, *prev_block_id};
-               if (current_request->fetch_block) {
-                  plugin->get_block( current_request->start_block_num, block_state, result.block );
-               }
-               if (current_request->fetch_traces && plugin->trace_log)
-                  plugin->get_log_entry(*plugin->trace_log, current_request->start_block_num, result.traces);
-               if (current_request->fetch_deltas && plugin->chain_state_log)
-                  plugin->get_log_entry(*plugin->chain_state_log, current_request->start_block_num, result.deltas);
-            }
-            ++current_request->start_block_num;
+         if (to_send_block_num > current || to_send_block_num >= current_request->end_block_num) {
+            fc_dlog( _log, "Not sending, to_send_block_num: ${s}, current: ${c} current_request.end_block_num: ${b}",
+                     ("s", to_send_block_num)("c", current)("b", current_request->end_block_num) );
+            return;
          }
 
-         auto& block_num = current_request->start_block_num;
-         auto get_blk = [&chain, block_num, block_state]() -> signed_block_ptr {
-            try {
-               if (block_state && block_state->block_num == block_num)
-                  return block_state->block;
-               return chain.fetch_block_by_number(block_num);
-            } catch (...) {
-               return {};
+         auto block_id = plugin->get_block_id(to_send_block_num);
+
+         if (block_id && position_it && (*position_it)->block_num == to_send_block_num) {
+            // This branch happens when the head block of nodeos is behind the head block of connecting client.
+            // In addition, the client told us the corresponding block id for block_num we are going to send.
+            // We can send the block when the block_id is different.
+            auto& itr = *position_it;
+            auto block_id_seen_by_client = itr->block_id;
+            ++itr;
+            if (itr == current_request->have_positions.end())
+               position_it.reset();
+
+            if(block_id_seen_by_client == *block_id) {
+               ++to_send_block_num;
+               return;
             }
-         };
-         auto block = get_blk();
+         }
+
+         if (block_id) {
+            result.this_block  = block_position{to_send_block_num, *block_id};
+            auto prev_block_id = plugin->get_block_id(to_send_block_num - 1);
+            if (prev_block_id)
+               result.prev_block = block_position{to_send_block_num - 1, *prev_block_id};
+            if (current_request->fetch_block) {
+               plugin->get_block( to_send_block_num, head_block_state, result.block );
+            }
+            if (current_request->fetch_traces && plugin->trace_log) {
+               get_log_entry(*plugin->trace_log, to_send_block_num, result.traces);
+            }
+            if (current_request->fetch_deltas && plugin->chain_state_log) {
+               plugin->get_log_entry(*plugin->chain_state_log, to_send_block_num, result.deltas);
+            }
+         }
+         ++to_send_block_num;
 
          // during syncing if block is older than 5 min, log every 1000th block
-         bool fresh_block = block && fc::time_point::now() - block->timestamp < fc::minutes(5);
+         bool fresh_block = fc::time_point::now() - chain.head_block_time() < fc::minutes(5);
          if( fresh_block || (result.this_block && result.this_block->block_num % 1000 == 0) ) {
             fc_ilog(_log, "pushing result "
-                  "{\"head\":{\"block_num\":${head}},\"last_irreversible\":{\"block_num\":${last_irr}},\"this_block\":{"
-                  "\"block_num\":${this_block}}} to send queue",
-                  ("head", result.head.block_num)("last_irr", result.last_irreversible.block_num)(
-                        "this_block", result.this_block ? result.this_block->block_num : fc::variant()));
+                          "{\"head\":{\"block_num\":${head}},\"last_irreversible\":{\"block_num\":${last_irr}},\"this_block\":{"
+                          "\"block_num\":${this_block}}} to send queue",
+                    ("head", result.head.block_num)("last_irr", result.last_irreversible.block_num)(
+                     "this_block", result.this_block ? result.this_block->block_num : fc::variant()));
          }
 
          send(std::move(result));
          --current_request->max_messages_in_flight;
-         need_to_send_update = current_request->start_block_num <= current &&
-                               current_request->start_block_num < current_request->end_block_num;
+         need_to_send_update = to_send_block_num <= current &&
+                               to_send_block_num < current_request->end_block_num;
       }
 
-      void send_update(const block_state_ptr& block_state) {
-         need_to_send_update = true;
-         if (!send_queue.empty() || !current_request || !current_request->max_messages_in_flight)
+      // called from the main thread
+      void send_update(const block_state_ptr& block_state) override {
+         if (!current_request || !current_request->max_messages_in_flight)
             return;
+
          get_blocks_result_v0 result;
          result.head = {block_state->block_num, block_state->id};
          send_update(std::move(result), block_state);
       }
 
+      // called from the main thread
       void send_update(bool changed = false) {
-         if (changed)
-            need_to_send_update = true;
-         if (!send_queue.empty() || !need_to_send_update || !current_request ||
-             !current_request->max_messages_in_flight)
-            return;
-         auto&                chain = plugin->chain_plug->chain();
-         get_blocks_result_v0 result;
-         result.head = {chain.head_block_num(), chain.head_block_id()};
-         send_update(std::move(result), {});
-      }
-
-      template <typename F>
-      void catch_and_close(F f) {
-         try {
-            f();
-         } catch (const fc::exception& e) {
-            fc_elog(_log, "${e}", ("e", e.to_detail_string()));
-            close();
-         } catch (const std::exception& e) {
-            fc_elog(_log,"${e}", ("e", e.what()));
-            close();
-         } catch (...) {
-            fc_elog(_log, "unknown exception");
-            close();
+         if (changed || need_to_send_update) {
+            auto& chain = plugin->chain_plug->chain();
+            send_update(chain.head_block_state());
          }
       }
 
+      // called from ship thread
       template <typename F>
       void callback(boost::system::error_code ec, const char* what, F f) {
-         app().post(priority::medium, [=]() {
-            if (plugin->stopping)
-               return;
-            if (ec)
-               return on_fail(ec, what);
-            catch_and_close(f);
-         });
-      }
+         if (this->plugin->stopping)
+            return;
 
-      void on_fail(boost::system::error_code ec, const char* what) {
-         try {
+         if (ec) {
             if (ec == boost::asio::error::eof) {
                fc_dlog(_log, "${w}: ${m}", ("w", what)("m", ec.message()));
             } else {
                fc_elog(_log, "${w}: ${m}", ("w", what)("m", ec.message()));
             }
-            close();
+            close_i();
+            return;
+         }
+
+         try {
+            f();
+         } catch (const fc::exception& e) {
+            fc_elog(_log, "${e}", ("e", e.to_detail_string()));
+            close_i();
+         } catch (const std::exception& e) {
+            fc_elog(_log,"${e}", ("e", e.what()));
+            close_i();
          } catch (...) {
-            fc_elog(_log,"uncaught exception on close");
+            fc_elog(_log, "unknown exception");
+            close_i();
          }
       }
 
-      void close() {
+      void close() override {
+         boost::asio::post(plugin->thread_pool.get_executor(), [self = this->shared_from_this()]() {
+            self->close_i();
+         });
+      }
+
+      // called from ship thread
+      void close_i() {
          boost::system::error_code ec;
-         socket_stream.next_layer().close(ec);
+         socket_stream->next_layer().close(ec);
          if (ec) {
             fc_elog(_log, "close: ${m}", ("m", ec.message()));
          }
-         plugin->sessions.remove(this->shared_from_this());
-      }
-   };
-
-   class session_manager_t {
-      std::mutex                                                  mx;
-      boost::container::flat_set<std::shared_ptr<session_base>>   session_set;
-
-    public:
-      template <typename SocketType>
-      void add(std::shared_ptr<state_history_plugin_impl> plugin, std::shared_ptr<SocketType> socket) {
-         auto s = std::make_shared<session<SocketType>>(plugin, std::move(*socket));
-         s->start();
-         std::lock_guard lock(mx);
-         session_set.insert(std::move(s));
+         socket_stream.reset();
+         app().post(priority::high,
+                    [self = this->shared_from_this(), plugin=plugin]() { plugin->session_set.erase(self); });
       }
 
-      void remove(std::shared_ptr<session_base> s) {
-         std::lock_guard lock(mx);
-         session_set.erase(s);
-      }
-
-      template <typename F>
-      void for_each(F&& f) {
-         std::lock_guard lock(mx);
-         for (auto& s : session_set) {
-            f(s);
-         }
-      }
-   } sessions;
+   }; // session
 
    void listen() {
       boost::system::error_code ec;
@@ -435,10 +419,10 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
          if (!ec)
             return;
          fc_elog(_log, "${w}: ${m}", ("w", what)("m", ec.message()));
-         EOS_ASSERT(false, plugin_exception, "unable to open listen socket");
+         FC_THROW_EXCEPTION(plugin_exception, "unable to open listen socket");
       };
 
-      auto init_tcp_acceptor  = [&]() { acceptor.insert(std::make_unique<tcp::acceptor>(app().get_io_service())); };
+      auto init_tcp_acceptor  = [&]() { acceptors.insert(std::make_unique<tcp::acceptor>(thread_pool.get_executor())); };
       auto init_unix_acceptor = [&]() {
          // take a sniff and see if anything is already listening at the given socket path, or if the socket path exists
          //  but nothing is listening
@@ -457,16 +441,16 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
                ec = test_ec;
          }
          check_ec("open");
-         acceptor.insert(std::make_unique<unixs::acceptor>(this->ctx));
+         acceptors.insert(std::make_unique<unixs::acceptor>(thread_pool.get_executor()));
       };
 
       // create and configure acceptors, can be both
-      if (endpoint_address.size()) init_tcp_acceptor();
-      if (unix_path.size())        init_unix_acceptor();
+      if (!endpoint_address.empty()) init_tcp_acceptor();
+      if (!unix_path.empty())        init_unix_acceptor();
 
       // start it
-      std::for_each(acceptor.begin(), acceptor.end(), [&](const acceptor_type& acc) {
-         std::visit(overload{[&](const std::unique_ptr<tcp::acceptor>& tcp_acc) {
+      std::for_each(acceptors.begin(), acceptors.end(), [&](const acceptor_type& acc) {
+         std::visit(overloaded{[&](const std::unique_ptr<tcp::acceptor>& tcp_acc) {
                                 auto address  = boost::asio::ip::make_address(endpoint_address);
                                 auto endpoint = tcp::endpoint{address, endpoint_port};
                                 tcp_acc->open(endpoint.protocol(), ec);
@@ -493,27 +477,32 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
 
    template <typename Acceptor>
    void do_accept(Acceptor& acceptor) {
-      auto socket = std::make_shared<typename Acceptor::protocol_type::socket>(this->ctx);
-      acceptor.async_accept(*socket, [self = shared_from_this(), this, socket, &acceptor](const boost::system::error_code& ec) {
-         if (stopping)
+      auto socket = std::make_shared<typename Acceptor::protocol_type::socket>(this->thread_pool.get_executor());
+      // &acceptor kept alive by self, reference into acceptors set
+      acceptor.async_accept(*socket, [self = shared_from_this(), socket, &acceptor](const boost::system::error_code& ec) {
+         if (self->stopping)
             return;
          if (ec) {
             if (ec == boost::system::errc::too_many_files_open)
-               catch_and_log([&] { do_accept(acceptor); });
+               catch_and_log([&] { self->do_accept(acceptor); });
             return;
          }
          catch_and_log([&] {
-            sessions.add(self, socket);
+            auto s = std::make_shared<session<typename Acceptor::protocol_type::socket>>(self, std::move(*socket));
+            s->start();
+            app().post(priority::high, [self, s]() mutable { self->session_set.insert(std::move(s)); });
          });
-         catch_and_log([&] { do_accept(acceptor); });
+         catch_and_log([&] { self->do_accept(acceptor); });
       });
    }
 
+   // called from main thread
    void on_applied_transaction(const transaction_trace_ptr& p, const packed_transaction_ptr& t) {
       if (trace_log)
          trace_converter.add_transaction(p, t);
    }
 
+   // called from main thread
    void on_accepted_block(const block_state_ptr& block_state) {
       try {
          store_traces(block_state);
@@ -530,22 +519,23 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
              "the process");
       }
 
-      sessions.for_each([&block_state](auto& p) {
-         if (p) {
-            if (p->current_request && block_state->block_num < p->current_request->start_block_num)
-               p->current_request->start_block_num = block_state->block_num;
-            p->send_update(block_state);
-         }
-      });
+      for( auto& s : session_set ) {
+         s->send_update(block_state);
+      }
    }
 
-   void on_block_start(uint32_t block_num) { clear_caches(); }
+   // called from main thread
+   void on_block_start(uint32_t block_num) {
+      clear_caches();
+   }
 
+   // called from main thread
    void clear_caches() {
       trace_converter.cached_traces.clear();
       trace_converter.onblock_trace.reset();
    }
 
+   // called from main thread
    void store_traces(const block_state_ptr& block_state) {
       if (!trace_log)
          return;
@@ -565,6 +555,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
       });
    }
 
+   // called from main thread
    void store_chain_state(const block_state_ptr& block_state) {
       if (!chain_state_log)
          return;
@@ -590,12 +581,26 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
             stream.write(deltas_bin.data(), deltas_bin.size());
       });
    } // store_chain_state
+
+   ~state_history_plugin_impl() {
+      std::for_each(acceptors.begin(), acceptors.end(), [&](const acceptor_type& acc) {
+         std::visit(overloaded{
+            []( const std::unique_ptr<unixs::acceptor>& a ) {
+               boost::system::error_code ec;
+               if( const auto ep = a->local_endpoint( ec ); !ec )
+                  ::unlink( ep.path().c_str() );
+            },
+            []( const std::unique_ptr<tcp::acceptor>& a) {}
+         }, acc);
+      });
+   }
+
 };   // state_history_plugin_impl
 
 state_history_plugin::state_history_plugin()
     : my(std::make_shared<state_history_plugin_impl>()) {}
 
-state_history_plugin::~state_history_plugin() {}
+state_history_plugin::~state_history_plugin() = default;
 
 void state_history_plugin::set_program_options(options_description& cli, options_description& cfg) {
    auto options = cfg.add_options();
@@ -643,7 +648,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) {
 
       auto ip_port = options.at("state-history-endpoint").as<string>();
 
-      if (ip_port.size()) {
+      if (!ip_port.empty()) {
          auto port            = ip_port.substr(ip_port.find(':') + 1, ip_port.size());
          auto host            = ip_port.substr(0, ip_port.find(':'));
          my->endpoint_address = host;
@@ -693,8 +698,8 @@ void state_history_plugin::plugin_startup() {
    handle_sighup(); // setup logging
 
    try {
-      my->thr = std::thread([ptr = my.get()] { ptr->ctx.run(); });
       my->listen();
+      my->thread_pool.start();
    } catch (std::exception& ex) {
       appbase::app().quit();
    }
@@ -704,15 +709,11 @@ void state_history_plugin::plugin_shutdown() {
    my->applied_transaction_connection.reset();
    my->accepted_block_connection.reset();
    my->block_start_connection.reset();
-   my->sessions.for_each([](auto& s) { s->close(); });
+   std::for_each(my->session_set.begin(), my->session_set.end(), [](auto& s){ s->close(); } );
    my->stopping = true;
    my->trace_log->stop();
    my->chain_state_log->stop();
-   if (my->thr.joinable()) {
-      my->work_guard.reset();
-      my->ctx.stop();
-      my->thr.join();
-   }
+   my->thread_pool.stop();
 }
 
 void state_history_plugin::handle_sighup() { fc::logger::update(logger_name, _log); }

plugins/test_control_api_plugin/test_control.swagger.yaml

@@ -1,12 +1,13 @@
 openapi: 3.0.0
 info:
   title: Test Control API
+  description: Nodeos Test Control API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eos.io
+    url: https://eosnetwork.com
 tags:
   - name: eosio
 servers:

plugins/test_control_plugin/test_control_plugin.cpp

@@ -1,5 +1,4 @@
 #include <eosio/test_control_plugin/test_control_plugin.hpp>
-#include <atomic>
 
 namespace fc { class variant; }
 
@@ -9,7 +8,7 @@ static appbase::abstract_plugin& _test_control_plugin = app().register_plugin<te
 
 class test_control_plugin_impl {
 public:
-   test_control_plugin_impl(chain::controller& c) : _chain(c) {}
+   explicit test_control_plugin_impl(chain::controller& c) : _chain(c) {}
    void connect();
    void disconnect();
    void kill_on_lib(account_name prod, uint32_t where_in_seq);
@@ -24,12 +23,11 @@ private:
    std::optional<boost::signals2::scoped_connection> _irreversible_block_connection;
    chain::controller&  _chain;
    account_name        _producer;
-   int32_t             _where_in_sequence{-1};
-   int32_t             _producer_sequence{-1};
-   uint32_t            _first_sequence_timeslot{0};
+   uint32_t            _where_in_sequence{};
    bool                _clean_producer_sequence{false};
-   std::atomic_bool    _track_lib;
-   std::atomic_bool    _track_head;
+   bool                _started_production_round{false};
+   bool                _track_lib{false};
+   bool                _track_head{false};
 };
 
 void test_control_plugin_impl::connect() {
@@ -59,68 +57,54 @@ void test_control_plugin_impl::accepted_block(const chain::block_state_ptr& bsp)
 }
 
 void test_control_plugin_impl::process_next_block_state(const chain::block_state_ptr& bsp) {
+   // Tests expect the shutdown only after signaling a producer shutdown and seeing a full production cycle
    const auto block_time = _chain.head_block_time() + fc::microseconds(chain::config::block_interval_us);
    const auto& producer_authority = bsp->get_scheduled_producer(block_time);
    const auto producer_name = producer_authority.producer_name;
-   if (_producer != account_name())
-      ilog("producer ${cprod}, looking for ${lprod}", ("cprod", producer_name.to_string())("lprod", _producer.to_string()));
-
-   // start counting sequences for this producer (once we have a sequence that we saw the initial block for that producer)
-   if (producer_name == _producer && _clean_producer_sequence) {
-      auto slot = bsp->block->timestamp.slot;
-      _producer_sequence += 1;
-      ilog("producer ${prod} seq: ${seq} slot: ${slot}", 
-           ("prod", producer_name.to_string())
-           ("seq", _producer_sequence+1) // _producer_sequence is index, aligning it with slot number
-           ("slot", slot - _first_sequence_timeslot));
-
-      bool last_slot = false;
-      if (_where_in_sequence + 1 == chain::config::producer_repetitions){
-         auto last_slot_time = _first_sequence_timeslot + chain::config::producer_repetitions;
-         last_slot = slot == last_slot_time;
+   const auto slot = bsp->block->timestamp.slot % chain::config::producer_repetitions;
+   if (_producer != account_name()) {
+      if( _producer != producer_name ) _clean_producer_sequence = true;
+      if( _clean_producer_sequence ) {
+         ilog( "producer ${cprod} slot ${pslot}, looking for ${lprod} slot ${slot}",
+               ("cprod", producer_name)("pslot", slot)("lprod", _producer)("slot", _where_in_sequence) );
+      } else {
+         ilog( "producer ${cprod} slot ${pslot}, looking for start of ${lprod} production round",
+               ("cprod", producer_name)("pslot", slot)("lprod", _producer) );
       }
+   }
 
-      if (_producer_sequence >= _where_in_sequence || last_slot) {
-         int32_t slot_index = slot - _first_sequence_timeslot;
-         if (last_slot && slot_index > _producer_sequence + 1){
-            wlog("Producer produced less than ${n} blocks, ${l}th block is last in sequence. Likely performance issue, check timing",
-                 ("n", chain::config::producer_repetitions)("l", _producer_sequence + 1));
-         }
+   // check started_production_round in case where producer does not produce a full round, still want to shut down
+   if( _clean_producer_sequence && (producer_name == _producer || _started_production_round) ) {
+      _started_production_round = true;
+      const auto current_slot = chain::block_timestamp_type( block_time ).slot % chain::config::producer_repetitions;
+      ilog( "producer ${prod} slot: ${slot}", ("prod", producer_name)("slot", slot) );
+
+      if( current_slot >= _where_in_sequence || producer_name != _producer ) {
          ilog("shutting down");
          app().quit();
       }
-   } else if (producer_name != _producer) {
-      if (_producer_sequence != -1)
-         ilog("producer changed, restarting");
-      _producer_sequence = -1;
-      // can now guarantee we are at the start of the producer
-      _clean_producer_sequence = true;
-
-      _first_sequence_timeslot = bsp->block->timestamp.slot;
    }
 }
 
 void test_control_plugin_impl::kill_on_lib(account_name prod, uint32_t where_in_seq) {
    _track_head = false;
    _producer = prod;
-   _where_in_sequence = static_cast<uint32_t>(where_in_seq);
-   _producer_sequence = -1;
+   _where_in_sequence = where_in_seq;
    _clean_producer_sequence = false;
+   _started_production_round = false;
    _track_lib = true;
 }
 
 void test_control_plugin_impl::kill_on_head(account_name prod, uint32_t where_in_seq) {
    _track_lib = false;
    _producer = prod;
-   _where_in_sequence = static_cast<uint32_t>(where_in_seq);
-   _producer_sequence = -1;
+   _where_in_sequence = where_in_seq;
    _clean_producer_sequence = false;
+   _started_production_round = false;
    _track_head = true;
 }
 
-test_control_plugin::test_control_plugin()
-{
-}
+test_control_plugin::test_control_plugin() = default;
 
 void test_control_plugin::set_program_options(options_description& cli, options_description& cfg) {
 }

plugins/trace_api_plugin/store_provider.cpp

@@ -307,8 +307,11 @@ namespace eosio::trace_api {
                _maintenance_condition.wait(lock);
             }
 
+            if (_maintenance_shutdown) {
+               break;
+            }
+
             uint32_t best_known_lib = _best_known_lib;
-            bool shutdown = _maintenance_shutdown;
             lock.unlock();
 
             log(std::string("Waking up to handle lib: ") + std::to_string(best_known_lib));
@@ -319,10 +322,6 @@ namespace eosio::trace_api {
                   last_lib = best_known_lib;
                } FC_LOG_AND_DROP();
             }
-
-            if (shutdown) {
-               break;
-            }
          }
       });
    }

plugins/trace_api_plugin/trace_api.swagger.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.0
 info:
   title: Trace API
-  description: See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
+  description: Nodeos Trace API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT

plugins/trace_api_plugin/trace_api_plugin.cpp

@@ -453,6 +453,7 @@ void trace_api_plugin::plugin_startup() {
 void trace_api_plugin::plugin_shutdown() {
    my->plugin_shutdown();
    rpc->plugin_shutdown();
+   fc_ilog( _log, "exit shutdown");
 }
 
 void trace_api_plugin::handle_sighup() {

programs/cleos/main.cpp

@@ -173,6 +173,7 @@ bool   tx_skip_sign = false;
 bool   tx_print_json = false;
 bool   tx_rtn_failure_trace = true;
 bool   tx_read_only = false;
+bool   tx_dry_run = false;
 bool   tx_retry_lib = false;
 uint16_t tx_retry_num_blocks = 0;
 bool   tx_use_old_rpc = false;
@@ -441,7 +442,7 @@ fc::variant push_transaction( signed_transaction& trx, const std::vector<public_
       trx.delay_sec = delaysec;
    }
 
-   if (!tx_skip_sign) {
+   auto sign_trx = [&] () {
       fc::variant required_keys;
       if (signing_keys.size() > 0) {
          required_keys = fc::variant(signing_keys);
@@ -450,6 +451,16 @@ fc::variant push_transaction( signed_transaction& trx, const std::vector<public_
          required_keys = determine_required_keys(trx);
       }
       sign_transaction(trx, required_keys, info.chain_id);
+   };
+   if (!tx_skip_sign) {
+      // sign dry-run transactions only when explcitly requested
+      if ( tx_dry_run ) {
+         if ( signing_keys.size() > 0 ) {
+            sign_trx();
+         }
+      } else {
+         sign_trx();
+      }
    }
 
    packed_transaction::compression_type compression = to_compression_type( tx_compression );
@@ -458,12 +469,14 @@ fc::variant push_transaction( signed_transaction& trx, const std::vector<public_
       EOSC_ASSERT( !(tx_retry_lib && tx_retry_num_blocks > 0), "ERROR: --retry-irreversible and --retry-num-blocks are mutually exclusive" );
       if (tx_use_old_rpc) {
          EOSC_ASSERT( !tx_read_only, "ERROR: --read-only can not be used with --use-old-rpc" );
+         EOSC_ASSERT( !tx_dry_run, "ERROR: --dry-run can not be used with --use-old-rpc" );
          EOSC_ASSERT( !tx_rtn_failure_trace, "ERROR: --return-failure-trace can not be used with --use-old-rpc" );
          EOSC_ASSERT( !tx_retry_lib, "ERROR: --retry-irreversible can not be used with --use-old-rpc" );
          EOSC_ASSERT( !tx_retry_num_blocks, "ERROR: --retry-num-blocks can not be used with --use-old-rpc" );
          return call( push_txn_func, packed_transaction( trx, compression ) );
       } else if (tx_use_old_send_rpc) {
          EOSC_ASSERT( !tx_read_only, "ERROR: --read-only can not be used with --use-old-send-rpc" );
+         EOSC_ASSERT( !tx_dry_run, "ERROR: --dry-run can not be used with --use-old-send-rpc" );
          EOSC_ASSERT( !tx_rtn_failure_trace, "ERROR: --return-failure-trace can not be used with --use-old-send-rpc" );
          EOSC_ASSERT( !tx_retry_lib, "ERROR: --retry-irreversible can not be used with --use-old-send-rpc" );
          EOSC_ASSERT( !tx_retry_num_blocks, "ERROR: --retry-num-blocks can not be used with --use-old-send-rpc" );
@@ -474,9 +487,9 @@ fc::variant push_transaction( signed_transaction& trx, const std::vector<public_
             throw;
          }
       } else {
-         if( tx_read_only ) {
-            EOSC_ASSERT( !tx_retry_lib, "ERROR: --retry-irreversible can not be used with --read-only" );
-            EOSC_ASSERT( !tx_retry_num_blocks, "ERROR: --retry-num-blocks can not be used with --read-only" );
+         if( tx_dry_run || tx_read_only ) {
+            EOSC_ASSERT( !tx_retry_lib, "ERROR: --retry-irreversible can not be used with --dry-run or --read-only" );
+            EOSC_ASSERT( !tx_retry_num_blocks, "ERROR: --retry-num-blocks can not be used with --dry-run or --read-only" );
             try {
                auto compute_txn_arg = fc::mutable_variant_object ("transaction",
                                                                   packed_transaction(trx,compression));
@@ -2759,8 +2772,16 @@ int main( int argc, char** argv ) {
    wallet_url = default_wallet_url;
 
    CLI::App app{"Command Line Interface to EOSIO Client"};
+
+   // custom leap formatter
+   auto fmt = std::make_shared<CLI::LeapFormatter>();
+   app.formatter(fmt);
+
+   // enable help-all, display help on error
    app.set_help_all_flag("--help-all", "Show all help");
+   app.failure_message(CLI::FailureMessage::help);
    app.require_subcommand();
+
    // Hide obsolete options by putting them into a group with an empty name.
    app.add_option( "-H,--host", obsoleted_option_host_port, localized("The host where ${n} is running", ("n", node_executable_name)) )->group("");
    app.add_option( "-p,--port", obsoleted_option_host_port, localized("The port where ${n} is running", ("n", node_executable_name)) )->group("");
@@ -3904,7 +3925,8 @@ int main( int argc, char** argv ) {
    trxSubcommand->add_option("transaction", trx_to_push, localized("The JSON string or filename defining the transaction to push"))->required();
    trxSubcommand->add_option("--signature", extra_sig_opt_callback, localized("append a signature to the transaction; repeat this option to append multiple signatures"))->type_size(0, 1000);
    add_standard_transaction_options_plus_signing(trxSubcommand);
-   trxSubcommand->add_flag("-o,--read-only", tx_read_only, localized("Specify a transaction is read-only"));
+   trxSubcommand->add_flag("-o,--read-only", tx_read_only, localized("Deprecated, use --dry-run instead"));
+   trxSubcommand->add_flag("--dry-run", tx_dry_run, localized("Specify a transaction is dry-run"));
 
    trxSubcommand->callback([&] {
       fc::variant trx_var = json_from_file_or_string(trx_to_push);

programs/keosd/main.cpp

@@ -81,7 +81,8 @@ int main(int argc, char** argv)
       app().set_default_config_dir(home / "eosio-wallet");
       http_plugin::set_defaults({
          .default_unix_socket_path = keosd::config::key_store_executable_name + ".sock",
-         .default_http_port = 0
+         .default_http_port = 0,
+         .server_header = keosd::config::key_store_executable_name + "/" + app().version_string()
       });
       app().register_plugin<wallet_api_plugin>();
       if(!app().initialize<wallet_plugin, wallet_api_plugin, http_plugin>(argc, argv)) {

programs/leap-util/actions/blocklog.cpp

@@ -47,142 +47,77 @@ struct report_time {
 
 
 void blocklog_actions::setup(CLI::App& app) {
+   // callback helper with error code handling
+   auto err_guard = [this](int (blocklog_actions::*fun)()) {
+      try {
+         initialize();
+         int rc = (this->*fun)();
+         if(rc) throw(CLI::RuntimeError(rc));
+      } catch(...) {
+         print_exception();
+         throw(CLI::RuntimeError(-1));
+      }
+   };
+
+   // main command
    auto* sub = app.add_subcommand("block-log", "Blocklog utility");
    sub->require_subcommand(1);
 
-   // callback with error code handling
-   auto cb = [this]() {
-      int rc = run_subcommand();
-      // properly return err code in main
-      if(rc) throw(CLI::RuntimeError(rc));
-   };
+   // subcommand - print log
+   auto* print_log = sub->add_subcommand("print-log", "Print  blocks.log as JSON")->callback([err_guard]() { err_guard(&blocklog_actions::read_log); });
+   print_log->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
+   print_log->add_option("--output-file,-o", opt->output_file, "The file to write the output to (absolute or relative path).  If not specified then output is to stdout.");
+   print_log->add_option("--first,-f", opt->first_block, "The first block number to log or the first to keep if trim-blocklog.");
+   print_log->add_option("--last,-l", opt->last_block, "The last block number to log or the last to keep if trim-blocklog.");
+   print_log->add_flag("--no-pretty-print", opt->no_pretty_print, "Do not pretty print the output.  Useful if piping to jq to improve performance.");
+   print_log->add_flag("--as-json-array", opt->as_json_array, "Print out json blocks wrapped in json array (otherwise the output is free-standing json objects).");
 
-   // options
-   sub->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
-   sub->add_option("--output-file,-o", opt->output_file, "The file to write the output to (absolute or relative path).  If not specified then output is to stdout.");
-   sub->add_option("--first,-f", opt->first_block, "The first block number to log or the first to keep if trim-blocklog.");
-   sub->add_option("--last,-l", opt->last_block, "The last block number to log or the last to keep if trim-blocklog.");
-   sub->add_option("--output-dir", opt->output_dir, "The output directory for the block log extracted from blocks-dir.");
+   // subcommand - make index
+   auto* make_index = sub->add_subcommand("make-index", "Create blocks.index from blocks.log. Must give 'blocks-dir'. Give 'output-file' relative to current directory or absolute path (default is <blocks-dir>/blocks.index).")->callback([err_guard]() { err_guard(&blocklog_actions::make_index); });
+   make_index->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
+   make_index->add_option("--output-file,-o", opt->output_file, "The file to write the output to (absolute or relative path).  If not specified then output is to stdout.");
 
-   // flags
-   sub->add_flag("--no-pretty-print", opt->no_pretty_print, "Do not pretty print the output.  Useful if piping to jq to improve performance.");
-   sub->add_flag("--as-json-array", opt->as_json_array, "Print out json blocks wrapped in json array (otherwise the output is free-standing json objects).");
+   // subcommand - trim blocklog
+   auto* trim_blocklog = sub->add_subcommand("trim-blocklog", "Trim blocks.log and blocks.index. Must give 'blocks-dir' and 'first' and/or 'last'.")->callback([err_guard]() { err_guard(&blocklog_actions::trim_blocklog); });
+   trim_blocklog->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
+   trim_blocklog->add_option("--first,-f", opt->first_block, "The first block number to keep.")->required();
+   trim_blocklog->add_option("--last,-l", opt->last_block, "The last block number to keep.")->required();
 
-   // subcommands
-   sub->add_subcommand("make-index", "Create blocks.index from blocks.log. Must give 'blocks-dir'. Give 'output-file' relative to current directory or absolute path (default is <blocks-dir>/blocks.index).")->callback([this, cb]() {opt->make_index=true; cb(); });
-   sub->add_subcommand("trim-blocklog", "Trim blocks.log and blocks.index. Must give 'blocks-dir' and 'first' and/or 'last'.")->callback([this, cb]() {opt->trim_blocklog=true; cb(); });
-   sub->add_subcommand("extract-blocks", "Extract range of blocks from blocks.log and write to output-dir.  Must give 'first' and/or 'last'.")->callback([this, cb]() {opt->extract_blocks=true; cb(); });
-   sub->add_subcommand("smoke-test", "Quick test that blocks.log and blocks.index are well formed and agree with each other.")->callback([this, cb]() {opt->smoke_test=true; cb(); });
-   sub->add_subcommand("vacuum", "Vacuum a pruned blocks.log in to an un-pruned blocks.log")->callback([this, cb]() {opt->vacuum=true; cb(); });
-   sub->add_subcommand("genesis", "Extract genesis_state from blocks.log as JSON")->callback([this, cb]() {opt->genesis=true; cb(); });
+   // subcommand - extract blocks
+   auto* extract_blocks = sub->add_subcommand("extract-blocks", "Extract range of blocks from blocks.log and write to output-dir.  Must give 'first' and/or 'last'.")->callback([err_guard]() { err_guard(&blocklog_actions::extract_blocks); });
+   extract_blocks->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
+   extract_blocks->add_option("--first,-f", opt->first_block, "The first block number to keep.")->required();
+   extract_blocks->add_option("--last,-l", opt->last_block, "The last block number to keep.")->required();
+   extract_blocks->add_option("--output-dir", opt->output_dir, "The output directory for the block log extracted from blocks-dir.");
 
-   sub->callback([cb]() { cb(); });
+   // subcommand - smoke test
+   auto* smoke_test = sub->add_subcommand("smoke-test", "Quick test that blocks.log and blocks.index are well formed and agree with each other.")->callback([err_guard]() { err_guard(&blocklog_actions::smoke_test); });
+   smoke_test->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
+
+   // subcommand - vacuum
+   auto* vacuum = sub->add_subcommand("vacuum", "Vacuum a pruned blocks.log in to an un-pruned blocks.log")->callback([err_guard]() { err_guard(&blocklog_actions::do_vacuum); });
+   vacuum->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
+
+   // subcommand - genesis
+   auto* genesis = sub->add_subcommand("genesis", "Extract genesis_state from blocks.log as JSON")->callback([err_guard]() { err_guard(&blocklog_actions::do_genesis); });
+   genesis->add_option("--blocks-dir", opt->blocks_dir, "The location of the blocks directory (absolute path or relative to the current directory).");
+   genesis->add_option("--output-file,-o", opt->output_file, "The file to write the output to (absolute or relative path).  If not specified then output is to stdout.");
 }
 
-int blocklog_actions::run_subcommand() {
-   std::ios::sync_with_stdio(false);// for potential performance boost for large block log files
+void blocklog_actions::print_exception() noexcept {
    try {
-      if(opt->trim_blocklog) {
-         if(opt->first_block == 0 && opt->last_block == std::numeric_limits<uint32_t>::max()) {
-            std::cerr << "trim-blocklog does nothing unless first and/or last block are specified.";
-            return -1;
-         }
-         if(opt->last_block != std::numeric_limits<uint32_t>::max()) {
-            if(trim_blocklog_end(opt->blocks_dir, opt->last_block) != 0)
-               return -1;
-         }
-         if(opt->first_block != 0) {
-            if(!trim_blocklog_front(opt->blocks_dir, opt->first_block))
-               return -1;
-         }
-         return 0;
-      }
-      if(opt->extract_blocks) {
-         if(opt->first_block == 0 && opt->last_block == std::numeric_limits<uint32_t>::max()) {
-            std::cerr << "extract-blocklog does nothing unless first and/or last block are specified.";
-            return -1;
-         }
-         if(!extract_block_range(opt->blocks_dir, opt->output_dir, opt->first_block, opt->last_block))
-            return -1;
-         return 0;
-      }
-      if(opt->vacuum) {
-         initialize();
-         return do_vacuum();
-      }
-      if(opt->genesis) {
-         initialize();
-         return do_genesis();
-      }
-      if(opt->make_index) {
-         const bfs::path blocks_dir = opt->blocks_dir;
-         bfs::path out_file = blocks_dir / "blocks.index";
-         const bfs::path block_file = blocks_dir / "blocks.log";
-
-         if(!opt->output_file.empty())
-            out_file = opt->output_file;
-
-         report_time rt("making index");
-         const auto log_level = fc::logger::get(DEFAULT_LOGGER).get_log_level();
-         fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
-         block_log::construct_index(block_file.generic_string(), out_file.generic_string());
-         fc::logger::get(DEFAULT_LOGGER).set_log_level(log_level);
-         rt.report();
-         return 0;
-      }
-      //else print blocks.log as JSON
-      initialize();
-      read_log();
+      throw;
    } catch(const fc::exception& e) {
       elog("${e}", ("e", e.to_detail_string()));
-      return -1;
    } catch(const boost::exception& e) {
       elog("${e}", ("e", boost::diagnostic_information(e)));
-      return -1;
+   } catch(const CLI::RuntimeError& e) {
+      // avoid reporting it twice, RuntimeError is only for cli11
    } catch(const std::exception& e) {
       elog("${e}", ("e", e.what()));
-      return -1;
    } catch(...) {
       elog("unknown exception");
-      return -1;
    }
-   return 0;
-}
-
-int blocklog_actions::do_genesis() {
-   std::optional<genesis_state> gs;
-   bfs::path bld = opt->blocks_dir;
-   auto full_path = (bld / "blocks.log").generic_string();
-
-   if(fc::exists(bld / "blocks.log")) {
-      gs = block_log::extract_genesis_state(opt->blocks_dir);
-      if (!gs) {
-         std::cerr << "Block log at '" << full_path 
-                   << "' does not contain a genesis state, it only has the chain-id." << std::endl;
-         return -1;
-      }
-   } else {
-      std::cerr << "No blocks.log found at '" << full_path << "'." << std::endl;
-      return -1;
-   }
-
-   // just print if output not set
-   if(opt->output_file.empty()) {
-      std::cout << json::to_pretty_string(*gs) << std::endl;
-   } else {
-      bfs::path p = opt->output_file;
-      if(p.is_relative()) {
-         p = bfs::current_path() / p;
-      }
-
-      if (!fc::json::save_to_file(*gs, p, true)) {
-         std::cerr <<  "Error occurred while writing genesis JSON to '" << p.generic_string() << "'" << std::endl;    
-         return -1;
-      }
-    
-      std::cout << "Saved genesis JSON to '" << p.generic_string() << "'" << std::endl;
-   }
-   return 0;
 }
 
 void blocklog_actions::initialize() {
@@ -211,6 +146,74 @@ void blocklog_actions::initialize() {
    FC_LOG_AND_RETHROW()
 }
 
+int blocklog_actions::make_index() {
+   const bfs::path blocks_dir = opt->blocks_dir;
+   bfs::path out_file = blocks_dir / "blocks.index";
+   const bfs::path block_file = blocks_dir / "blocks.log";
+   if(!opt->output_file.empty()) out_file = opt->output_file;
+
+   report_time rt("making index");
+   const auto log_level = fc::logger::get(DEFAULT_LOGGER).get_log_level();
+   fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
+   block_log::construct_index(block_file.generic_string(), out_file.generic_string());
+   fc::logger::get(DEFAULT_LOGGER).set_log_level(log_level);
+   rt.report();
+
+   return 0;
+}
+
+int blocklog_actions::trim_blocklog() {
+   if(opt->last_block != std::numeric_limits<uint32_t>::max()) {
+      if(trim_blocklog_end(opt->blocks_dir, opt->last_block) != 0) return -1;
+   }
+   if(opt->first_block != 0) {
+      if(!trim_blocklog_front(opt->blocks_dir, opt->first_block)) return -1;
+   }
+   return 0;
+}
+
+int blocklog_actions::extract_blocks() {
+   if(!extract_block_range(opt->blocks_dir, opt->output_dir, opt->first_block, opt->last_block))
+      return -1;
+   return 0;
+}
+
+int blocklog_actions::do_genesis() {
+   std::optional<genesis_state> gs;
+   bfs::path bld = opt->blocks_dir;
+   auto full_path = (bld / "blocks.log").generic_string();
+
+   if(fc::exists(bld / "blocks.log")) {
+      gs = block_log::extract_genesis_state(opt->blocks_dir);
+      if(!gs) {
+         std::cerr << "Block log at '" << full_path
+                   << "' does not contain a genesis state, it only has the chain-id." << std::endl;
+         return -1;
+      }
+   } else {
+      std::cerr << "No blocks.log found at '" << full_path << "'." << std::endl;
+      return -1;
+   }
+
+   // just print if output not set
+   if(opt->output_file.empty()) {
+      std::cout << json::to_pretty_string(*gs) << std::endl;
+   } else {
+      bfs::path p = opt->output_file;
+      if(p.is_relative()) {
+         p = bfs::current_path() / p;
+      }
+
+      if(!fc::json::save_to_file(*gs, p, true)) {
+         std::cerr << "Error occurred while writing genesis JSON to '" << p.generic_string() << "'" << std::endl;
+         return -1;
+      }
+
+      std::cout << "Saved genesis JSON to '" << p.generic_string() << "'" << std::endl;
+   }
+   return 0;
+}
+
 int blocklog_actions::trim_blocklog_end(bfs::path block_dir, uint32_t n) {//n is last block to keep (remove later blocks)
    report_time rt("trimming blocklog end");
    using namespace std;
@@ -250,9 +253,9 @@ bool blocklog_actions::extract_block_range(bfs::path block_dir, bfs::path output
    return status;
 }
 
-
-void blocklog_actions::smoke_test(bfs::path block_dir) {
+int blocklog_actions::smoke_test() {
    using namespace std;
+   bfs::path block_dir = opt->blocks_dir;
    cout << "\nSmoke test of blocks.log and blocks.index in directory " << block_dir << '\n';
    trim_data td(block_dir);
    auto status = fseek(td.blk_in, -sizeof(uint64_t), SEEK_END);//get last_block from blocks.log, compare to from blocks.index
@@ -279,10 +282,19 @@ void blocklog_actions::smoke_test(bfs::path block_dir) {
          break;
    }
    cout << "\nno problems found\n";//if get here there were no exceptions
+   return 0;
 }
 
 int blocklog_actions::do_vacuum() {
-   if (!opt->blog_keep_prune_conf) {
+   bfs::path bld = opt->blocks_dir;
+   auto full_path = (bld / "blocks.log").generic_string();
+
+   if(!fc::exists(bld / "blocks.log")) {
+      std::cerr << "No blocks.log found at '" << full_path << "'." << std::endl;
+      return -1;
+   }
+
+   if(!opt->blog_keep_prune_conf) {
       std::cerr << "blocks.log is not a pruned log; nothing to vacuum" << std::endl;
       return -1;
    }
@@ -291,7 +303,8 @@ int blocklog_actions::do_vacuum() {
    return 0;
 }
 
-void blocklog_actions::read_log() {
+int blocklog_actions::read_log() {
+   initialize();
    report_time rt("reading log");
    block_log block_logger(opt->blocks_dir, opt->blog_keep_prune_conf);
    const auto end = block_logger.read_head();
@@ -385,4 +398,6 @@ void blocklog_actions::read_log() {
    if(opt->as_json_array)
       *out << "]";
    rt.report();
+
+   return 0;
 }

programs/leap-util/actions/blocklog.hpp

@@ -12,20 +12,12 @@ struct blocklog_options {
    std::string output_file = "";
    uint32_t first_block = 0;
    uint32_t last_block = std::numeric_limits<uint32_t>::max();
-   std::string output_dir;
+   std::string output_dir = "";
 
    // flags
    bool no_pretty_print = false;
    bool as_json_array = false;
 
-   // subcommands
-   bool make_index = false;
-   bool trim_blocklog = false;
-   bool extract_blocks = false;
-   bool smoke_test = false;
-   bool vacuum = false;
-   bool genesis = false;
-
    std::optional<block_log_prune_config> blog_keep_prune_conf;
 };
 
@@ -34,16 +26,19 @@ public:
    blocklog_actions() : sub_command() {}
    void setup(CLI::App& app);
 
-   // callbacks
-   int run_subcommand();
-
 protected:
+   void print_exception() noexcept;
+
    void initialize();
    int trim_blocklog_end(bfs::path block_dir, uint32_t n);
    bool trim_blocklog_front(bfs::path block_dir, uint32_t n);
    bool extract_block_range(bfs::path block_dir, bfs::path output_dir, uint32_t start, uint32_t end);
-   void smoke_test(bfs::path block_dir);
-   int  do_vacuum();
-   int  do_genesis();
-   void read_log();
+
+   int make_index();
+   int trim_blocklog();
+   int extract_blocks();
+   int smoke_test();
+   int do_vacuum();
+   int do_genesis();
+   int read_log();
 };

programs/leap-util/actions/chain.cpp

@@ -37,8 +37,8 @@ void chain_actions::setup(CLI::App& app) {
    sub->require_subcommand(1);
 
    auto* build = sub->add_subcommand("build-info", "extract build environment information as JSON");
-   auto opt1 = build->add_option("--output-file,-o", opt->build_output_file, "write into specified file")->capture_default_str();
-   auto opt2 = build->add_flag("--print,-p", opt->build_just_print, "print to console");
+   build->add_option("--output-file,-o", opt->build_output_file, "write into specified file")->capture_default_str();
+   build->add_flag("--print,-p", opt->build_just_print, "print to console");
    build->require_option(1);
 
    build->callback([&]() {

programs/leap-util/leap_formatter.hpp

@@ -1,12 +0,0 @@
-
-#include <cli11/CLI11.hpp>
-#include <iostream>
-#include <memory>
-
-class leap_formatter : public CLI::Formatter {
-  public:
-    leap_formatter() : Formatter() {
-        // 30 is default for CLI11, but seems like overkill
-        column_width(25);
-    }
-};

programs/leap-util/main.cpp

@@ -12,18 +12,17 @@
 
 #include <memory>
 
-#include "leap_formatter.hpp"
-
 int main(int argc, char** argv) {
    fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
 
    CLI::App app{"Leap Command Line Utility"};
 
    // custom leap formatter
-   auto fmt = std::make_shared<leap_formatter>();
+   auto fmt = std::make_shared<CLI::LeapFormatter>();
    app.formatter(fmt);
 
    app.set_help_all_flag("--help-all", "Show all help");
+   app.failure_message(CLI::FailureMessage::help);
    app.require_subcommand(1, 2);
 
    // generics sc tree

programs/nodeos/main.cpp

@@ -120,7 +120,8 @@ int main(int argc, char** argv)
       app().set_default_config_dir(root / "eosio" / nodeos::config::node_executable_name / "config" );
       http_plugin::set_defaults({
          .default_unix_socket_path = "",
-         .default_http_port = 8888
+         .default_http_port = 8888,
+         .server_header = nodeos::config::node_executable_name + "/" + app().version_string()
       });
       if(!app().initialize<chain_plugin, net_plugin, producer_plugin, resource_monitor_plugin>(argc, argv)) {
          const auto& opts = app().get_options();

tests/TestHarness/Cluster.py

@@ -783,7 +783,7 @@ class Cluster(object):
             #Utils.Print("nextEosIdx: %d, count: %d" % (nextEosIdx, count))
             node=self.nodes[nextEosIdx]
             if Utils.Debug: Utils.Print("Wait for transaction id %s on node port %d" % (transId, node.port))
-            if node.waitForTransInBlock(transId) is False:
+            if node.waitForTransactionInBlock(transId) is False:
                 Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port))
                 return False
 
@@ -804,7 +804,7 @@ class Cluster(object):
         # As an extra step wait for last transaction on the root node
         node=self.nodes[0]
         if Utils.Debug: Utils.Print("Wait for transaction id %s on node port %d" % (transId, node.port))
-        if node.waitForTransInBlock(transId) is False:
+        if node.waitForTransactionInBlock(transId) is False:
             Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port))
             return False
 
@@ -1062,7 +1062,7 @@ class Cluster(object):
 
         Utils.Print("Wait for last transfer transaction to become finalized.")
         transId=Node.getTransId(trans[1])
-        if not biosNode.waitForTransInBlock(transId):
+        if not biosNode.waitForTransactionInBlock(transId):
             Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
             return None
 
@@ -1146,7 +1146,7 @@ class Cluster(object):
             accounts.append(initx)
 
         transId=Node.getTransId(trans)
-        if not biosNode.waitForTransInBlock(transId):
+        if not biosNode.waitForTransactionInBlock(transId):
             Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
             return None
 
@@ -1195,7 +1195,7 @@ class Cluster(object):
 
             trans=trans[1]
             transId=Node.getTransId(trans)
-            if not biosNode.waitForTransInBlock(transId):
+            if not biosNode.waitForTransactionInBlock(transId):
                 Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
                 return None
 
@@ -1238,7 +1238,7 @@ class Cluster(object):
 
         Node.validateTransaction(trans)
         transId=Node.getTransId(trans)
-        if not biosNode.waitForTransInBlock(transId):
+        if not biosNode.waitForTransactionInBlock(transId):
             Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
             return None
 
@@ -1265,7 +1265,7 @@ class Cluster(object):
 
         Node.validateTransaction(trans[1])
         transId=Node.getTransId(trans[1])
-        if not biosNode.waitForTransInBlock(transId):
+        if not biosNode.waitForTransactionInBlock(transId):
             Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
             return None
 
@@ -1282,7 +1282,7 @@ class Cluster(object):
         Node.validateTransaction(trans[1])
         Utils.Print("Wait for issue action transaction to become finalized.")
         transId=Node.getTransId(trans[1])
-        # biosNode.waitForTransInBlock(transId)
+        # biosNode.waitForTransactionInBlock(transId)
         # guesstimating block finalization timeout. Two production rounds of 12 blocks per node, plus 60 seconds buffer
         timeout = .5 * 12 * 2 * len(producerKeys) + 60
         if not biosNode.waitForTransFinalization(transId, timeout=timeout):
@@ -1327,7 +1327,7 @@ class Cluster(object):
 
         Utils.Print("Wait for last transfer transaction to become finalized.")
         transId=Node.getTransId(trans[1])
-        if not biosNode.waitForTransInBlock(transId):
+        if not biosNode.waitForTransactionInBlock(transId):
             Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
             return None
 
@@ -1339,7 +1339,7 @@ class Cluster(object):
             trans=biosNode.pushMessage(eosioAccount.name, action, data, opts)
             transId=Node.getTransId(trans[1])
             Utils.Print("Wait for system init transaction to be in a block.")
-            if not biosNode.waitForTransInBlock(transId):
+            if not biosNode.waitForTransactionInBlock(transId):
                 Utils.Print("ERROR: Failed to validate transaction %s in block on server port %d." % (transId, biosNode.port))
                 return None
 
@@ -1566,7 +1566,7 @@ class Cluster(object):
         if waitForTransBlock and transId is not None:
             node=self.nodes[0]
             if Utils.Debug: Utils.Print("Wait for transaction id %s on server port %d." % ( transId, node.port))
-            if node.waitForTransInBlock(transId) is False:
+            if node.waitForTransactionInBlock(transId) is False:
                 Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port))
                 return False
 

tests/TestHarness/Node.py

@@ -384,7 +384,7 @@ class Node(object):
         transId=Node.getTransId(trans)
 
         if stakedDeposit > 0:
-            self.waitForTransInBlock(transId) # seems like account creation needs to be finalized before transfer can happen
+            self.waitForTransactionInBlock(transId) # seems like account creation needs to be finalized before transfer can happen
             trans = self.transferFunds(creatorAccount, account, Node.currencyIntToStr(stakedDeposit, CORE_SYMBOL), "init")
             transId=Node.getTransId(trans)
 
@@ -403,7 +403,7 @@ class Node(object):
         transId=Node.getTransId(trans)
 
         if stakedDeposit > 0:
-            self.waitForTransInBlock(transId) # seems like account creation needs to be finlized before transfer can happen
+            self.waitForTransactionInBlock(transId) # seems like account creation needs to be finlized before transfer can happen
             trans = self.transferFunds(creatorAccount, account, "%0.04f %s" % (stakedDeposit/10000, CORE_SYMBOL), "init")
             self.trackCmdTransaction(trans)
             transId=Node.getTransId(trans)
@@ -420,7 +420,7 @@ class Node(object):
 
     def getTable(self, contract, scope, table, exitOnError=False):
         cmdDesc = "get table"
-        cmd="%s %s %s %s" % (cmdDesc, contract, scope, table)
+        cmd="%s --time-limit 99999 %s %s %s" % (cmdDesc, contract, scope, table)
         msg="contract=%s, scope=%s, table=%s" % (contract, scope, table);
         return self.processCleosCmd(cmd, cmdDesc, exitOnError=exitOnError, exitMsg=msg)
 
@@ -482,13 +482,17 @@ class Node(object):
 
         return None
 
-    def waitForTransInBlock(self, transId, timeout=None):
+    def waitForTransactionInBlock(self, transId, timeout=None):
         """Wait for trans id to be finalized."""
         assert(isinstance(transId, str))
         lam = lambda: self.isTransInAnyBlock(transId)
         ret=Utils.waitForBool(lam, timeout)
         return ret
 
+    def waitForTransactionsInBlock(self, transIds, timeout=None):
+        for transId in transIds:
+            self.waitForTransactionInBlock(transId, timeout)
+
     def waitForTransFinalization(self, transId, timeout=None):
         """Wait for trans id to be finalized."""
         assert(isinstance(transId, str))
@@ -1079,7 +1083,7 @@ class Node(object):
             return trans
 
         transId=Node.getTransId(trans)
-        if not self.waitForTransInBlock(transId):
+        if not self.waitForTransactionInBlock(transId):
             if exitOnError:
                 Utils.cmdError("transaction with id %s never made it to a block" % (transId))
                 Utils.errorExit("Failed to find transaction with id %s in a block before timeout" % (transId))

tests/compute_transaction_test.py

@@ -115,6 +115,8 @@ try:
 
     results = node.pushTransaction(trx, opts='--read-only', permissions=account1.name)
     assert(results[0])
+    results = node.pushTransaction(trx, opts='--dry-run', permissions=account1.name)
+    assert(results[0])
     node.waitForLibToAdvance(30)
 
     postBalances = node.getEosBalances([account1, account2])
@@ -122,6 +124,8 @@ try:
 
     results = node.pushTransaction(trx, opts='--read-only --skip-sign')
     assert(results[0])
+    results = node.pushTransaction(trx, opts='--dry-run --skip-sign')
+    assert(results[0])
     node.waitForLibToAdvance(30)
 
     postBalances = node.getEosBalances([account1, account2])
@@ -141,6 +145,8 @@ try:
 
         results = npnode.pushTransaction(trx2, opts="--read-only")
         assert(not results[0])
+        results = npnode.pushTransaction(trx2, opts="--dry-run")
+        assert(not results[0])
 
 # Verify that no subjective billing was charged
     acct2 = npnode.getAccountSubjectiveInfo("account2")
@@ -174,10 +180,12 @@ try:
     }
     results = npnode.pushTransaction(trx3, opts="--read-only")
     assert(results[0])
+    results = npnode.pushTransaction(trx3, opts="--dry-run")
+    assert(results[0])
 
     testSuccessful = True
 finally:
     TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails)
 
 errorCode = 0 if testSuccessful else 1
-exit(errorCode)
+exit(errorCode)

tests/launcher_test.py

@@ -180,7 +180,7 @@ try:
         cmdError("FAILURE - transfer failed")
         errorExit("Transfer verification failed. Excepted %s, actual: %s" % (expectedAmount, actualAmount))
 
-    node.waitForTransInBlock(transId)
+    node.waitForTransactionInBlock(transId)
 
     transaction=node.getTransaction(transId, exitOnError=True, delayedRetry=False)
 

tests/nodeos_forked_chain_test.py

@@ -230,7 +230,7 @@ try:
         transferAmount="100000000.0000 {0}".format(CORE_SYMBOL)
         Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, account.name))
         node.transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer", waitForTransBlock=True)
-        trans=node.delegatebw(account, 20000000.0000, 20000000.0000, waitForTransBlock=True, exitOnError=True)
+        trans=node.delegatebw(account, 20000000.0000, 20000000.0000, waitForTransBlock=False, exitOnError=True)
 
 
     # ***   vote using accounts   ***
@@ -240,7 +240,7 @@ try:
     index=0
     for account in accounts:
         Print("Vote for producers=%s" % (producers))
-        trans=prodNodes[index % len(prodNodes)].vote(account, producers, waitForTransBlock=True)
+        trans=prodNodes[index % len(prodNodes)].vote(account, producers, waitForTransBlock=False)
         index+=1
 
 
@@ -372,6 +372,9 @@ try:
     # block number to start expecting node killed after
     preKillBlockNum=nonProdNode.getBlockNum()
     preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum)
+    if preKillBlockProducer == "defproducerj" or preKillBlockProducer == "defproducerk":
+        # wait for defproduceri so there is plenty of time to send kill before defproducerk
+        nonProdNode.waitForProducer("defproduceri")
     Print("preKillBlockProducer = {}".format(preKillBlockProducer))
     # kill at last block before defproducerl, since the block it is killed on will get propagated
     killAtProducer="defproducerk"

tests/nodeos_high_transaction_test.py

@@ -171,7 +171,7 @@ try:
                         Print("Transaction not found for trans id: %s. Will wait %d seconds to see if it arrives in a block." %
                               (transId, args.transaction_time_delta))
                     transTimeDelayed = True
-                    node.waitForTransInBlock(transId, timeout = args.transaction_time_delta)
+                    node.waitForTransactionInBlock(transId, timeout = args.transaction_time_delta)
                     continue
 
             lastIrreversibleBlockNum = node.getIrreversibleBlockNum()

tests/nodeos_retry_transaction_test.py

@@ -192,7 +192,7 @@ try:
                         Print("Transaction not found for trans id: %s. Will wait %d seconds to see if it arrives in a block." %
                               (transId, args.transaction_time_delta))
                     transTimeDelayed = True
-                    node.waitForTransInBlock(transId, timeout = args.transaction_time_delta)
+                    node.waitForTransactionInBlock(transId, timeout = args.transaction_time_delta)
                     continue
 
             lastIrreversibleBlockNum = node.getIrreversibleBlockNum()

tests/nodeos_run_test.py

@@ -277,7 +277,7 @@ try:
         cmdError("FAILURE - transfer failed")
         errorExit("Transfer verification failed. Excepted %s, actual: %s" % (expectedAmount, actualAmount))
 
-    node.waitForTransInBlock(transId)
+    node.waitForTransactionInBlock(transId)
 
     transaction=node.getTransaction(transId, exitOnError=True, delayedRetry=False)
 
@@ -417,7 +417,7 @@ try:
         errorExit("Failed to reject duplicate message for currency1111 contract")
 
     Print("verify transaction exists")
-    if not node.waitForTransInBlock(transId):
+    if not node.waitForTransactionInBlock(transId):
         cmdError("%s get transaction trans_id" % (ClientName))
         errorExit("Failed to verify push message transaction id.")
 

tests/p2p_stress.py

@@ -59,7 +59,7 @@ class StressNetwork:
         if trid is None:
             return ([], "", 0.0, "failed to issue currency0000")
         print("transaction id %s" % (trid))
-        node.waitForTransInBlock(trid)
+        node.waitForTransactionInBlock(trid)
 
         self.trList = []
         expBal = 0
@@ -97,7 +97,7 @@ class StressNetwork:
         for tr in self.trList:
             trid = node.getTransId(tr)
             transIdlist.append(trid)
-            node.waitForTransInBlock(trid)
+            node.waitForTransactionInBlock(trid)
         return (transIdlist, acc2.name, expBal, "")
     
     def on_exit(self):

tests/trx_finality_status_forked_test.py

@@ -86,10 +86,10 @@ try:
     nonProdNode=None
     prodNodes=[]
     producers=[]
-    for node in cluster.getNodes():
+    for i, node in enumerate(cluster.getNodes()):
         node.producers=Cluster.parseProducers(node.nodeId)
         numProducers=len(node.producers)
-        Print(f"node has producers={node.producers}")
+        Print(f"node {i} has producers={node.producers}")
         if numProducers==0:
             if nonProdNode is None:
                 nonProdNode=node
@@ -99,9 +99,8 @@ try:
             prodNodes.append(node)
             producers.extend(node.producers)
 
-
-    node=prodNodes[0]
-    node1=prodNodes[1]
+    prodAB=prodNodes[0]  # defproducera, defproducerb
+    prodC=prodNodes[1]   # defproducerc
 
     # ***   Identify a block where production is stable   ***
 
@@ -115,24 +114,15 @@ try:
     account1.activePublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
     cluster.createAccountAndVerify(account1, cluster.eosioAccount, stakedDeposit=1000)
 
-    Print("Creating account2")
-    account2 = Account('account2')
-    account2.ownerPublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
-    account2.activePublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
-    cluster.createAccountAndVerify(account2, cluster.eosioAccount, stakedDeposit=1000)
-
     Print("Validating accounts after bootstrap")
-    cluster.validateAccounts([account1, account2])
+    cluster.validateAccounts([account1])
 
     # ***   Killing the "bridge" node   ***
     Print("Sending command to kill \"bridge\" node to separate the 2 producer groups.")
-    # block number to start expecting node killed after
-    preKillBlockNum=nonProdNode.getBlockNum()
-    preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum)
-    # kill at the end of the production window for defproducera, so there is still time for the fork for
+    # kill at the beginning of the production window for defproducera, so there is time for the fork for
     # defproducerc to grow before it would overtake the fork for defproducera and defproducerb
-    killAtProducer="defproducerc"
-    nonProdNode.killNodeOnProducer(producer=killAtProducer, whereInSequence=11)
+    killAtProducer="defproducera"
+    nonProdNode.killNodeOnProducer(producer=killAtProducer, whereInSequence=1)
 
     #verify that the non producing node is not alive (and populate the producer nodes with current getInfo data to report if
     #an error occurs)
@@ -143,7 +133,7 @@ try:
     while nonProdNode.verifyAlive() and count > 0:
         # wait on prodNode 0 since it will continue to advance, since defproducera and defproducerb are its producers
         Print("Wait for next block")
-        assert prodNodes[0].waitForNextBlock(timeout=6), "Production node 0 should continue to advance, even after bridge node is killed"
+        assert prodAB.waitForNextBlock(timeout=6), "Production node AB should continue to advance, even after bridge node is killed"
         count -= 1
 
     assert not nonProdNode.verifyAlive(), "Bridge node should have been killed if test was functioning correctly."
@@ -155,9 +145,9 @@ try:
         return status["state"]
 
     transferAmount = 10
-    prodNodes[1].transferFunds(cluster.eosioAccount, account1, f"{transferAmount}.0000 {CORE_SYMBOL}", "fund account")
-    transId = prodNodes[1].getLastTrackedTransactionId()
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    prodC.transferFunds(cluster.eosioAccount, account1, f"{transferAmount}.0000 {CORE_SYMBOL}", "fund account")
+    transId = prodC.getLastTrackedTransactionId()
+    retStatus = prodC.getTransactionStatus(transId)
     state = getState(retStatus)
 
     localState = "LOCALLY_APPLIED"
@@ -169,21 +159,21 @@ try:
     assert state == localState, \
         f"ERROR: getTransactionStatus didn't return \"{localState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}"
 
-    assert prodNodes[1].waitForNextBlock(), "Production node 1 should continue to advance, even after bridge node is killed"
+    assert prodC.waitForNextBlock(), "Production node C should continue to advance, even after bridge node is killed"
 
     # since the Bridge node is killed when this producer is producing its last block in its window, there is plenty of time for the transfer to be
     # sent before the first block is created, but adding this to ensure it is in one of these blocks
     numTries = 2
-    preInfo = prodNodes[1].getInfo()
+    preInfo = prodC.getInfo()
     while numTries > 0:
-        retStatus = prodNodes[1].getTransactionStatus(transId)
+        retStatus = prodC.getTransactionStatus(transId)
         state = getState(retStatus)
         if state == inBlockState:
             break
         numTries -= 1
-        assert prodNodes[1].waitForNextBlock(), "Production node 1 should continue to advance, even after bridge node is killed"
+        assert prodC.waitForNextBlock(), "Production node C should continue to advance, even after bridge node is killed"
 
-    postInfo = prodNodes[1].getInfo()
+    postInfo = prodC.getInfo()
 
     Print(f"getTransactionStatus returned status: {json.dumps(retStatus, indent=1)}")
     assert state == inBlockState, \
@@ -195,42 +185,31 @@ try:
     if not nonProdNode.relaunch():
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
-    Print("Wait for LIB to move, which indicates prodNode[1] has forked out the branch")
-    assert prodNodes[1].waitForLibToAdvance(), \
-        "ERROR: Network did not reach concensus after bridge node was restarted."
+    Print("Wait for LIB to move, which indicates prodC has forked out the branch")
+    assert prodC.waitForLibToAdvance(), \
+        "ERROR: Network did not reach consensus after bridge node was restarted."
 
-    Print("Wait till prodNodes[1] is reporting at least the same head block number as the forked out block")
-    assert prodNodes[1].waitForBlock(originalInBlockState["block_number"]), \
-        f"Production node 1 should continue to advance after LIB starts advancing. \n\originalInBlockState: {json.dumps(originalInBlockState, indent=1)}"
-
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    retStatus = prodC.getTransactionStatus(transId)
     state = getState(retStatus)
 
     assert state == forkedOutState, \
         f"ERROR: getTransactionStatus didn't return \"{forkedOutState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
+        f"\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}"
 
     for prodNode in prodNodes:
         info=prodNode.getInfo()
         Print(f"node info: {json.dumps(info, indent=1)}")
 
-    retStatus = prodNodes[1].getTransactionStatus(transId)
-    state = getState(retStatus)
+    assert prodC.waitForProducer("defproducerc"), \
+        f"Waiting for prodC to produce, but it never happened" + \
+        f"\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}"
 
-    assert state == forkedOutState, \
-        f"ERROR: getTransactionStatus didn't return \"{forkedOutState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
-
-    assert prodNodes[1].waitForProducer("defproducerc"), \
-        f"Waiting for prodNode 1 to produce, but it never happened" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
-
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    retStatus = prodC.getTransactionStatus(transId)
     state = getState(retStatus)
 
     assert state == inBlockState, \
         f"ERROR: getTransactionStatus didn't return \"{inBlockState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
+        f"\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}"
 
     afterForkInBlockState = retStatus
     afterForkBlockId = retStatus["block_id"]
@@ -238,22 +217,22 @@ try:
         "ERROR: The way the test is designed, the transaction should be added to a block that has a higher number than it was in originally before it was forked out." + \
        f"\n\noriginal in block state: {json.dumps(originalInBlockState, indent=1)}\n\nafter fork in block state: {json.dumps(afterForkInBlockState, indent=1)}"
 
-    assert prodNodes[1].waitForBlock(afterForkInBlockState["block_number"], timeout=120, blockType=BlockType.lib), \
-        f"ERROR: Block never finalized.\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}" + \
+    assert prodC.waitForBlock(afterForkInBlockState["block_number"], timeout=120, blockType=BlockType.lib), \
+        f"ERROR: Block never finalized.\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}" + \
         f"\n\nafter fork in block state: {json.dumps(afterForkInBlockState, indent=1)}"
 
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    retStatus = prodC.getTransactionStatus(transId)
     if afterForkBlockId != retStatus["block_id"]: # might have been forked out, if so wait for new block to become LIB
-        assert prodNodes[1].waitForBlock(retStatus["block_number"], timeout=120, blockType=BlockType.lib), \
-            f"ERROR: Block never finalized.\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}" + \
+        assert prodC.waitForBlock(retStatus["block_number"], timeout=120, blockType=BlockType.lib), \
+            f"ERROR: Block never finalized.\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}" + \
             f"\n\nafter fork in block state: {json.dumps(afterForkInBlockState, indent=1)}"
 
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    retStatus = prodC.getTransactionStatus(transId)
     state = getState(retStatus)
 
     assert state == irreversibleState, \
         f"ERROR: getTransactionStatus didn't return \"{irreversibleState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
+        f"\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}"
 
     testSuccessful=True
 finally:

tests/trx_finality_status_test.py

@@ -142,7 +142,7 @@ try:
     state = getState(retStatus)
 
     assert (state == localState or state == inBlockState), \
-        f"ERROR: getTransactionStatus didn't return \"{localState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}"
+        f"ERROR: getTransactionStatus didn't return \"{localState}\" or \"{inBlockState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}"
     status.append(copy.copy(retStatus))
     startingBlockNum=testNode.getInfo()["head_block_num"]
 
@@ -150,12 +150,13 @@ try:
         bnPresent = "block_number" in status
         biPresent = "block_id" in status
         btPresent = "block_timestamp" in status
-        desc = "" if present else "not "
+        desc = "" if present else " not"
         group = bnPresent and biPresent and btPresent if present else not bnPresent and not biPresent and not btPresent
         assert group, \
             f"ERROR: getTransactionStatus should{desc} contain \"block_number\", \"block_id\", or \"block_timestamp\" since state was \"{getState(status)}\".\nstatus: {json.dumps(status, indent=1)}"
 
-    validateTrxState(status[0], present=False)
+    present = True if state == inBlockState else False
+    validateTrxState(status[0], present)
 
     def validate(status, knownTrx=True):
         assert "head_number" in status and "head_id" in status and "head_timestamp" in status, \

unittests/api_tests.cpp

@@ -1152,8 +1152,8 @@ BOOST_AUTO_TEST_CASE(checktime_pause_block_deadline_not_extended_test) { try {
    dlog("elapsed ${e}us", ("e", dur) );
    BOOST_CHECK( dur >= 75'000 ); // should never fail
    // If this check fails but duration is >= 75'000 (previous check did not fail), then the check here is likely
-   // because it took longer than 2 ms for checktime to trigger, trace to be created, and to get to the now() call.
-   BOOST_CHECK_MESSAGE( dur < 77'000, "elapsed " << dur << "us" );
+   // because it took longer than 10 ms for checktime to trigger, trace to be created, and to get to the now() call.
+   BOOST_CHECK_MESSAGE( dur < 85'000, "elapsed " << dur << "us" );
 
    BOOST_REQUIRE_EQUAL( t.validate(), true );
 } FC_LOG_AND_RETHROW() }

unittests/block_tests.cpp

@@ -44,10 +44,10 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test)
 
    // Push block with invalid transaction to other chain
    tester validator;
-   auto bs = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
+   auto bsf = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
    validator.control->abort_block();
    controller::block_report br;
-   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
+   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
    [] (const fc::exception &e)->bool {
       return e.code() == account_name_exists_exception::code_value ;
    }) ;
@@ -83,10 +83,10 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_mroot_test)
 
    // Push block with invalid transaction to other chain
    tester validator;
-   auto bs = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
+   auto bsf = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
    validator.control->abort_block();
    controller::block_report br;
-   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
+   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( br, bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
                            [] (const fc::exception &e)->bool {
                               return e.code() == block_validate_exception::code_value &&
                                      e.to_detail_string().find("invalid block transaction merkle root") != std::string::npos;

unittests/forked_tests.cpp

@@ -267,10 +267,10 @@ BOOST_AUTO_TEST_CASE( forking ) try {
    signed_block bad_block = std::move(*b);
    bad_block.action_mroot = bad_block.previous;
    auto bad_id = bad_block.calculate_id();
-   auto bad_block_bs = c.control->create_block_state_future( bad_id, std::make_shared<signed_block>(std::move(bad_block)) );
+   auto bad_block_bsf = c.control->create_block_state_future( bad_id, std::make_shared<signed_block>(std::move(bad_block)) );
    c.control->abort_block();
    controller::block_report br;
-   BOOST_REQUIRE_EXCEPTION(c.control->push_block( br, bad_block_bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception,
+   BOOST_REQUIRE_EXCEPTION(c.control->push_block( br, bad_block_bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception,
       [] (const fc::exception &ex)->bool {
          return ex.to_detail_string().find("block signed by unexpected key") != std::string::npos;
       });

unittests/snapshot_tests.cpp

@@ -159,7 +159,7 @@ namespace {
    }
 
    template <typename SNAPSHOT_SUITE>
-   void verify_integrity_hash(const controller& lhs, const controller& rhs) {
+   void verify_integrity_hash(controller& lhs, controller& rhs) {
       const auto lhs_integrity_hash = lhs.calculate_integrity_hash();
       const auto rhs_integrity_hash = rhs.calculate_integrity_hash();
       if (std::is_same_v<SNAPSHOT_SUITE, variant_snapshot_suite> && lhs_integrity_hash.str() != rhs_integrity_hash.str()) {

unittests/unapplied_transaction_queue_tests.cpp

@@ -339,7 +339,7 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try {
    q.add_aborted( { trx20, trx22 } );
    q.add_persisted( trx21 );
    q.add_persisted( trx23 );
-   q.clear_expired( fc::time_point::now(), fc::time_point::now() + fc::seconds( 300 ), [](auto, auto){} );
+   q.clear_expired( fc::time_point::now(), [](){ return false; }, [](auto, auto){} );
    BOOST_CHECK( q.size() == 2 );
    BOOST_REQUIRE( next( q ) == trx23 );
    BOOST_REQUIRE( next( q ) == trx22 );