Bump Leap to release/3.1.5

[3.1] Pinned Builds manual dispatch workflow in CI

.github/workflows/pinned_build.yaml

@@ -0,0 +1,58 @@
+name: "Pinned Build"
+
+on:
+  workflow_dispatch:
+
+permissions:
+  packages: read
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  Build:
+    name: Build
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu18, ubuntu20, ubuntu22]
+    runs-on: ["self-hosted", "enf-x86-beefy-long"]
+    container: ${{ matrix.platform == 'ubuntu18' && 'ubuntu:bionic' || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }}
+    steps:
+      - name: Conditionally update git repo
+        if: ${{ matrix.platform == 'ubuntu18' }}
+        run: |
+          apt-get update
+          apt-get install -y software-properties-common
+          apt-get update
+          add-apt-repository ppa:git-core/ppa
+      - name: Update and Install git
+        run: |
+          apt-get update
+          apt-get install -y git
+          git --version
+      - name: Clone leap
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Install dependencies
+        run: |
+          # https://github.com/actions/runner/issues/2033
+          chown -R $(id -u):$(id -g) $PWD
+          ./scripts/install_deps.sh
+      - name: Build Pinned Build
+        env:
+          LEAP_PINNED_INSTALL_PREFIX: /usr
+        run: |
+          ./scripts/pinned_build.sh deps build "$(nproc)"
+      - name: Upload package
+        uses: actions/upload-artifact@v3
+        with:
+          name: leap-${{matrix.platform}}-pinned-amd64
+          path: build/leap-3*.deb
+      - name: Run Parallel Tests
+        run: |
+          cd build
+          ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)" --timeout 420

.gitignore

@@ -78,6 +78,8 @@ Testing/*
 build.tar.gz
 build/*
 build-debug/*
+deps/*
+dependencies/*
 
 etc/eosio/node_*
 var/lib/node_*

CMakeLists.txt

@@ -15,7 +15,7 @@ set( CXX_STANDARD_REQUIRED ON)
 
 set(VERSION_MAJOR 3)
 set(VERSION_MINOR 1)
-set(VERSION_PATCH 1)
+set(VERSION_PATCH 5)
 #set(VERSION_SUFFIX rc4)
 
 if(VERSION_SUFFIX)

README.md

@@ -1,126 +1,257 @@
 # Leap
+Leap is a C++ implementation of the [Antelope](https://github.com/AntelopeIO) protocol. It contains blockchain node software and supporting tools for developers and node operators.
 
-Leap is blockchain node software and supporting tools that implements the [Antelope](https://github.com/AntelopeIO) protocol.
+## Branches
+The `main` branch is the development branch; do not use it for production. Refer to the [release page](https://github.com/AntelopeIO/leap/releases) for current information on releases, pre-releases, and obsolete releases, as well as the corresponding tags for those releases.
 
-## Repo Organization
+## Supported Operating Systems
+We currently support the following operating systems.
+- Ubuntu 22.04 Jammy
+- Ubuntu 20.04 Focal
+- Ubuntu 18.04 Bionic
 
-`main` branch is the development branch: do not use this for production. Refer to the [release page](https://github.com/AntelopeIO/leap/releases) for current information on releases, pre-releases, and obsolete releases as well as the corresponding tags for those releases.
+Other Unix derivatives such as macOS are tended to on a best-effort basis and may not be full featured. If you aren't using Ubuntu, please visit the "[Build Unsupported OS](./docs/00_install/01_build-from-source/00_build-unsupported-os.md)" page to explore your options.
 
-## Software Installation
-
-Visit the [release page](https://github.com/AntelopeIO/leap/releases) for Ubuntu binaries. This is the fastest way to get started with the software.
-
-### Building From Source
-
-Recent Ubuntu LTS releases are the only Linux distributions that we fully support. Other Linux distros and other POSIX operating systems (such as macOS) are tended to on a best-effort basis and may not be full featured. Notable requirements to build are:
-* C++17 compiler and standard library
-* boost 1.67+
-* CMake 3.8+
-* (for Linux only) LLVM 7 - 11 (newer versions do not work)
-
-A few other common libraries are tools also required such as openssl 1.1+, libcurl, curl, libusb, GMP, Python 3, and zlib.
-
-**A Warning On Parallel Compilation Jobs (`-j` flag)**: When building C/C++ software often the build is performed in parallel via a command such as `make -j $(nproc)` which uses the number of CPU cores as the number of compilation jobs to perform simultaneously. However, be aware that some compilation units (.cpp files) in Leap are extremely complex and will consume nearly 4GB of memory to compile. You may need to reduce the level of parallelization depending on the amount of memory on your build host. e.g. instead of `make -j $(nproc)` run `make -j2`. Failures due to memory exhaustion will typically but not always manifest as compiler crashes.
-
-Generally we recommend performing what we refer to as a "pinned build" which ensures the compiler and boost version remain the same between builds of different Leap versions (Leap requires these versions remain the same otherwise its state needs to be repopulated from a portable snapshot).
-
-#### Building Pinned Build Binary Packages
-In the directory `<leap src>/scripts` you will find the two scripts `install_deps.sh` and `pinned_build.sh`. If you haven't installed build dependencies then run `install_deps.sh`. Then run `pinned_build.sh <dependencies directory> <leap build directory> <number of jobs>`.
-
-The dependencies directory is where the script will pull the C++ dependencies that need to be built with the pinned compiler for building the pinned binaries for binary packaging.
-
-The binary package will be produced in the Leap build directory that was supplied.
-
-#### Manual (non "pinned") Build Instructions
-
-<details>
-  <summary>Ubuntu 20.04 & 22.04 Build Instructions</summary>
-
-Install required dependencies: 
+If you are running an unsupported Ubuntu derivative, such as Linux Mint, you can find the version of Ubuntu your distribution was based on by using this command:
+```bash
+cat /etc/upstream-release/lsb-release
 ```
-apt-get update && apt-get install   \
-        build-essential             \
-        cmake                       \
-        curl                        \
-        git                         \
-        libboost-all-dev            \
-        libcurl4-openssl-dev        \
-        libgmp-dev                  \
-        libssl-dev                  \
-        libusb-1.0-0-dev            \
-        llvm-11-dev                 \
+Your best bet is to follow the instructions for your Ubuntu base, but we make no guarantees.
+
+## Binary Installation
+This is the fastest way to get started. From the [latest release](https://github.com/AntelopeIO/leap/releases/latest) page, download a binary for one of our [supported operating systems](#supported-operating-systems), or visit the [release tags](https://github.com/AntelopeIO/leap/releases) page to download a binary for a specific version of Leap.
+
+Once you have a `*.deb` file downloaded for your version of Ubuntu, you can install it as follows:
+```bash
+sudo apt-get update
+sudo apt-get install -y ~/Downloads/leap*.deb
+```
+Your download path may vary. If you are in an Ubuntu docker container, omit `sudo` because you run as `root` by default.
+
+Finally, verify Leap was installed correctly:
+```bash
+nodeos --full-version
+```
+You should see a [semantic version](https://semver.org) string followed by a `git` commit hash with no errors. For example:
+```
+v3.1.2-0b64f879e3ebe2e4df09d2e62f1fc164cc1125d1
+```
+
+## Build and Install from Source
+You can also build and install Leap from source.
+
+### Prerequisites
+You will need to build on a [supported operating system](#supported-operating-systems).
+
+Requirements to build:
+- C++17 compiler and standard library
+- boost 1.67+
+- CMake 3.8+
+- LLVM 7 - 11 - for Linux only
+  - newer versions do not work
+- openssl 1.1+
+- libcurl
+- curl
+- libusb
+- git
+- GMP
+- Python 3
+- zlib
+
+### Step 1 - Clone
+If you don't have the Leap repo cloned to your computer yet, [open a terminal](https://itsfoss.com/open-terminal-ubuntu) and navigate to the folder where you want to clone the Leap repository:
+```bash
+cd ~/Downloads
+```
+Clone Leap using either HTTPS...
+```bash
+git clone --recursive https://github.com/AntelopeIO/leap.git
+```
+...or SSH:
+```bash
+git clone --recursive git@github.com:AntelopeIO/leap.git
+```
+
+> ℹ️ **HTTPS vs. SSH Clone** ℹ️  
+Both an HTTPS or SSH git clone will yield the same result - a folder named `leap` containing our source code. It doesn't matter which type you use.
+
+Navigate into that folder:
+```bash
+cd leap
+```
+
+### Step 2 - Checkout Release Tag or Branch
+Choose which [release](https://github.com/AntelopeIO/leap/releases) or [branch](#branches) you would like to build, then check it out. If you are not sure, use the [latest release](https://github.com/AntelopeIO/leap/releases/latest). For example, if you want to build release 3.1.2 then you would check it out using its tag, `v3.1.2`. In the example below, replace `v0.0.0` with your selected release tag accordingly:
+```bash
+git fetch --all --tags
+git checkout v0.0.0
+```
+
+Once you are on the branch or release tag you want to build, make sure everything is up-to-date:
+```bash
+git pull
+git submodule update --init --recursive
+```
+
+### Step 3 - Build
+Select build instructions below for a [pinned build](#pinned-build) (preferred) or an [unpinned build](#unpinned-build).
+
+> ℹ️ **Pinned vs. Unpinned Build** ℹ️  
+We have two types of builds for Leap: "pinned" and "unpinned." The only difference is that pinned builds use specific versions for some dependencies hand-picked by the Leap engineers - they are "pinned" to those versions. In contrast, unpinned builds use the default dependency versions available on the build system at the time. We recommend performing a "pinned" build to ensure the compiler and boost versions remain the same between builds of different Leap versions. Leap requires these versions to remain the same, otherwise its state might need to be recovered from a portable snapshot or the chain needs to be replayed.
+
+> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️  
+When building C/C++ software, often the build is performed in parallel via a command such as `make -j "$(nproc)"` which uses all available CPU threads. However, be aware that some compilation units (`*.cpp` files) in Leap will consume nearly 4GB of memory. Failures due to memory exhaustion will typically, but not always, manifest as compiler crashes. Using all available CPU threads may also prevent you from doing other things on your computer during compilation. For these reasons, consider reducing this value.
+
+> 🐋 **Docker and `sudo`** 🐋  
+If you are in an Ubuntu docker container, omit `sudo` from all commands because you run as `root` by default. Most other docker containers also exclude `sudo`, especially Debian-family containers. If your shell prompt is a hash tag (`#`), omit `sudo`.
+
+#### Pinned Build
+Make sure you are in the root of the `leap` repo, then run the `install_depts.sh` script to install dependencies:
+```bash
+sudo scripts/install_deps.sh
+```
+
+Next, run the pinned build script. You have to give it three arguments, in the following order:
+  - A temporary folder, for all dependencies that need to be built from source.
+  - A build folder, where the binaries you need to install will be built to.
+  - The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
+
+The following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads (Note: you don't need `sudo` for this command):
+```bash
+scripts/pinned_build.sh deps build "$(nproc)"
+```
+Now you can optionally [test](#step-4---test) your build, or [install](#step-5---install) the `*.deb` binary packages, which will be in the root of your build directory.
+
+#### Unpinned Build
+The following instructions are valid for this branch. Other release branches may have different requirements, so ensure you follow the directions in the branch or release you intend to build. If you are in an Ubuntu docker container, omit `sudo` because you run as `root` by default.
+
+<details> <summary>Ubuntu 22.04 Jammy & Ubuntu 20.04 Focal</summary>
+
+Install dependencies:
+```bash
+sudo apt-get update
+sudo apt-get install -y \
+        build-essential \
+        cmake \
+        curl \
+        git \
+        libboost-all-dev \
+        libcurl4-openssl-dev \
+        libgmp-dev \
+        libssl-dev \
+        libusb-1.0-0-dev \
+        llvm-11-dev \
         pkg-config
 ```
-and perform the build:
-```
-git submodule update --init --recursive
-mkdir build
+To build, make sure you are in the root of the `leap` repo, then run the following command:
+```bash
+mkdir -p build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/usr/lib/llvm-11 ..
 make -j $(nproc) package
 ```
 </details>
 
-<details>
-  <summary>Ubuntu 18.04 Build Instructions</summary>
+<details> <summary>Ubuntu 18.04 Bionic</summary>
 
-Install required dependencies. You will need to build Boost from source on this distribution. 
-```
-apt-get update && apt-get install   \
-        build-essential             \
-        cmake                       \
-        curl                        \
-        g++-8                       \
-        git                         \
-        libcurl4-openssl-dev        \
-        libgmp-dev                  \
-        libssl-dev                  \
-        libusb-1.0-0-dev            \
-        llvm-7-dev                  \
-        pkg-config                  \
-        python3                     \
+Install dependencies:
+```bash
+sudo apt-get update
+sudo apt-get install -y \
+        build-essential \
+        cmake \
+        curl \
+        g++-8 \
+        git \
+        libcurl4-openssl-dev \
+        libgmp-dev \
+        libssl-dev \
+        libusb-1.0-0-dev \
+        llvm-7-dev \
+        pkg-config \
+        python3 \
         zlib1g-dev
-        
-curl -L https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 | tar jx && \
-   cd boost_1_79_0 &&                                                                                     \
-   ./bootstrap.sh --prefix=$HOME/boost1.79 &&                                                             \
-   ./b2 --with-iostreams --with-date_time --with-filesystem --with-system                                 \
-        --with-program_options --with-chrono --with-test -j$(nproc) install &&                            \
-   cd ..
 ```
-and perform the build:
+You need to build Boost from source on this distribution:
+```bash
+curl -fL https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 -o ~/Downloads/boost_1_79_0.tar.bz2
+tar -jvxf ~/Downloads/boost_1_79_0.tar.bz2 -C ~/Downloads/
+pushd ~/Downloads/boost_1_79_0
+./bootstrap.sh --prefix="$HOME/boost1.79"
+./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -j "$(nproc)" install
+popd
 ```
-git submodule update --init --recursive
-mkdir build
+The Boost `*.tar.bz2` download and `boost_1_79_0` folder can be removed now if you want more space.
+```bash
+rm -r ~/Downloads/boost_1_79_0.tar.bz2 ~/Downloads/boost_1_79_0
+```
+From a terminal in the root of the `leap` repo, build.
+```bash
+mkdir -p build
 cd build
-cmake -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 \
-      -DCMAKE_PREFIX_PATH="$HOME/boost1.79;/usr/lib/llvm-7/"  -DCMAKE_BUILD_TYPE=Release .. \
-make -j $(nproc) package
+cmake -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 -DCMAKE_PREFIX_PATH="$HOME/boost1.79;/usr/lib/llvm-7/" -DCMAKE_BUILD_TYPE=Release ..
+make -j "$(nproc)" package
 ```
-After building you may remove the `$HOME/boost1.79` directory, or you may keep it around until next time building the software.
+After building, you may remove the `~/boost1.79` directory or you may keep it around for your next build.
 </details>
 
-### Running Tests
+Now you can optionally [test](#step-4---test) your build, or [install](#step-5---install) the `*.deb` binary packages, which will be in the root of your build directory.
 
-When building from source it's recommended to run at least what we refer to as the "parallelizable tests". Not included by default in the "parallelizable tests" are the WASM spec tests which can add additional coverage and can also be run in parallel.
+### Step 4 - Test
+Leap supports the following test suites:
 
-```
-cd build
+Test Suite | Test Type | [Test Size](https://testing.googleblog.com/2010/12/test-sizes.html) | Notes
+---|:---:|:---:|---
+[Parallelizable tests](#parallelizable-tests) | Unit tests | Small
+[WASM spec tests](#wasm-spec-tests) | Unit tests | Small | Unit tests for our WASM runtime, each short but _very_ CPU-intensive
+[Serial tests](#serial-tests) | Component/Integration | Medium
+[Long-running tests](#long-running-tests) | Integration | Medium-to-Large | Tests which take an extraordinarily long amount of time to run
 
-# "parallelizable tests": the minimum test set that should be run
-ctest -j $(nproc) -LE _tests
+When building from source, we recommended running at least the [parallelizable tests](#parallelizable-tests).
 
-# Also consider running the WASM spec tests for more coverage
-ctest -j $(nproc) -L wasm_spec_tests
+#### Parallelizable Tests
+This test suite consists of any test that does not require shared resources, such as file descriptors, specific folders, or ports, and can therefore be run concurrently in different threads without side effects (hence, easily parallelized). These are mostly unit tests and [small tests](https://testing.googleblog.com/2010/12/test-sizes.html) which complete in a short amount of time.
+
+You can invoke them by running `ctest` from a terminal in your Leap build directory and specifying the following arguments:
+```bash
+ctest -j "$(nproc)" -LE _tests
 ```
 
-Some other tests are available and recommended but be aware they can be sensitive to other software running on the same host and they may **SIGKILL** other nodeos instances running on the host.
-```
-cd build
+#### WASM Spec Tests
+The WASM spec tests verify that our WASM execution engine is compliant with the web assembly standard. These are very [small](https://testing.googleblog.com/2010/12/test-sizes.html), very fast unit tests. However, there are over a thousand of them so the suite can take a little time to run. These tests are extremely CPU-intensive.
 
-# These tests can't run in parallel but are recommended.
+You can invoke them by running `ctest` from a terminal in your Leap build directory and specifying the following arguments:
+```bash
+ctest -j "$(nproc)" -L wasm_spec_tests
+```
+We have observed severe performance issues when multiple virtual machines are running this test suite on the same physical host at the same time, for example in a CICD system. This can be resolved by disabling hyperthreading on the host.
+
+#### Serial Tests
+The serial test suite consists of [medium](https://testing.googleblog.com/2010/12/test-sizes.html) component or integration tests that use specific paths, ports, rely on process names, or similar, and cannot be run concurrently with other tests. Serial tests can be sensitive to other software running on the same host and they may `SIGKILL` other `nodeos` processes. These tests take a moderate amount of time to complete, but we recommend running them.
+
+You can invoke them by running `ctest` from a terminal in your Leap build directory and specifying the following arguments:
+```bash
 ctest -L "nonparallelizable_tests"
+```
 
-# These tests can't run in parallel. They also take a long time to run.
+#### Long-Running Tests
+The long-running tests are [medium-to-large](https://testing.googleblog.com/2010/12/test-sizes.html) integration tests that rely on shared resources and take a very long time to run.
+
+You can invoke them by running `ctest` from a terminal in your Leap build directory and specifying the following arguments:
+```bash
 ctest -L "long_running_tests"
 ```
+
+### Step 5 - Install
+Once you have [built](#step-3---build-the-source-code) Leap and [tested](#step-4---test) your build, you can install Leap on your system. Don't forget to omit `sudo` if you are running in a docker container.
+
+We recommend installing the binary package you just built. Navigate to your Leap build directory in a terminal and run this command:
+```bash
+sudo apt-get update
+sudo apt-get install -y ./leap[-_][0-9]*.deb
+```
+
+It is also possible to install using `make` instead:
+```bash
+sudo make install
+```

docs.json

@@ -1,65 +0,0 @@
-{
-    "name": "eos",
-    "generators": [
-        {
-            "name": "collate_markdown",
-            "options": {
-                "docs_dir": "docs"
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/chain_api_plugin/chain.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/chain_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/db_size_api_plugin/db_size.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/db_size_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/producer_api_plugin/producer.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/producer_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/net_api_plugin/net.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/net_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/test_control_api_plugin/test_control.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/test_control_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        },
-        {
-            "name": "swagger",
-            "options": {
-                "swagger_path": "plugins/trace_api_plugin/trace_api.swagger.yaml",
-                "swagger_dest_path": "nodeos/plugins/trace_api_plugin/api-reference",
-                "disable_filters": true,
-                "disable_summary_gen": true
-            }
-        }
-    ]
-}

docs/00_install/01_build-from-source/index.md

@@ -2,123 +2,19 @@
 content_title: Build Antelope from Source
 ---
 
-The shell scripts previously recommended for building the software have been removed in favor of a build process entirely driven by CMake. Those wishing to build from source are now responsible for installing the necessary dependencies. The list of dependencies and the recommended build procedure are in the README.md file. Instructions are also included for efficiently running the tests.
+The shell scripts previously recommended for building the software have been removed in favor of a build process entirely driven by CMake. Those wishing to build from source are now responsible for installing the necessary dependencies. The list of dependencies and the recommended build procedure are in the [`README.md`](https://github.com/AntelopeIO/leap/blob/release/3.1/README.md) file. Instructions are also included for efficiently running the tests.
 
 ### Using DUNE
-
-As an alternative to building from source try [Docker Utilities for Node Execution](https://github.com/AntelopeIO/DUNE) for the easiest way to get started, and for multi-platform support.  
+As an alternative to building from source, try [Docker Utilities for Node Execution](https://github.com/AntelopeIO/DUNE) for the easiest way to get started and for multi-platform support.
 
 ### Building From Source
-
-Recent Ubuntu LTS releases are the only Linux distributions that we fully support. Other Linux distros and other POSIX operating systems (such as macOS) are tended to on a best-effort basis and may not be full featured. Notable requirements to build are:
-* C++17 compiler and standard library
-* boost 1.67+
-* CMake 3.8+
-* (for Linux only) LLVM 7 - 11 (newer versions do not work)
-
-A few other common libraries are tools also required such as openssl 1.1+, libcurl, curl, libusb, GMP, Python 3, and zlib.
-
-**A Warning On Parallel Compilation Jobs (`-j` flag)**: When building C/C++ software often the build is performed in parallel via a command such as `make -j $(nproc)` which uses the number of CPU cores as the number of compilation jobs to perform simultaneously. However, be aware that some compilation units (.cpp files) in mandel are extremely complex and will consume nearly 4GB of memory to compile. You may need to reduce the level of parallelization depending on the amount of memory on your build host. e.g. instead of `make -j $(nproc)` run `make -j2`. Failures due to memory exhaustion will typically but not always manifest as compiler crashes.
-
-Generally we recommend performing what we refer to as a "pinned build" which ensures the compiler and boost version remain the same between builds of different mandel versions (mandel requires these versions remain the same otherwise its state needs to be repopulated from a portable snapshot).
+You can also build and install Leap from source. Instructions for that currently live [here](https://github.com/AntelopeIO/leap/blob/release/3.1/README.md#build-and-install-from-source).
 
 #### Building Pinned Build Binary Packages
-In the directory `<mandel src>/scripts` you will find the two scripts `install_deps.sh` and `pinned_build.sh`. If you haven't installed build dependencies then run `install_deps.sh`. Then run `pinned_build.sh <dependencies directory> <mandel build directory> <number of jobs>`.
-
-The dependencies directory is where the script will pull the C++ dependencies that need to be built with the pinned compiler for building the pinned binaries for binary packaging.
-
-The binary package will be produced in the mandel build directory that was supplied.
+The pinned build instructions have moved [here](https://github.com/AntelopeIO/leap/blob/release/3.1/README.md#pinned-build). You may want to look at the [prerequisites](https://github.com/AntelopeIO/leap/blob/release/3.1/README.md#prerequisites) and our warning on parallelization using the `-j` jobs flag [here](https://github.com/AntelopeIO/leap/blob/release/3.1/README.md#step-3---build) before you build.
 
 #### Manual (non "pinned") Build Instructions
-
-<details>
-  <summary>Ubuntu 20.04 & 22.04 Build Instructions</summary>
-
-Install required dependencies:
-```
-apt-get update && apt-get install   \
-        build-essential             \
-        cmake                       \
-        curl                        \
-        git                         \
-        libboost-all-dev            \
-        libcurl4-openssl-dev        \
-        libgmp-dev                  \
-        libssl-dev                  \
-        libusb-1.0-0-dev            \
-        llvm-11-dev                 \
-        pkg-config
-```
-and perform the build:
-```
-git submodule update --init --recursive
-mkdir build
-cd build
-cmake -DCMAKE_BUILD_TYPE=Release ..
-make -j $(nproc) package
-```
-</details>
-
-<details>
-  <summary>Ubuntu 18.04 Build Instructions</summary>
-
-Install required dependencies. You will need to build Boost from source on this distribution.
-```
-apt-get update && apt-get install   \
-        build-essential             \
-        cmake                       \
-        curl                        \
-        g++-8                       \
-        git                         \
-        libcurl4-openssl-dev        \
-        libgmp-dev                  \
-        libssl-dev                  \
-        libusb-1.0-0-dev            \
-        llvm-7-dev                  \
-        pkg-config                  \
-        python3                     \
-        zlib1g-dev
-
-curl -L https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 | tar jx && \
-   cd boost_1_79_0 &&                                                                                     \
-   ./bootstrap.sh --prefix=$HOME/boost1.79 &&                                                             \
-   ./b2 --with-iostreams --with-date_time --with-filesystem --with-system                                 \
-        --with-program_options --with-chrono --with-test -j$(nproc) install &&                            \
-   cd ..
-```
-and perform the build:
-```
-git submodule update --init --recursive
-mkdir build
-cd build
-cmake -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 \
-      -DCMAKE_PREFIX_PATH="$HOME/boost1.79;/usr/lib/llvm-7/"  -DCMAKE_BUILD_TYPE=Release .. \
-make -j $(nproc) package
-```
-After building you may remove the `$HOME/boost1.79` directory, or you may keep it around until next time building the software.
-</details>
+The unpinned build instructions have moved [here](https://github.com/AntelopeIO/leap/blob/release/3.1/README.md#unpinned-build). You may want to look at the [prerequisites](https://github.com/AntelopeIO/leap/blob/release/3.1/README.md#prerequisites) and our warning on parallelization using the `-j` jobs flag [here](https://github.com/AntelopeIO/leap/blob/release/3.1/README.md#step-3---build) before you build.
 
 ### Running Tests
-
-When building from source it's recommended to run at least what we refer to as the "parallelizable tests". Not included by default in the "parallelizable tests" are the WASM spec tests which can add additional coverage and can also be run in parallel.
-
-```
-cd build
-
-# "parallelizable tests": the minimum test set that should be run
-ctest -j $(nproc) -LE _tests
-
-# Also consider running the WASM spec tests for more coverage
-ctest -j $(nproc) -L wasm_spec_tests
-```
-
-Some other tests are available and recommended but be aware they can be sensitive to other software running on the same host and they may **SIGKILL** other nodeos instances running on the host.
-```
-cd build
-
-# These tests can't run in parallel but are recommended.
-ctest -L "nonparallelizable_tests"
-
-# These tests can't run in parallel. They also take a long time to run.
-ctest -L "long_running_tests"
-```
+Documentation on available test suites and how to run them has moved [here](https://github.com/AntelopeIO/leap/blob/release/3.1/README.md#test).

docs/00_install/index.md

@@ -10,9 +10,9 @@ The best way to install and use the Antelope software is to build it from source
 
 Antelope currently supports the following operating systems:
 
-1. Ubuntu 18.04
-2. Ubuntu 20.04
-3. Ubuntu 22.04
+1. Ubuntu 22.04 Jammy
+2. Ubuntu 20.04 Focal
+3. Ubuntu 18.04 Bionic
 
 [[info | Note]]
 | It may be possible to build and install Antelope on other Unix-based operating systems. We gathered helpful information on the following page but please keep in mind that it is experimental and not officially supported. 

docs/01_nodeos/03_plugins/chain_plugin/index.md

@@ -21,33 +21,35 @@ These can only be specified from the `nodeos` command-line:
 ```console
 Command Line Options for eosio::chain_plugin:
   --genesis-json arg                    File to read Genesis State from
-  --genesis-timestamp arg               override the initial timestamp in the 
+  --genesis-timestamp arg               override the initial timestamp in the
                                         Genesis State file
-  --print-genesis-json                  extract genesis_state from blocks.log 
+  --print-genesis-json                  extract genesis_state from blocks.log
                                         as JSON, print to console, and exit
-  --extract-genesis-json arg            extract genesis_state from blocks.log 
+  --extract-genesis-json arg            extract genesis_state from blocks.log
                                         as JSON, write into specified file, and
                                         exit
-  --print-build-info                    print build environment information to 
+  --print-build-info                    print build environment information to
                                         console as JSON and exit
-  --extract-build-info arg              extract build environment information 
+  --extract-build-info arg              extract build environment information
                                         as JSON, write into specified file, and
                                         exit
-  --force-all-checks                    do not skip any checks that can be 
-                                        skipped while replaying irreversible 
-                                        blocks
+  --force-all-checks                    do not skip any validation checks while
+                                        replaying blocks (useful for replaying
+                                        blocks from untrusted source)
   --disable-replay-opts                 disable optimizations that specifically
                                         target replay
-  --replay-blockchain                   clear chain state database and replay 
+  --replay-blockchain                   clear chain state database and replay
                                         all blocks
-  --hard-replay-blockchain              clear chain state database, recover as 
-                                        many blocks as possible from the block 
+  --hard-replay-blockchain              clear chain state database, recover as
+                                        many blocks as possible from the block
                                         log, and then replay those blocks
-  --delete-all-blocks                   clear chain state database and block 
+  --delete-all-blocks                   clear chain state database and block
                                         log
-  --truncate-at-block arg (=0)          stop hard replay / block log recovery 
-                                        at this block number (if set to 
+  --truncate-at-block arg (=0)          stop hard replay / block log recovery
+                                        at this block number (if set to
                                         non-zero number)
+  --terminate-at-block arg (=0)         terminate after reaching this block
+                                        number (if set to a non-zero number)
   --snapshot arg                        File to read Snapshot State from
 
 ```
@@ -58,162 +60,170 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::chain_plugin:
-  --blocks-dir arg (="blocks")          the location of the blocks directory 
-                                        (absolute path or relative to 
+  --blocks-dir arg (="blocks")          the location of the blocks directory
+                                        (absolute path or relative to
                                         application data dir)
   --protocol-features-dir arg (="protocol_features")
-                                        the location of the protocol_features 
+                                        the location of the protocol_features
                                         directory (absolute path or relative to
                                         application config dir)
-  --checkpoint arg                      Pairs of [BLOCK_NUM,BLOCK_ID] that 
+  --checkpoint arg                      Pairs of [BLOCK_NUM,BLOCK_ID] that
                                         should be enforced as checkpoints.
-  --wasm-runtime runtime (=eos-vm-jit)  Override default WASM runtime ( 
+  --wasm-runtime runtime (=eos-vm-jit)  Override default WASM runtime (
                                         "eos-vm-jit", "eos-vm")
-                                        "eos-vm-jit" : A WebAssembly runtime 
-                                        that compiles WebAssembly code to 
+                                        "eos-vm-jit" : A WebAssembly runtime
+                                        that compiles WebAssembly code to
                                         native x86 code prior to execution.
                                         "eos-vm" : A WebAssembly interpreter.
-                                        
-  --profile-account arg                 The name of an account whose code will 
+
+  --profile-account arg                 The name of an account whose code will
                                         be profiled
   --abi-serializer-max-time-ms arg (=15)
-                                        Override default maximum ABI 
+                                        Override default maximum ABI
                                         serialization time allowed in ms
-  --chain-state-db-size-mb arg (=1024)  Maximum size (in MiB) of the chain 
+  --chain-state-db-size-mb arg (=1024)  Maximum size (in MiB) of the chain
                                         state database
   --chain-state-db-guard-size-mb arg (=128)
-                                        Safely shut down node when free space 
-                                        remaining in the chain state database 
+                                        Safely shut down node when free space
+                                        remaining in the chain state database
                                         drops below this size (in MiB).
   --signature-cpu-billable-pct arg (=50)
                                         Percentage of actual signature recovery
-                                        cpu to bill. Whole number percentages, 
+                                        cpu to bill. Whole number percentages,
                                         e.g. 50 for 50%
-  --chain-threads arg (=2)              Number of worker threads in controller 
+  --chain-threads arg (=2)              Number of worker threads in controller
                                         thread pool
   --contracts-console                   print contract's output to console
-  --deep-mind                           print deeper information about chain 
+  --deep-mind                           print deeper information about chain
                                         operations
-  --actor-whitelist arg                 Account added to actor whitelist (may 
+  --actor-whitelist arg                 Account added to actor whitelist (may
                                         specify multiple times)
-  --actor-blacklist arg                 Account added to actor blacklist (may 
+  --actor-blacklist arg                 Account added to actor blacklist (may
                                         specify multiple times)
-  --contract-whitelist arg              Contract account added to contract 
+  --contract-whitelist arg              Contract account added to contract
                                         whitelist (may specify multiple times)
-  --contract-blacklist arg              Contract account added to contract 
+  --contract-blacklist arg              Contract account added to contract
                                         blacklist (may specify multiple times)
   --action-blacklist arg                Action (in the form code::action) added
-                                        to action blacklist (may specify 
+                                        to action blacklist (may specify
                                         multiple times)
-  --key-blacklist arg                   Public key added to blacklist of keys 
-                                        that should not be included in 
-                                        authorities (may specify multiple 
+  --key-blacklist arg                   Public key added to blacklist of keys
+                                        that should not be included in
+                                        authorities (may specify multiple
                                         times)
-  --sender-bypass-whiteblacklist arg    Deferred transactions sent by accounts 
-                                        in this list do not have any of the 
-                                        subjective whitelist/blacklist checks 
-                                        applied to them (may specify multiple 
+  --sender-bypass-whiteblacklist arg    Deferred transactions sent by accounts
+                                        in this list do not have any of the
+                                        subjective whitelist/blacklist checks
+                                        applied to them (may specify multiple
                                         times)
-  --read-mode arg (=speculative)        Database read mode ("speculative", 
+  --read-mode arg (=speculative)        Database read mode ("speculative",
                                         "head", "read-only", "irreversible").
-                                        In "speculative" mode: database 
-                                        contains state changes by transactions 
-                                        in the blockchain up to the head block 
-                                        as well as some transactions not yet 
+                                        In "speculative" mode: database
+                                        contains state changes by transactions
+                                        in the blockchain up to the head block
+                                        as well as some transactions not yet
                                         included in the blockchain.
                                         In "head" mode: database contains state
-                                        changes by only transactions in the 
-                                        blockchain up to the head block; 
-                                        transactions received by the node are 
+                                        changes by only transactions in the
+                                        blockchain up to the head block;
+                                        transactions received by the node are
                                         relayed if valid.
-                                        In "read-only" mode: (DEPRECATED: see 
-                                        p2p-accept-transactions & 
-                                        api-accept-transactions) database 
-                                        contains state changes by only 
-                                        transactions in the blockchain up to 
-                                        the head block; transactions received 
+                                        In "read-only" mode: (DEPRECATED: see
+                                        p2p-accept-transactions &
+                                        api-accept-transactions) database
+                                        contains state changes by only
+                                        transactions in the blockchain up to
+                                        the head block; transactions received
                                         via the P2P network are not relayed and
-                                        transactions cannot be pushed via the 
+                                        transactions cannot be pushed via the
                                         chain API.
-                                        In "irreversible" mode: database 
-                                        contains state changes by only 
-                                        transactions in the blockchain up to 
-                                        the last irreversible block; 
-                                        transactions received via the P2P 
-                                        network are not relayed and 
-                                        transactions cannot be pushed via the 
+                                        In "irreversible" mode: database
+                                        contains state changes by only
+                                        transactions in the blockchain up to
+                                        the last irreversible block;
+                                        transactions received via the P2P
+                                        network are not relayed and
+                                        transactions cannot be pushed via the
                                         chain API.
-                                        
-  --api-accept-transactions arg (=1)    Allow API transactions to be evaluated 
+
+  --api-accept-transactions arg (=1)    Allow API transactions to be evaluated
                                         and relayed if valid.
-  --validation-mode arg (=full)         Chain validation mode ("full" or 
+  --validation-mode arg (=full)         Chain validation mode ("full" or
                                         "light").
                                         In "full" mode all incoming blocks will
                                         be fully validated.
-                                        In "light" mode all incoming blocks 
-                                        headers will be fully validated; 
-                                        transactions in those validated blocks 
-                                        will be trusted 
-                                        
-  --disable-ram-billing-notify-checks   Disable the check which subjectively 
+                                        In "light" mode all incoming blocks
+                                        headers will be fully validated;
+                                        transactions in those validated blocks
+                                        will be trusted
+
+  --disable-ram-billing-notify-checks   Disable the check which subjectively
                                         fails a transaction if a contract bills
-                                        more RAM to another account within the 
+                                        more RAM to another account within the
                                         context of a notification handler (i.e.
-                                        when the receiver is not the code of 
+                                        when the receiver is not the code of
                                         the action).
   --maximum-variable-signature-length arg (=16384)
-                                        Subjectively limit the maximum length 
-                                        of variable components in a variable 
+                                        Subjectively limit the maximum length
+                                        of variable components in a variable
                                         legnth signature to this size in bytes
-  --trusted-producer arg                Indicate a producer whose blocks 
-                                        headers signed by it will be fully 
-                                        validated, but transactions in those 
+  --trusted-producer arg                Indicate a producer whose blocks
+                                        headers signed by it will be fully
+                                        validated, but transactions in those
                                         validated blocks will be trusted.
   --database-map-mode arg (=mapped)     Database map mode ("mapped", "heap", or
                                         "locked").
-                                        In "mapped" mode database is memory 
+                                        In "mapped" mode database is memory
                                         mapped as a file.
                                         In "heap" mode database is preloaded in
-                                        to swappable memory and will use huge 
+                                        to swappable memory and will use huge
                                         pages if available.
                                         In "locked" mode database is preloaded,
-                                        locked in to memory, and will use huge 
+                                        locked in to memory, and will use huge
                                         pages if available.
-                                        
-  --enable-account-queries arg (=0)     enable queries to find accounts by 
+
+  --eos-vm-oc-cache-size-mb arg (=1024) Maximum size (in MiB) of the EOS VM OC
+                                        code cache
+  --eos-vm-oc-compile-threads arg (=1)  Number of threads to use for EOS VM OC
+                                        tier-up
+  --eos-vm-oc-enable                    Enable EOS VM OC tier-up runtime
+  --enable-account-queries arg (=0)     enable queries to find accounts by
                                         various metadata.
   --max-nonprivileged-inline-action-size arg (=4096)
-                                        maximum allowed size (in bytes) of an 
-                                        inline action for a nonprivileged 
+                                        maximum allowed size (in bytes) of an
+                                        inline action for a nonprivileged
                                         account
   --transaction-retry-max-storage-size-gb arg
-                                        Maximum size (in GiB) allowed to be 
-                                        allocated for the Transaction Retry 
-                                        feature. Setting above 0 enables this 
+                                        Maximum size (in GiB) allowed to be
+                                        allocated for the Transaction Retry
+                                        feature. Setting above 0 enables this
                                         feature.
   --transaction-retry-interval-sec arg (=20)
-                                        How often, in seconds, to resend an 
-                                        incoming transaction to network if not 
+                                        How often, in seconds, to resend an
+                                        incoming transaction to network if not
                                         seen in a block.
-  --transaction-retry-max-expiration-sec arg (=90)
-                                        Maximum allowed transaction expiration 
-                                        for retry transactions, will retry 
+  --transaction-retry-max-expiration-sec arg (=120)
+                                        Maximum allowed transaction expiration
+                                        for retry transactions, will retry
                                         transactions up to this value.
   --transaction-finality-status-max-storage-size-gb arg
-                                        Maximum size (in GiB) allowed to be 
-                                        allocated for the Transaction Finality 
+                                        Maximum size (in GiB) allowed to be
+                                        allocated for the Transaction Finality
                                         Status feature. Setting above 0 enables
                                         this feature.
   --transaction-finality-status-success-duration-sec arg (=180)
-                                        Duration (in seconds) a successful 
-                                        transaction's Finality Status will 
-                                        remain available from being first 
+                                        Duration (in seconds) a successful
+                                        transaction's Finality Status will
+                                        remain available from being first
                                         identified.
   --transaction-finality-status-failure-duration-sec arg (=180)
-                                        Duration (in seconds) a failed 
-                                        transaction's Finality Status will 
-                                        remain available from being first 
+                                        Duration (in seconds) a failed
+                                        transaction's Finality Status will
+                                        remain available from being first
                                         identified.
+  --block-log-retain-blocks arg         if set, periodically prune the block
+                                        log to store only configured number of
+                                        most recent blocks
 
 ```
 

docs/01_nodeos/03_plugins/history_api_plugin/index.md

@@ -1,59 +0,0 @@
-[[warning | Deprecation Notice]]
-| The `history_plugin` that the `history_api_plugin` depends upon is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead.
-
-## Description
-
-The `history_api_plugin` exposes functionality from the [`history_plugin`](../history_plugin/index.md) to the RPC API interface managed by the [`http_plugin`](../http_plugin/index.md), providing read-only access to blockchain data.
-
-It provides four RPC API endpoints:
-
-* get_actions
-* get_transaction
-* get_key_accounts
-* get_controlled_accounts
-
-The four actions listed above are used by the following `cleos` commands (matching order):
-
-* get actions
-* get transaction
-* get accounts
-* get servants
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::history_api_plugin
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_api_plugin
-```
-
-## Options
-
-None
-
-## Dependencies
-
-* [`history_plugin`](../history_plugin/index.md)
-* [`chain_plugin`](../chain_plugin/index.md)
-* [`http_plugin`](../http_plugin/index.md)
-
-### Load Dependency Examples
-
-```console
-# config.ini
-plugin = eosio::history_plugin
-[options]
-plugin = eosio::chain_plugin
-[options]
-plugin = eosio::http_plugin
-[options]
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_plugin [options]  \
-           --plugin eosio::chain_plugin [operations] [options]  \
-           --plugin eosio::http_plugin [options]
-```

docs/01_nodeos/03_plugins/history_plugin/index.md

@@ -1,41 +0,0 @@
-[[warning | Deprecation Notice]]
-| The `history_plugin` is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead.
-
-## Description
-
-The `history_plugin` provides a cache layer to obtain historical data about the blockchain objects. It depends on [`chain_plugin`](../chain_plugin/index.md) for the data.
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::history_plugin
-[options]
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_plugin [options]
-```
-
-## Options
-
-These can be specified from both the `nodeos` command-line or the `config.ini` file:
-
-```console
-Config Options for eosio::history_plugin:
-  -f [ --filter-on ] arg                Track actions which match 
-                                        receiver:action:actor. Actor may be 
-                                        blank to include all. Action and Actor 
-                                        both blank allows all from Recieiver. 
-                                        Receiver may not be blank.
-  -F [ --filter-out ] arg               Do not track actions which match 
-                                        receiver:action:actor. Action and Actor
-                                        both blank excludes all from Reciever. 
-                                        Actor blank excludes all from 
-                                        reciever:action. Receiver may not be 
-                                        blank.
-```
-
-## Dependencies
-
-* [`chain_plugin`](../chain_plugin/index.md)

docs/01_nodeos/03_plugins/http_plugin/index.md

@@ -22,46 +22,47 @@ These can be specified from both the command-line or the `config.ini` file:
 
 ```console
 Config Options for eosio::http_plugin:
-  --unix-socket-path arg                The filename (relative to data-dir) to 
-                                        create a unix socket for HTTP RPC; set 
-                                        blank to disable (=keosd.sock for keosd)
-  --http-server-address arg (=127.0.0.1:8888 for nodeos)
-                                        The local IP and port to listen for 
+  --unix-socket-path arg                The filename (relative to data-dir) to
+                                        create a unix socket for HTTP RPC; set
+                                        blank to disable.
+  --http-server-address arg (=127.0.0.1:8888)
+                                        The local IP and port to listen for
                                         incoming http connections; set blank to
                                         disable.
-  --https-server-address arg            The local IP and port to listen for 
+  --https-server-address arg            The local IP and port to listen for
                                         incoming https connections; leave blank
                                         to disable.
-  --https-certificate-chain-file arg    Filename with the certificate chain to 
-                                        present on https connections. PEM 
+  --https-certificate-chain-file arg    Filename with the certificate chain to
+                                        present on https connections. PEM
                                         format. Required for https.
-  --https-private-key-file arg          Filename with https private key in PEM 
+  --https-private-key-file arg          Filename with https private key in PEM
                                         format. Required for https
-  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use: 
+  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use:
                                         secp384r1 or prime256v1
   --access-control-allow-origin arg     Specify the Access-Control-Allow-Origin
                                         to be returned on each request.
   --access-control-allow-headers arg    Specify the Access-Control-Allow-Header
                                         s to be returned on each request.
-  --access-control-max-age arg          Specify the Access-Control-Max-Age to 
+  --access-control-max-age arg          Specify the Access-Control-Max-Age to
                                         be returned on each request.
   --access-control-allow-credentials    Specify if Access-Control-Allow-Credent
-                                        ials: true should be returned on each 
+                                        ials: true should be returned on each
                                         request.
-  --max-body-size arg (=1048576)        The maximum body size in bytes allowed 
+  --max-body-size arg (=2097152)        The maximum body size in bytes allowed
                                         for incoming RPC requests
   --http-max-bytes-in-flight-mb arg (=500)
-                                        Maximum size in megabytes http_plugin 
-                                        should use for processing http 
-                                        requests. 503 error response when 
-                                        exceeded.
+                                        Maximum size in megabytes http_plugin
+                                        should use for processing http
+                                        requests. -1 for unlimited. 429 error
+                                        response when exceeded.
+  --http-max-response-time-ms arg (=30) Maximum time for processing a request.
   --verbose-http-errors                 Append the error log to HTTP responses
-  --http-validate-host arg (=1)         If set to false, then any incoming 
+  --http-validate-host arg (=1)         If set to false, then any incoming
                                         "Host" header is considered valid
-  --http-alias arg                      Additionaly acceptable values for the 
-                                        "Host" header of incoming HTTP 
-                                        requests, can be specified multiple 
-                                        times.  Includes http/s_server_address 
+  --http-alias arg                      Additionaly acceptable values for the
+                                        "Host" header of incoming HTTP
+                                        requests, can be specified multiple
+                                        times.  Includes http/s_server_address
                                         by default.
   --http-threads arg (=2)               Number of worker threads in http thread
                                         pool

docs/01_nodeos/03_plugins/net_plugin/index.md

@@ -44,7 +44,7 @@ Config Options for eosio::net_plugin:
   --p2p-accept-transactions arg (=1)    Allow transactions received over p2p
                                         network to be evaluated and relayed if
                                         valid.
-  --agent-name arg (="EOS Test Agent")  The name supplied to identify this node
+  --agent-name arg (=EOS Test Agent)    The name supplied to identify this node
                                         amongst the peers.
   --allowed-connection arg (=any)       Can be 'any' or 'producers' or
                                         'specified' or 'none'. If 'specified',
@@ -73,13 +73,15 @@ Config Options for eosio::net_plugin:
                                         synchronization
   --use-socket-read-watermark arg (=0)  Enable experimental socket read
                                         watermark optimization
-  --peer-log-format arg (=["${_name}" ${_ip}:${_port}])
+  --peer-log-format arg (=["${_name}" - ${_cid} ${_ip}:${_port}] )
                                         The string used to format peers when
                                         logging messages about them.  Variables
                                         are escaped with ${<variable name>}.
                                         Available Variables:
                                            _name  self-reported name
 
+                                           _cid   assigned connection id
+
                                            _id    self-reported ID (64 hex
                                                   characters)
 
@@ -94,7 +96,10 @@ Config Options for eosio::net_plugin:
                                                   peer
 
                                            _lport local port number connected
-                                                  to peer                                        
+                                                  to peer
+  --p2p-keepalive-interval-ms arg (=10000)
+                                        peer heartbeat keepalive message
+                                        interval in milliseconds
 ```
 
 ## Dependencies

docs/01_nodeos/03_plugins/producer_plugin/index.md

@@ -23,7 +23,6 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::producer_plugin:
-
   -e [ --enable-stale-production ]      Enable block production, even if the
                                         chain is stale.
   -x [ --pause-on-startup ]             Start this node in a state where
@@ -49,7 +48,7 @@ Config Options for eosio::producer_plugin:
                                         <public-key>=<provider-spec>
                                         Where:
                                            <public-key>    is a string form of
-                                                           a vaild Antelope public
+                                                           a valid EOS public
                                                            key
 
                                            <provider-spec> is a string in the
@@ -59,7 +58,7 @@ Config Options for eosio::producer_plugin:
                                            <provider-type> is KEY, or KEOSD
 
                                            KEY:<data>      is a string form of
-                                                           a valid Antelope
+                                                           a valid EOS
                                                            private key which
                                                            maps to the provided
                                                            public key
@@ -119,11 +118,6 @@ Config Options for eosio::producer_plugin:
                                         Sets the maximum amount of failures
                                         that are allowed for a given account
                                         per block.
-                                        Disregarded for accounts that have been
-                                        whitelisted by disabling subjective
-                                        billing for the account using the
-                                        disable-subjective-account-billing
-                                        configuration option.
   --subjective-account-decay-time-minutes arg (=1440)
                                         Sets the time to return full subjective
                                         cpu for accounts
@@ -142,9 +136,6 @@ Config Options for eosio::producer_plugin:
   --disable-subjective-account-billing arg
                                         Account which is excluded from
                                         subjective CPU billing
-                                        Account is considered whitelisted and
-                                        will not be subject to enforcement of
-                                        subjective-account-max-failures.
   --disable-subjective-p2p-billing arg (=1)
                                         Disable subjective CPU billing for P2P
                                         transactions

docs/01_nodeos/03_plugins/resource_monitor_plugin/index.md

@@ -1,19 +1,10 @@
 
 ## Overview
 
-The `resource_monitor_plugin` monitors space usage in the computing system where `nodeos` is running. Specifically, every `resource-monitor-interval-seconds` seconds,
-it measures the individual space used by each of the file systems mounted
-by `data-dir`, `state-dir`, `blocks-log-dir`, `snapshots-dir`,
-`state-history-dir`, and `trace-dir`.
-When space usage in any of the monitored file system is within `5%` of the threshold
-specified by `resource-monitor-space-threshold`, a warning containing the file system
-path and percentage of space has used is printed out.
-When space usage exceeds the threshold,
-if `resource-monitor-not-shutdown-on-threshold-exceeded` is not set,
-`nodeos` gracefully shuts down; if `resource-monitor-not-shutdown-on-threshold-exceeded` is set, `nodeos` prints out warnings periodically
-until space usage goes under the threshold.
+The `resource_monitor_plugin` monitors space usage in the computing system where `nodeos` is running. Specifically, every `resource-monitor-interval-seconds` seconds, it measures the individual space used by each of the file systems mounted by `data-dir`, `state-dir`, `blocks-log-dir`, `snapshots-dir`, `state-history-dir`, and `trace-dir`. When space usage in any of the monitored file system is within `5%` of the threshold specified by `resource-monitor-space-threshold`, a warning containing the file system path and percentage of space has used is printed out. When space usage exceeds the threshold, if `resource-monitor-not-shutdown-on-threshold-exceeded` is not set, `nodeos` gracefully shuts down; if `resource-monitor-not-shutdown-on-threshold-exceeded` is set, `nodeos` prints out warnings periodically until space usage goes under the threshold.
 
 `resource_monitor_plugin` is always loaded.
+
 ## Usage
 
 ```console
@@ -32,33 +23,28 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::resource_monitor_plugin:
-
   --resource-monitor-interval-seconds arg (=2)
-                                        Time in seconds between two consecutive checks
-                                        of space usage. Should be between 1 and 300.
+                                        Time in seconds between two consecutive
+                                        checks of resource usage. Should be
+                                        between 1 and 300
   --resource-monitor-space-threshold arg (=90)
-                                        Threshold in terms of percentage of used space
-                                        vs total space. If the used space is within
-                                        `5%` of the threshold, a warning is generated.
-                                        If the used space is above the threshold and
-                                        `resource-monitor-not-shutdown-on-threshold-exceeded`
-                                        is enabled, a shutdown is initiated; otherwise
-                                        a warning will be continuously printed out.
-                                        The value should be between 6 and 99.
+                                        Threshold in terms of percentage of
+                                        used space vs total space. If used
+                                        space is above (threshold - 5%), a
+                                        warning is generated.  Unless
+                                        resource-monitor-not-shutdown-on-thresh
+                                        old-exceeded is enabled, a graceful
+                                        shutdown is initiated if used space is
+                                        above the threshold. The value should
+                                        be between 6 and 99
   --resource-monitor-not-shutdown-on-threshold-exceeded
-                                        A switch used to indicate `nodeos` will "not"
-                                        shutdown when threshold is exceeded. When not
-                                        set, `nodeos` will shutdown.
+                                        Used to indicate nodeos will not
+                                        shutdown when threshold is exceeded.
   --resource-monitor-warning-interval arg (=30)
-                                        Number of monitor intervals between which a
-                                        warning is displayed.  For example, if
-                                        `resource-monitor-warning-interval` is to 10
-                                        and `resource-monitor-interval-seconds` is 2,
-                                        a warning will be displayed every 20 seconds,
-                                        even though the space usage is checked every
-                                        2 seconds.  This is used to throttle the
-                                        number of warnings in the `nodeos` log file.
-                                        Should be between 1 and 450.
+                                        Number of resource monitor intervals
+                                        between two consecutive warnings when
+                                        the threshold is hit. Should be between
+                                        1 and 450
 ```
 
 ## Plugin Dependencies

docs/01_nodeos/03_plugins/state_history_plugin/index.md

@@ -31,19 +31,21 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::state_history_plugin:
-
   --state-history-dir arg (="state-history")
-                                        the location of the state-history 
+                                        the location of the state-history
                                         directory (absolute path or relative to
                                         application data dir)
   --trace-history                       enable trace history
   --chain-state-history                 enable chain state history
   --state-history-endpoint arg (=127.0.0.1:8080)
-                                        the endpoint upon which to listen for 
-                                        incoming connections. Caution: only 
-                                        expose this port to your internal 
+                                        the endpoint upon which to listen for
+                                        incoming connections. Caution: only
+                                        expose this port to your internal
                                         network.
   --trace-history-debug-mode            enable debug mode for trace history
+  --state-history-log-retain-blocks arg if set, periodically prune the state
+                                        history files to store only configured
+                                        number of most recent blocks
 ```
 
 ## Examples

eos.doxygen.in

@@ -4,8 +4,8 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 DOXYFILE_ENCODING      = UTF-8
-PROJECT_NAME           = "EOS.IO"
-PROJECT_NUMBER         = "EOSIO ${DOXY_EOS_VERSION}"
+PROJECT_NAME           = "LEAP"
+PROJECT_NUMBER         = "${DOXY_EOS_VERSION}"
 PROJECT_BRIEF          =
 PROJECT_LOGO           = eos-logo.png
 OUTPUT_DIRECTORY       =
@@ -209,9 +209,9 @@ HTML_DYNAMIC_SECTIONS  = NO
 HTML_INDEX_NUM_ENTRIES = 100
 GENERATE_DOCSET        = NO
 DOCSET_FEEDNAME        = "Doxygen generated docs"
-DOCSET_BUNDLE_ID       = io.eos
-DOCSET_PUBLISHER_ID    = one.block
-DOCSET_PUBLISHER_NAME  = block.one
+DOCSET_BUNDLE_ID       = io.antelope
+DOCSET_PUBLISHER_ID    = com.eosnetwork
+DOCSET_PUBLISHER_NAME  = eosnetwork.com
 GENERATE_HTMLHELP      = NO
 CHM_FILE               =
 HHC_LOCATION           =
@@ -221,14 +221,14 @@ BINARY_TOC             = NO
 TOC_EXPAND             = NO
 GENERATE_QHP           = NO
 QCH_FILE               =
-QHP_NAMESPACE          = io.eos
+QHP_NAMESPACE          = io.antelope
 QHP_VIRTUAL_FOLDER     = doc
 QHP_CUST_FILTER_NAME   =
 QHP_CUST_FILTER_ATTRS  =
 QHP_SECT_FILTER_ATTRS  =
 QHG_LOCATION           =
 GENERATE_ECLIPSEHELP   = NO
-ECLIPSE_DOC_ID         = io.eos
+ECLIPSE_DOC_ID         = io.antelope
 DISABLE_INDEX          = NO
 GENERATE_TREEVIEW      = NO
 ENUM_VALUES_PER_LINE   = 4

libraries/chain/controller.cpp

@@ -238,7 +238,7 @@ struct controller_impl {
    protocol_feature_manager        protocol_features;
    controller::config              conf;
    const chain_id_type             chain_id; // read by thread_pool threads, value will not be changed
-   std::optional<fc::time_point>   replay_head_time;
+   bool                            replaying = false;
    db_read_mode                    read_mode = db_read_mode::SPECULATIVE;
    bool                            in_trx_requiring_checks = false; ///< if true, checks that are normally skipped on replay (e.g. auth checks) cannot be skipped
    std::optional<fc::microseconds> subjective_cpu_leeway;
@@ -477,8 +477,7 @@ struct controller_impl {
       }
 
       auto blog_head = blog.head();
-      auto blog_head_time = blog_head ? blog_head->timestamp.to_time_point() : fork_db.root()->header.timestamp.to_time_point();
-      replay_head_time = blog_head_time;
+      replaying = true;
       auto start_block_num = head->block_num + 1;
       auto start = fc::time_point::now();
 
@@ -546,7 +545,7 @@ struct controller_impl {
       ilog( "replayed ${n} blocks in ${duration} seconds, ${mspb} ms/block",
             ("n", head->block_num + 1 - start_block_num)("duration", (end-start).count()/1000000)
             ("mspb", ((end-start).count()/1000.0)/(head->block_num-start_block_num)) );
-      replay_head_time.reset();
+      replaying = false;
 
       if( except_ptr ) {
          std::rethrow_exception( except_ptr );
@@ -798,7 +797,10 @@ struct controller_impl {
       });
    }
 
-   void add_to_snapshot( const snapshot_writer_ptr& snapshot ) const {
+   void add_to_snapshot( const snapshot_writer_ptr& snapshot ) {
+      // clear in case the previous call to clear did not finish in time of deadline
+      clear_expired_input_transactions( fc::time_point::maximum() );
+
       snapshot->write_section<chain_snapshot_header>([this]( auto &section ){
          section.add_row(chain_snapshot_header(), db);
       });
@@ -971,7 +973,7 @@ struct controller_impl {
       );
    }
 
-   sha256 calculate_integrity_hash() const {
+   sha256 calculate_integrity_hash() {
       sha256::encoder enc;
       auto hash_writer = std::make_shared<integrity_hash_snapshot_writer>(enc);
       add_to_snapshot(hash_writer);
@@ -1499,10 +1501,8 @@ struct controller_impl {
                trx_context.init_for_implicit_trx();
                trx_context.enforce_whiteblacklist = false;
             } else {
-               bool skip_recording = replay_head_time && (time_point(trn.expiration) < *replay_head_time);
                trx_context.init_for_input_trx( trx->packed_trx()->get_unprunable_size(),
-                                               trx->packed_trx()->get_prunable_size(),
-                                               skip_recording);
+                                               trx->packed_trx()->get_prunable_size() );
             }
 
             trx_context.delay = fc::seconds(trn.delay_sec);
@@ -1700,7 +1700,7 @@ struct controller_impl {
          )
          {
             // Promote proposed schedule to pending schedule.
-            if( !replay_head_time ) {
+            if( !replaying ) {
                ilog( "promoting proposed schedule (set in block ${proposed_num}) to pending; current block: ${n} lib: ${lib} schedule: ${schedule} ",
                      ("proposed_num", *gpo.proposed_schedule_block_num)("n", pbhs.block_num)
                      ("lib", pbhs.dpos_irreversible_blocknum)
@@ -2056,43 +2056,64 @@ struct controller_impl {
       }
    } FC_CAPTURE_AND_RETHROW() } /// apply_block
 
+
+   // thread safe, expected to be called from thread other than the main thread
+   block_state_ptr create_block_state_i( const block_id_type& id, const signed_block_ptr& b, const block_header_state& prev ) {
+      auto trx_mroot = calculate_trx_merkle( b->transactions );
+      EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception,
+                  "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) );
+
+      const bool skip_validate_signee = false;
+      auto bsp = std::make_shared<block_state>(
+            prev,
+            b,
+            protocol_features.get_protocol_feature_set(),
+            [this]( block_timestamp_type timestamp,
+                    const flat_set<digest_type>& cur_features,
+                    const vector<digest_type>& new_features )
+            { check_protocol_features( timestamp, cur_features, new_features ); },
+            skip_validate_signee
+      );
+
+      EOS_ASSERT( id == bsp->id, block_validate_exception,
+                  "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id) );
+      return bsp;
+   }
+
    std::future<block_state_ptr> create_block_state_future( const block_id_type& id, const signed_block_ptr& b ) {
       EOS_ASSERT( b, block_validate_exception, "null block" );
 
+      return async_thread_pool( thread_pool.get_executor(), [b, id, control=this]() {
+         // no reason for a block_state if fork_db already knows about block
+         auto existing = control->fork_db.get_block( id );
+         EOS_ASSERT( !existing, fork_database_exception, "we already know about this block: ${id}", ("id", id) );
+
+         auto prev = control->fork_db.get_block_header( b->previous );
+         EOS_ASSERT( prev, unlinkable_block_exception,
+                     "unlinkable block ${id}", ("id", id)("previous", b->previous) );
+
+         return control->create_block_state_i( id, b, *prev );
+      } );
+   }
+
+   // thread safe, expected to be called from thread other than the main thread
+   block_state_ptr create_block_state( const block_id_type& id, const signed_block_ptr& b ) {
+      EOS_ASSERT( b, block_validate_exception, "null block" );
+
       // no reason for a block_state if fork_db already knows about block
       auto existing = fork_db.get_block( id );
       EOS_ASSERT( !existing, fork_database_exception, "we already know about this block: ${id}", ("id", id) );
 
+      // previous not found could mean that previous block not applied yet
       auto prev = fork_db.get_block_header( b->previous );
-      EOS_ASSERT( prev, unlinkable_block_exception,
-                  "unlinkable block ${id}", ("id", id)("previous", b->previous) );
+      if( !prev ) return {};
 
-      return async_thread_pool( thread_pool.get_executor(), [b, prev, id, control=this]() {
-         const bool skip_validate_signee = false;
-
-         auto trx_mroot = calculate_trx_merkle( b->transactions );
-         EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception,
-                     "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) );
-
-         auto bsp = std::make_shared<block_state>(
-                        *prev,
-                        move( b ),
-                        control->protocol_features.get_protocol_feature_set(),
-                        [control]( block_timestamp_type timestamp,
-                                   const flat_set<digest_type>& cur_features,
-                                   const vector<digest_type>& new_features )
-                        { control->check_protocol_features( timestamp, cur_features, new_features ); },
-                        skip_validate_signee
-         );
-
-         EOS_ASSERT( id == bsp->id, block_validate_exception,
-                     "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id) );
-         return bsp;
-      } );
+      return create_block_state_i( id, b, *prev );
    }
 
-   void push_block( std::future<block_state_ptr>& block_state_future,
-                    const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup )
+   void push_block( const block_state_ptr& bsp,
+                    const forked_branch_callback& forked_branch_cb,
+                    const trx_meta_cache_lookup& trx_lookup )
    {
       controller::block_status s = controller::block_status::complete;
       EOS_ASSERT(!pending, block_validate_exception, "it is not valid to push a block when there is a pending block");
@@ -2101,10 +2122,10 @@ struct controller_impl {
          trusted_producer_light_validation = old_value;
       });
       try {
-         block_state_ptr bsp = block_state_future.get();
+         EOS_ASSERT( bsp, block_validate_exception, "null block" );
          const auto& b = bsp->block;
 
-         if( conf.terminate_at_block > 0 && conf.terminate_at_block < self.head_block_num()) {
+         if( conf.terminate_at_block > 0 && conf.terminate_at_block <= self.head_block_num()) {
             ilog("Reached configured maximum block ${num}; terminating", ("num", conf.terminate_at_block) );
             shutdown();
             return;
@@ -2139,7 +2160,7 @@ struct controller_impl {
          EOS_ASSERT( (s == controller::block_status::irreversible || s == controller::block_status::validated),
                      block_validate_exception, "invalid block status for replay" );
 
-         if( conf.terminate_at_block > 0 && conf.terminate_at_block < self.head_block_num() ) {
+         if( conf.terminate_at_block > 0 && conf.terminate_at_block <= self.head_block_num() ) {
             ilog("Reached configured maximum block ${num}; terminating", ("num", conf.terminate_at_block) );
             shutdown();
             return;
@@ -2340,7 +2361,7 @@ struct controller_impl {
       //Look for expired transactions in the deduplication list, and remove them.
       auto& transaction_idx = db.get_mutable_index<transaction_multi_index>();
       const auto& dedupe_index = transaction_idx.indices().get<by_expiration>();
-      auto now = self.pending_block_time();
+      auto now = self.is_building_block() ? self.pending_block_time() : self.head_block_time();
       const auto total = dedupe_index.size();
       uint32_t num_removed = 0;
       while( (!dedupe_index.empty()) && ( now > fc::time_point(dedupe_index.begin()->expiration) ) ) {
@@ -2816,11 +2837,16 @@ std::future<block_state_ptr> controller::create_block_state_future( const block_
    return my->create_block_state_future( id, b );
 }
 
-void controller::push_block( std::future<block_state_ptr>& block_state_future,
-                             const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup )
+block_state_ptr controller::create_block_state( const block_id_type& id, const signed_block_ptr& b ) const {
+   return my->create_block_state( id, b );
+}
+
+void controller::push_block( const block_state_ptr& bsp,
+                             const forked_branch_callback& forked_branch_cb,
+                             const trx_meta_cache_lookup& trx_lookup )
 {
    validate_db_available_size();
-   my->push_block( block_state_future, forked_branch_cb, trx_lookup );
+   my->push_block( bsp, forked_branch_cb, trx_lookup );
 }
 
 transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx,
@@ -3054,11 +3080,11 @@ block_id_type controller::get_block_id_for_num( uint32_t block_num )const { try
    return id;
 } FC_CAPTURE_AND_RETHROW( (block_num) ) }
 
-sha256 controller::calculate_integrity_hash()const { try {
+sha256 controller::calculate_integrity_hash() { try {
    return my->calculate_integrity_hash();
 } FC_LOG_AND_RETHROW() }
 
-void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) const {
+void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) {
    EOS_ASSERT( !my->pending, block_validate_exception, "cannot take a consistent snapshot with a pending block" );
    return my->add_to_snapshot(snapshot);
 }
@@ -3468,10 +3494,14 @@ std::optional<chain_id_type> controller::extract_chain_id_from_db( const path& s
 
       if( db.revision() < 1 ) return {};
 
-      return db.get<global_property_object>().chain_id;
-   } catch( const bad_database_version_exception& ) {
-      throw;
-   } catch( ... ) {
+      auto * gpo = db.find<global_property_object>();
+      if (gpo==nullptr) return {};
+
+      return gpo->chain_id;
+   } catch( const std::system_error& e ) {
+      // do not propagate db_error_code::not_found for absent db, so it will be created
+      if( e.code().value() != chainbase::db_error_code::not_found )
+         throw;
    }
 
    return {};

libraries/chain/fork_database.cpp

@@ -9,6 +9,7 @@
 #include <boost/multi_index/composite_key.hpp>
 #include <fc/io/fstream.hpp>
 #include <fstream>
+#include <shared_mutex>
 
 namespace eosio { namespace chain {
    using boost::multi_index_container;
@@ -60,27 +61,44 @@ namespace eosio { namespace chain {
    }
 
    struct fork_database_impl {
-      fork_database_impl( fork_database& self, const fc::path& data_dir )
-      :self(self)
-      ,datadir(data_dir)
+      explicit fork_database_impl( const fc::path& data_dir )
+      :datadir(data_dir)
       {}
 
-      fork_database&        self;
+      std::shared_mutex     mtx;
       fork_multi_index_type index;
       block_state_ptr       root; // Only uses the block_header_state portion
       block_state_ptr       head;
       fc::path              datadir;
 
-      void add( const block_state_ptr& n,
-                bool ignore_duplicate, bool validate,
-                const std::function<void( block_timestamp_type,
-                                          const flat_set<digest_type>&,
-                                          const vector<digest_type>& )>& validator );
+      void open_impl( const std::function<void( block_timestamp_type,
+                                                const flat_set<digest_type>&,
+                                                const vector<digest_type>& )>& validator );
+      void close_impl();
+
+
+      block_header_state_ptr  get_block_header_impl( const block_id_type& id )const;
+      block_state_ptr         get_block_impl( const block_id_type& id )const;
+      void            reset_impl( const block_header_state& root_bhs );
+      void            rollback_head_to_root_impl();
+      void            advance_root_impl( const block_id_type& id );
+      void            remove_impl( const block_id_type& id );
+      branch_type     fetch_branch_impl( const block_id_type& h, uint32_t trim_after_block_num )const;
+      block_state_ptr search_on_branch_impl( const block_id_type& h, uint32_t block_num )const;
+      pair<branch_type, branch_type> fetch_branch_from_impl( const block_id_type& first,
+                                                             const block_id_type& second )const;
+      void mark_valid_impl( const block_state_ptr& h );
+
+      void add_impl( const block_state_ptr& n,
+                     bool ignore_duplicate, bool validate,
+                     const std::function<void( block_timestamp_type,
+                                               const flat_set<digest_type>&,
+                                               const vector<digest_type>& )>& validator );
    };
 
 
    fork_database::fork_database( const fc::path& data_dir )
-   :my( new fork_database_impl( *this, data_dir ) )
+   :my( new fork_database_impl( data_dir ) )
    {}
 
 
@@ -88,10 +106,18 @@ namespace eosio { namespace chain {
                                                        const flat_set<digest_type>&,
                                                        const vector<digest_type>& )>& validator )
    {
-      if (!fc::is_directory(my->datadir))
-         fc::create_directories(my->datadir);
+      std::lock_guard g( my->mtx );
+      my->open_impl( validator );
+   }
 
-      auto fork_db_dat = my->datadir / config::forkdb_filename;
+   void fork_database_impl::open_impl( const std::function<void( block_timestamp_type,
+                                                                 const flat_set<digest_type>&,
+                                                                 const vector<digest_type>& )>& validator )
+   {
+      if (!fc::is_directory(datadir))
+         fc::create_directories(datadir);
+
+      auto fork_db_dat = datadir / config::forkdb_filename;
       if( fc::exists( fork_db_dat ) ) {
          try {
             string content;
@@ -102,29 +128,29 @@ namespace eosio { namespace chain {
             // validate totem
             uint32_t totem = 0;
             fc::raw::unpack( ds, totem );
-            EOS_ASSERT( totem == magic_number, fork_database_exception,
+            EOS_ASSERT( totem == fork_database::magic_number, fork_database_exception,
                         "Fork database file '${filename}' has unexpected magic number: ${actual_totem}. Expected ${expected_totem}",
                         ("filename", fork_db_dat.generic_string())
                         ("actual_totem", totem)
-                        ("expected_totem", magic_number)
+                        ("expected_totem", fork_database::magic_number)
             );
 
             // validate version
             uint32_t version = 0;
             fc::raw::unpack( ds, version );
-            EOS_ASSERT( version >= min_supported_version && version <= max_supported_version,
+            EOS_ASSERT( version >= fork_database::min_supported_version && version <= fork_database::max_supported_version,
                         fork_database_exception,
                        "Unsupported version of fork database file '${filename}'. "
                        "Fork database version is ${version} while code supports version(s) [${min},${max}]",
                        ("filename", fork_db_dat.generic_string())
                        ("version", version)
-                       ("min", min_supported_version)
-                       ("max", max_supported_version)
+                       ("min", fork_database::min_supported_version)
+                       ("max", fork_database::max_supported_version)
             );
 
             block_header_state bhs;
             fc::raw::unpack( ds, bhs );
-            reset( bhs );
+            reset_impl( bhs );
 
             unsigned_int size; fc::raw::unpack( ds, size );
             for( uint32_t i = 0, n = size.value; i < n; ++i ) {
@@ -132,27 +158,27 @@ namespace eosio { namespace chain {
                fc::raw::unpack( ds, s );
                // do not populate transaction_metadatas, they will be created as needed in apply_block with appropriate key recovery
                s.header_exts = s.block->validate_and_extract_header_extensions();
-               my->add( std::make_shared<block_state>( move( s ) ), false, true, validator );
+               add_impl( std::make_shared<block_state>( std::move( s ) ), false, true, validator );
             }
             block_id_type head_id;
             fc::raw::unpack( ds, head_id );
 
-            if( my->root->id == head_id ) {
-               my->head = my->root;
+            if( root->id == head_id ) {
+               head = root;
             } else {
-               my->head = get_block( head_id );
-               EOS_ASSERT( my->head, fork_database_exception,
+               head = get_block_impl( head_id );
+               EOS_ASSERT( head, fork_database_exception,
                            "could not find head while reconstructing fork database from file; '${filename}' is likely corrupted",
                            ("filename", fork_db_dat.generic_string()) );
             }
 
-            auto candidate = my->index.get<by_lib_block_num>().begin();
-            if( candidate == my->index.get<by_lib_block_num>().end() || !(*candidate)->is_valid() ) {
-               EOS_ASSERT( my->head->id == my->root->id, fork_database_exception,
+            auto candidate = index.get<by_lib_block_num>().begin();
+            if( candidate == index.get<by_lib_block_num>().end() || !(*candidate)->is_valid() ) {
+               EOS_ASSERT( head->id == root->id, fork_database_exception,
                            "head not set to root despite no better option available; '${filename}' is likely corrupted",
                            ("filename", fork_db_dat.generic_string()) );
             } else {
-               EOS_ASSERT( !first_preferred( **candidate, *my->head ), fork_database_exception,
+               EOS_ASSERT( !first_preferred( **candidate, *head ), fork_database_exception,
                            "head not set to best available option available; '${filename}' is likely corrupted",
                            ("filename", fork_db_dat.generic_string()) );
             }
@@ -163,10 +189,15 @@ namespace eosio { namespace chain {
    }
 
    void fork_database::close() {
-      auto fork_db_dat = my->datadir / config::forkdb_filename;
+      std::lock_guard g( my->mtx );
+      my->close_impl();
+   }
 
-      if( !my->root ) {
-         if( my->index.size() > 0 ) {
+   void fork_database_impl::close_impl() {
+      auto fork_db_dat = datadir / config::forkdb_filename;
+
+      if( !root ) {
+         if( index.size() > 0 ) {
             elog( "fork_database is in a bad state when closing; not writing out '${filename}'",
                   ("filename", fork_db_dat.generic_string()) );
          }
@@ -174,13 +205,13 @@ namespace eosio { namespace chain {
       }
 
       std::ofstream out( fork_db_dat.generic_string().c_str(), std::ios::out | std::ios::binary | std::ofstream::trunc );
-      fc::raw::pack( out, magic_number );
-      fc::raw::pack( out, max_supported_version ); // write out current version which is always max_supported_version
-      fc::raw::pack( out, *static_cast<block_header_state*>(&*my->root) );
-      uint32_t num_blocks_in_fork_db = my->index.size();
+      fc::raw::pack( out, fork_database::magic_number );
+      fc::raw::pack( out, fork_database::max_supported_version ); // write out current version which is always max_supported_version
+      fc::raw::pack( out, *static_cast<block_header_state*>(&*root) );
+      uint32_t num_blocks_in_fork_db = index.size();
       fc::raw::pack( out, unsigned_int{num_blocks_in_fork_db} );
 
-      const auto& indx = my->index.get<by_lib_block_num>();
+      const auto& indx = index.get<by_lib_block_num>();
 
       auto unvalidated_itr = indx.rbegin();
       auto unvalidated_end = boost::make_reverse_iterator( indx.lower_bound( false ) );
@@ -215,30 +246,40 @@ namespace eosio { namespace chain {
          fc::raw::pack( out, *(*itr) );
       }
 
-      if( my->head ) {
-         fc::raw::pack( out, my->head->id );
+      if( head ) {
+         fc::raw::pack( out, head->id );
       } else {
          elog( "head not set in fork database; '${filename}' will be corrupted",
                ("filename", fork_db_dat.generic_string()) );
       }
 
-      my->index.clear();
+      index.clear();
    }
 
    fork_database::~fork_database() {
-      close();
+      my->close_impl();
    }
 
    void fork_database::reset( const block_header_state& root_bhs ) {
-      my->index.clear();
-      my->root = std::make_shared<block_state>();
-      static_cast<block_header_state&>(*my->root) = root_bhs;
-      my->root->validated = true;
-      my->head = my->root;
+      std::lock_guard g( my->mtx );
+      my->reset_impl(root_bhs);
+   }
+
+   void fork_database_impl::reset_impl( const block_header_state& root_bhs ) {
+      index.clear();
+      root = std::make_shared<block_state>();
+      static_cast<block_header_state&>(*root) = root_bhs;
+      root->validated = true;
+      head = root;
    }
 
    void fork_database::rollback_head_to_root() {
-      auto& by_id_idx = my->index.get<by_block_id>();
+      std::lock_guard g( my->mtx );
+      my->rollback_head_to_root_impl();
+   }
+
+   void fork_database_impl::rollback_head_to_root_impl() {
+      auto& by_id_idx = index.get<by_block_id>();
       auto itr = by_id_idx.begin();
       while (itr != by_id_idx.end()) {
          by_id_idx.modify( itr, [&]( block_state_ptr& bsp ) {
@@ -246,13 +287,18 @@ namespace eosio { namespace chain {
          } );
          ++itr;
       }
-      my->head = my->root;
+      head = root;
    }
 
    void fork_database::advance_root( const block_id_type& id ) {
-      EOS_ASSERT( my->root, fork_database_exception, "root not yet set" );
+      std::lock_guard g( my->mtx );
+      my->advance_root_impl( id );
+   }
 
-      auto new_root = get_block( id );
+   void fork_database_impl::advance_root_impl( const block_id_type& id ) {
+      EOS_ASSERT( root, fork_database_exception, "root not yet set" );
+
+      auto new_root = get_block_impl( id );
       EOS_ASSERT( new_root, fork_database_exception,
                   "cannot advance root to a block that does not exist in the fork database" );
       EOS_ASSERT( new_root->is_valid(), fork_database_exception,
@@ -262,48 +308,53 @@ namespace eosio { namespace chain {
       deque<block_id_type> blocks_to_remove;
       for( auto b = new_root; b; ) {
          blocks_to_remove.emplace_back( b->header.previous );
-         b = get_block( blocks_to_remove.back() );
-         EOS_ASSERT( b || blocks_to_remove.back() == my->root->id, fork_database_exception, "invariant violation: orphaned branch was present in forked database" );
+         b = get_block_impl( blocks_to_remove.back() );
+         EOS_ASSERT( b || blocks_to_remove.back() == root->id, fork_database_exception, "invariant violation: orphaned branch was present in forked database" );
       }
 
       // The new root block should be erased from the fork database index individually rather than with the remove method,
       // because we do not want the blocks branching off of it to be removed from the fork database.
-      my->index.erase( my->index.find( id ) );
+      index.erase( index.find( id ) );
 
       // The other blocks to be removed are removed using the remove method so that orphaned branches do not remain in the fork database.
       for( const auto& block_id : blocks_to_remove ) {
-         remove( block_id );
+         remove_impl( block_id );
       }
 
       // Even though fork database no longer needs block or trxs when a block state becomes a root of the tree,
       // avoid mutating the block state at all, for example clearing the block shared pointer, because other
       // parts of the code which run asynchronously may later expect it remain unmodified.
 
-      my->root = new_root;
+      root = new_root;
    }
 
    block_header_state_ptr fork_database::get_block_header( const block_id_type& id )const {
-      if( my->root->id == id ) {
-         return my->root;
+      std::shared_lock g( my->mtx );
+      return my->get_block_header_impl( id );
+   }
+
+   block_header_state_ptr fork_database_impl::get_block_header_impl( const block_id_type& id )const {
+      if( root->id == id ) {
+         return root;
       }
 
-      auto itr = my->index.find( id );
-      if( itr != my->index.end() )
+      auto itr = index.find( id );
+      if( itr != index.end() )
          return *itr;
 
       return block_header_state_ptr();
    }
 
-   void fork_database_impl::add( const block_state_ptr& n,
-                                 bool ignore_duplicate, bool validate,
-                                 const std::function<void( block_timestamp_type,
-                                                           const flat_set<digest_type>&,
-                                                           const vector<digest_type>& )>& validator )
+   void fork_database_impl::add_impl( const block_state_ptr& n,
+                                      bool ignore_duplicate, bool validate,
+                                      const std::function<void( block_timestamp_type,
+                                                                const flat_set<digest_type>&,
+                                                                const vector<digest_type>& )>& validator )
    {
       EOS_ASSERT( root, fork_database_exception, "root not yet set" );
       EOS_ASSERT( n, fork_database_exception, "attempt to add null block state" );
 
-      auto prev_bh = self.get_block_header( n->header.previous );
+      auto prev_bh = get_block_header_impl( n->header.previous );
 
       EOS_ASSERT( prev_bh, unlinkable_block_exception,
                   "unlinkable block", ("id", n->id)("previous", n->header.previous) );
@@ -332,19 +383,27 @@ namespace eosio { namespace chain {
    }
 
    void fork_database::add( const block_state_ptr& n, bool ignore_duplicate ) {
-      my->add( n, ignore_duplicate, false,
-               []( block_timestamp_type timestamp,
-                   const flat_set<digest_type>& cur_features,
-                   const vector<digest_type>& new_features )
-               {}
+      std::lock_guard g( my->mtx );
+      my->add_impl( n, ignore_duplicate, false,
+                    []( block_timestamp_type timestamp,
+                        const flat_set<digest_type>& cur_features,
+                        const vector<digest_type>& new_features )
+                    {}
       );
    }
 
-   const block_state_ptr& fork_database::root()const { return my->root; }
+   block_state_ptr fork_database::root()const {
+      std::shared_lock g( my->mtx );
+      return my->root;
+   }
 
-   const block_state_ptr& fork_database::head()const { return my->head; }
+   block_state_ptr fork_database::head()const {
+      std::shared_lock g( my->mtx );
+      return my->head;
+   }
 
    block_state_ptr fork_database::pending_head()const {
+      std::shared_lock g( my->mtx );
       const auto& indx = my->index.get<by_lib_block_num>();
 
       auto itr = indx.lower_bound( false );
@@ -357,8 +416,13 @@ namespace eosio { namespace chain {
    }
 
    branch_type fork_database::fetch_branch( const block_id_type& h, uint32_t trim_after_block_num )const {
+      std::shared_lock g( my->mtx );
+      return my->fetch_branch_impl( h, trim_after_block_num );
+   }
+
+   branch_type fork_database_impl::fetch_branch_impl( const block_id_type& h, uint32_t trim_after_block_num )const {
       branch_type result;
-      for( auto s = get_block(h); s; s = get_block( s->header.previous ) ) {
+      for( auto s = get_block_impl(h); s; s = get_block_impl( s->header.previous ) ) {
          if( s->block_num <= trim_after_block_num )
              result.push_back( s );
       }
@@ -367,7 +431,12 @@ namespace eosio { namespace chain {
    }
 
    block_state_ptr fork_database::search_on_branch( const block_id_type& h, uint32_t block_num )const {
-      for( auto s = get_block(h); s; s = get_block( s->header.previous ) ) {
+      std::shared_lock g( my->mtx );
+      return my->search_on_branch_impl( h, block_num );
+   }
+
+   block_state_ptr fork_database_impl::search_on_branch_impl( const block_id_type& h, uint32_t block_num )const {
+      for( auto s = get_block_impl(h); s; s = get_block_impl( s->header.previous ) ) {
          if( s->block_num == block_num )
              return s;
       }
@@ -381,9 +450,15 @@ namespace eosio { namespace chain {
     */
    pair< branch_type, branch_type >  fork_database::fetch_branch_from( const block_id_type& first,
                                                                        const block_id_type& second )const {
+      std::shared_lock g( my->mtx );
+      return my->fetch_branch_from_impl( first, second );
+   }
+
+   pair< branch_type, branch_type >  fork_database_impl::fetch_branch_from_impl( const block_id_type& first,
+                                                                                 const block_id_type& second )const {
       pair<branch_type,branch_type> result;
-      auto first_branch = (first == my->root->id) ? my->root : get_block(first);
-      auto second_branch = (second == my->root->id) ? my->root : get_block(second);
+      auto first_branch = (first == root->id) ? root : get_block_impl(first);
+      auto second_branch = (second == root->id) ? root : get_block_impl(second);
 
       EOS_ASSERT(first_branch, fork_db_block_not_found, "block ${id} does not exist", ("id", first));
       EOS_ASSERT(second_branch, fork_db_block_not_found, "block ${id} does not exist", ("id", second));
@@ -392,7 +467,7 @@ namespace eosio { namespace chain {
       {
          result.first.push_back(first_branch);
          const auto& prev = first_branch->header.previous;
-         first_branch = (prev == my->root->id) ? my->root : get_block( prev );
+         first_branch = (prev == root->id) ? root : get_block_impl( prev );
          EOS_ASSERT( first_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", prev)
@@ -403,7 +478,7 @@ namespace eosio { namespace chain {
       {
          result.second.push_back( second_branch );
          const auto& prev = second_branch->header.previous;
-         second_branch = (prev == my->root->id) ? my->root : get_block( prev );
+         second_branch = (prev == root->id) ? root : get_block_impl( prev );
          EOS_ASSERT( second_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", prev)
@@ -417,9 +492,9 @@ namespace eosio { namespace chain {
          result.first.push_back(first_branch);
          result.second.push_back(second_branch);
          const auto &first_prev = first_branch->header.previous;
-         first_branch = get_block( first_prev );
+         first_branch = get_block_impl( first_prev );
          const auto &second_prev = second_branch->header.previous;
-         second_branch = get_block( second_prev );
+         second_branch = get_block_impl( second_prev );
          EOS_ASSERT( first_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", first_prev)
@@ -436,13 +511,18 @@ namespace eosio { namespace chain {
          result.second.push_back(second_branch);
       }
       return result;
-   } /// fetch_branch_from
+   } /// fetch_branch_from_impl
 
    /// remove all of the invalid forks built off of this id including this id
    void fork_database::remove( const block_id_type& id ) {
+      std::lock_guard g( my->mtx );
+      return my->remove_impl( id );
+   }
+
+   void fork_database_impl::remove_impl( const block_id_type& id ) {
       deque<block_id_type> remove_queue{id};
-      const auto& previdx = my->index.get<by_prev>();
-      const auto& head_id = my->head->id;
+      const auto& previdx = index.get<by_prev>();
+      const auto& head_id = head->id;
 
       for( uint32_t i = 0; i < remove_queue.size(); ++i ) {
          EOS_ASSERT( remove_queue[i] != head_id, fork_database_exception,
@@ -456,16 +536,19 @@ namespace eosio { namespace chain {
       }
 
       for( const auto& block_id : remove_queue ) {
-         auto itr = my->index.find( block_id );
-         if( itr != my->index.end() )
-            my->index.erase(itr);
+         index.erase( block_id );
       }
    }
 
    void fork_database::mark_valid( const block_state_ptr& h ) {
+      std::lock_guard g( my->mtx );
+      my->mark_valid_impl( h );
+   }
+
+   void fork_database_impl::mark_valid_impl( const block_state_ptr& h ) {
       if( h->validated ) return;
 
-      auto& by_id_idx = my->index.get<by_block_id>();
+      auto& by_id_idx = index.get<by_block_id>();
 
       auto itr = by_id_idx.find( h->id );
       EOS_ASSERT( itr != by_id_idx.end(), fork_database_exception,
@@ -476,15 +559,20 @@ namespace eosio { namespace chain {
          bsp->validated = true;
       } );
 
-      auto candidate = my->index.get<by_lib_block_num>().begin();
-      if( first_preferred( **candidate, *my->head ) ) {
-         my->head = *candidate;
+      auto candidate = index.get<by_lib_block_num>().begin();
+      if( first_preferred( **candidate, *head ) ) {
+         head = *candidate;
       }
    }
 
-   block_state_ptr   fork_database::get_block(const block_id_type& id)const {
-      auto itr = my->index.find( id );
-      if( itr != my->index.end() )
+   block_state_ptr fork_database::get_block(const block_id_type& id)const {
+      std::shared_lock g( my->mtx );
+      return my->get_block_impl(id);
+   }
+
+   block_state_ptr fork_database_impl::get_block_impl(const block_id_type& id)const {
+      auto itr = index.find( id );
+      if( itr != index.end() )
          return *itr;
       return block_state_ptr();
    }

libraries/chain/include/eosio/chain/controller.hpp

@@ -84,7 +84,7 @@ namespace eosio { namespace chain {
             bool                     allow_ram_billing_in_notify = false;
             uint32_t                 maximum_variable_signature_length = chain::config::default_max_variable_signature_length;
             bool                     disable_all_subjective_mitigations = false; //< for developer & testing purposes, can be configured using `disable-all-subjective-mitigations` when `EOSIO_DEVELOPER` build option is provided
-            uint32_t                 terminate_at_block     = 0; //< primarily for testing purposes
+            uint32_t                 terminate_at_block     = 0;
 
             wasm_interface::vm_type  wasm_runtime = chain::config::default_wasm_runtime;
             eosvmoc::config          eosvmoc_config;
@@ -166,14 +166,17 @@ namespace eosio { namespace chain {
          void sign_block( const signer_callback_type& signer_callback );
          void commit_block();
 
+         // thread-safe
          std::future<block_state_ptr> create_block_state_future( const block_id_type& id, const signed_block_ptr& b );
+         // thread-safe
+         block_state_ptr create_block_state( const block_id_type& id, const signed_block_ptr& b ) const;
 
          /**
-          * @param block_state_future provide from call to create_block_state_future
+          * @param bsp block to push
           * @param cb calls cb with forked applied transactions for each forked block
           * @param trx_lookup user provided lookup function for externally cached transaction_metadata
           */
-         void push_block( std::future<block_state_ptr>& block_state_future,
+         void push_block( const block_state_ptr& bsp,
                           const forked_branch_callback& cb,
                           const trx_meta_cache_lookup& trx_lookup );
 
@@ -244,12 +247,13 @@ namespace eosio { namespace chain {
          signed_block_ptr fetch_block_by_id( block_id_type id )const;
 
          block_state_ptr fetch_block_state_by_number( uint32_t block_num )const;
+         // return block_state from forkdb, thread-safe
          block_state_ptr fetch_block_state_by_id( block_id_type id )const;
 
          block_id_type get_block_id_for_num( uint32_t block_num )const;
 
-         sha256 calculate_integrity_hash()const;
-         void write_snapshot( const snapshot_writer_ptr& snapshot )const;
+         sha256 calculate_integrity_hash();
+         void write_snapshot( const snapshot_writer_ptr& snapshot );
 
          bool sender_avoids_whitelist_blacklist_enforcement( account_name sender )const;
          void check_actor_list( const flat_set<account_name>& actors )const;

libraries/chain/include/eosio/chain/fork_database.hpp

@@ -16,6 +16,8 @@ namespace eosio { namespace chain {
     * database tracks the longest chain and the last irreversible block number. All
     * blocks older than the last irreversible block are freed after emitting the
     * irreversible signal.
+    *
+    * An internal mutex is used to provide thread-safety.
     */
    class fork_database {
       public:
@@ -55,9 +57,9 @@ namespace eosio { namespace chain {
 
          void            remove( const block_id_type& id );
 
-         const block_state_ptr& root()const;
-         const block_state_ptr& head()const;
-         block_state_ptr        pending_head()const;
+         block_state_ptr root()const;
+         block_state_ptr head()const;
+         block_state_ptr pending_head()const;
 
          /**
           *  Returns the sequence of block states resulting from trimming the branch from the

libraries/chain/include/eosio/chain/transaction_context.hpp

@@ -45,8 +45,7 @@ namespace eosio { namespace chain {
          void init_for_implicit_trx( uint64_t initial_net_usage = 0 );
 
          void init_for_input_trx( uint64_t packed_trx_unprunable_size,
-                                  uint64_t packed_trx_prunable_size,
-                                  bool skip_recording);
+                                  uint64_t packed_trx_prunable_size );
 
          void init_for_deferred_trx( fc::time_point published );
 

libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp

@@ -96,15 +96,15 @@ public:
       return itr->trx_meta;
    }
 
-   template <typename Func>
-   bool clear_expired( const time_point& pending_block_time, const time_point& deadline, Func&& callback ) {
+   template <typename Yield, typename Callback>
+   bool clear_expired( const time_point& pending_block_time, Yield&& yield, Callback&& callback ) {
       auto& persisted_by_expiry = queue.get<by_expiry>();
       while( !persisted_by_expiry.empty() ) {
          const auto& itr = persisted_by_expiry.begin();
          if( itr->expiry > pending_block_time ) {
             break;
          }
-         if( deadline <= fc::time_point::now() ) {
+         if( yield() ) {
             return false;
          }
          callback( itr->trx_meta->packed_trx(), itr->trx_type );

libraries/chain/transaction_context.cpp

@@ -234,8 +234,7 @@ namespace eosio { namespace chain {
    }
 
    void transaction_context::init_for_input_trx( uint64_t packed_trx_unprunable_size,
-                                                 uint64_t packed_trx_prunable_size,
-                                                 bool skip_recording )
+                                                 uint64_t packed_trx_prunable_size )
    {
       const transaction& trx = packed_trx.get_transaction();
       if( trx.transaction_extensions.size() > 0 ) {
@@ -272,8 +271,7 @@ namespace eosio { namespace chain {
          validate_referenced_accounts( trx, enforce_whiteblacklist && control.is_producing_block() );
       }
       init( initial_net_usage);
-      if (!skip_recording)
-         record_transaction( packed_trx.id(), trx.expiration ); /// checks for dupes
+      record_transaction( packed_trx.id(), trx.expiration ); /// checks for dupes
    }
 
    void transaction_context::init_for_deferred_trx( fc::time_point p )

libraries/chain/webassembly/runtimes/eos-vm.cpp

@@ -247,7 +247,7 @@ std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instan
       wasm_code_ptr code((uint8_t*)code_bytes, code_size);
       apply_options options = { .max_pages = 65536,
                                 .max_call_depth = 0 };
-      std::unique_ptr<backend_t> bkend = std::make_unique<backend_t>(code, code_size, nullptr, options);
+      std::unique_ptr<backend_t> bkend = std::make_unique<backend_t>(code, code_size, nullptr, options, false); // uses 2-passes parsing
       eos_vm_host_functions_t::resolve(bkend->get_module());
       return std::make_unique<eos_vm_instantiated_module<Impl>>(this, std::move(bkend));
    } catch(eosio::vm::exception& e) {
@@ -273,7 +273,7 @@ std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::inst
       wasm_code_ptr code((uint8_t*)code_bytes, code_size);
       apply_options options = { .max_pages = 65536,
                                 .max_call_depth = 0 };
-      std::unique_ptr<backend_t> bkend = std::make_unique<backend_t>(code, code_size, nullptr, options);
+      std::unique_ptr<backend_t> bkend = std::make_unique<backend_t>(code, code_size, nullptr, options, false); // uses 2-passes parsing
       eos_vm_host_functions_t::resolve(bkend->get_module());
       return std::make_unique<eos_vm_profiling_module>(std::move(bkend), code_bytes, code_size);
    } catch(eosio::vm::exception& e) {

libraries/state_history/include/eosio/state_history/log.hpp

@@ -390,7 +390,7 @@ class state_history_log {
             index.skip(-sizeof(uint64_t));
 
             if (!(remaining % 10000))
-               ilog("${remaining} blocks remaining, log pos = ${pos}", ("num_found", remaining)("pos", pos));
+               ilog("${r} blocks remaining, log pos = ${pos}", ("r", remaining)("pos", pos));
          }
       }
 

libraries/testing/include/eosio/testing/tester.hpp

@@ -597,7 +597,7 @@ namespace eosio { namespace testing {
       signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override {
          auto sb = _produce_block(skip_time, false);
          auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb );
-         validating_node->push_block( bsf, forked_branch_callback{}, trx_meta_cache_lookup{} );
+         validating_node->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} );
 
          return sb;
       }
@@ -607,15 +607,15 @@ namespace eosio { namespace testing {
       }
 
       void validate_push_block(const signed_block_ptr& sb) {
-         auto bs = validating_node->create_block_state_future( sb->calculate_id(), sb );
-         validating_node->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} );
+         auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb );
+         validating_node->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} );
       }
 
       signed_block_ptr produce_empty_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override {
          unapplied_transactions.add_aborted( control->abort_block() );
          auto sb = _produce_block(skip_time, true);
          auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb );
-         validating_node->push_block( bsf, forked_branch_callback{}, trx_meta_cache_lookup{} );
+         validating_node->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} );
 
          return sb;
       }

libraries/testing/tester.cpp

@@ -322,7 +322,7 @@ namespace eosio { namespace testing {
    void base_tester::push_block(signed_block_ptr b) {
       auto bsf = control->create_block_state_future(b->calculate_id(), b);
       unapplied_transactions.add_aborted( control->abort_block() );
-      control->push_block( bsf, [this]( const branch_type& forked_branch ) {
+      control->push_block( bsf.get(), [this]( const branch_type& forked_branch ) {
          unapplied_transactions.add_forked( forked_branch );
       }, [this]( const transaction_id_type& id ) {
          return unapplied_transactions.get_trx( id );
@@ -1050,7 +1050,7 @@ namespace eosio { namespace testing {
             if( block ) { //&& !b.control->is_known_block(block->id()) ) {
                auto bsf = b.control->create_block_state_future( block->calculate_id(), block );
                b.control->abort_block();
-               b.control->push_block(bsf, forked_branch_callback{}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block);
+               b.control->push_block(bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block);
             }
          }
       };

package.cmake

@@ -47,13 +47,13 @@ set(CPACK_DEBIAN_BASE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.deb")
 string(REGEX REPLACE "^(${CMAKE_PROJECT_NAME})" "\\1-dev" CPACK_DEBIAN_DEV_FILE_NAME "${CPACK_DEBIAN_BASE_FILE_NAME}")
 
 #deb package tooling will be unable to detect deps for the dev package. llvm is tricky since we don't know what package could have been used; try to figure it out
-set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "libboost-all-dev;libssl-dev;libgmp-dev")
+set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "libboost-all-dev, libssl-dev, libgmp-dev")
 find_program(DPKG_QUERY "dpkg-query")
 if(DPKG_QUERY AND OS_RELEASE MATCHES "\n?ID=\"?ubuntu" AND LLVM_CMAKE_DIR)
    execute_process(COMMAND "${DPKG_QUERY}" -S "${LLVM_CMAKE_DIR}" COMMAND cut -d: -f1 RESULT_VARIABLE LLVM_PKG_FIND_RESULT OUTPUT_VARIABLE LLVM_PKG_FIND_OUTPUT)
    if(LLVM_PKG_FIND_OUTPUT)
       string(STRIP "${LLVM_PKG_FIND_OUTPUT}" LLVM_PKG_FIND_OUTPUT)
-      list(APPEND CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "${LLVM_PKG_FIND_OUTPUT}")
+      string(APPEND CPACK_DEBIAN_DEV_PACKAGE_DEPENDS ", ${LLVM_PKG_FIND_OUTPUT}")
    endif()
 endif()
 

plugins/chain_api_plugin/chain.swagger.yaml

@@ -1,13 +1,13 @@
 openapi: 3.0.0
 info:
   title: Chain API
-  description: "OAS 3.0 Nodeos Chain API Specification\r"
+  description: Nodeos Chain API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eos.io
+    url: https://eosnetwork.com
 servers:
   - url: "{protocol}://{host}:{port}/v1/chain"
     variables:

plugins/chain_interface/include/eosio/chain/plugin_interface.hpp

@@ -37,14 +37,9 @@ namespace eosio { namespace chain { namespace plugin_interface {
    }
 
    namespace incoming {
-      namespace channels {
-         using block                 = channel_decl<struct block_tag, signed_block_ptr>;
-         using transaction           = channel_decl<struct transaction_tag, packed_transaction_ptr>;
-      }
-
       namespace methods {
-         // synchronously push a block/trx to a single provider
-         using block_sync            = method_decl<chain_plugin_interface, bool(const signed_block_ptr&, const std::optional<block_id_type>&), first_provider_policy>;
+         // synchronously push a block/trx to a single provider, block_state_ptr may be null
+         using block_sync            = method_decl<chain_plugin_interface, bool(const signed_block_ptr&, const std::optional<block_id_type>&, const block_state_ptr&), first_provider_policy>;
          using transaction_async     = method_decl<chain_plugin_interface, void(const packed_transaction_ptr&, bool, bool, bool, next_function<transaction_trace_ptr>), first_provider_policy>;
       }
    }

plugins/chain_plugin/chain_plugin.cpp

@@ -142,7 +142,6 @@ public:
    ,irreversible_block_channel(app().get_channel<channels::irreversible_block>())
    ,accepted_transaction_channel(app().get_channel<channels::accepted_transaction>())
    ,applied_transaction_channel(app().get_channel<channels::applied_transaction>())
-   ,incoming_block_channel(app().get_channel<incoming::channels::block>())
    ,incoming_block_sync_method(app().get_method<incoming::methods::block_sync>())
    ,incoming_transaction_async_method(app().get_method<incoming::methods::transaction_async>())
    {}
@@ -170,7 +169,6 @@ public:
    channels::irreversible_block::channel_type&     irreversible_block_channel;
    channels::accepted_transaction::channel_type&   accepted_transaction_channel;
    channels::applied_transaction::channel_type&    applied_transaction_channel;
-   incoming::channels::block::channel_type&        incoming_block_channel;
 
    // retained references to methods for easy calling
    incoming::methods::block_sync::method_type&        incoming_block_sync_method;
@@ -1124,6 +1122,13 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
          // For the time being, when `deep-mind = true` is activated, we set `stdout` here to
          // be an unbuffered I/O stream.
          setbuf(stdout, NULL);
+         
+         //verify configuration is correct
+         EOS_ASSERT( options.at("api-accept-transactions").as<bool>() == false, plugin_config_exception,
+            "api-accept-transactions must be set to false in order to enable deep-mind logging.");
+
+         EOS_ASSERT( options.at("p2p-accept-transactions").as<bool>() == false, plugin_config_exception,
+            "p2p-accept-transactions must be set to false in order to enable deep-mind logging.");
 
          my->chain->enable_deep_mind( &_deep_mind_log );
       }
@@ -1326,8 +1331,8 @@ chain_apis::read_only chain_plugin::get_read_only_api() const {
 }
 
 
-bool chain_plugin::accept_block(const signed_block_ptr& block, const block_id_type& id ) {
-   return my->incoming_block_sync_method(block, id);
+bool chain_plugin::accept_block(const signed_block_ptr& block, const block_id_type& id, const block_state_ptr& bsp ) {
+   return my->incoming_block_sync_method(block, id, bsp);
 }
 
 void chain_plugin::accept_transaction(const chain::packed_transaction_ptr& trx, next_function<chain::transaction_trace_ptr> next) {
@@ -2123,7 +2128,7 @@ fc::variant read_only::get_block_header_state(const get_block_header_state_param
 
 void read_write::push_block(read_write::push_block_params&& params, next_function<read_write::push_block_results> next) {
    try {
-      app().get_method<incoming::methods::block_sync>()(std::make_shared<signed_block>(std::move(params)), {});
+      app().get_method<incoming::methods::block_sync>()(std::make_shared<signed_block>( std::move(params) ), std::optional<block_id_type>{}, block_state_ptr{});
    } catch ( boost::interprocess::bad_alloc& ) {
       chain_plugin::handle_db_exhaustion();
    } catch ( const std::bad_alloc& ) {

plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp

@@ -777,7 +777,7 @@ public:
    chain_apis::read_write get_read_write_api();
    chain_apis::read_only get_read_only_api() const;
 
-   bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id );
+   bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id, const chain::block_state_ptr& bsp );
    void accept_transaction(const chain::packed_transaction_ptr& trx, chain::plugin_interface::next_function<chain::transaction_trace_ptr> next);
 
    // Only call this after plugin_initialize()!

plugins/db_size_api_plugin/db_size.swagger.yaml

@@ -1,12 +1,13 @@
 openapi: 3.0.0
 info:
   title: DB Size API
+  description: Nodeos DB Size API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eos.io
+    url: https://eosnetwork.com
 servers:
   - url: '{protocol}://{host}:{port}/v1/'
     variables:

plugins/http_plugin/http_plugin.cpp

@@ -912,6 +912,7 @@ class http_plugin_impl : public std::enable_shared_from_this<http_plugin_impl> {
       my->url_handlers.clear();
 
       app().post( 0, [me = my](){} ); // keep my pointer alive until queue is drained
+      fc_ilog( logger, "exit shutdown");
    }
 
    void http_plugin::add_handler(const string& url, const url_handler& handler, int priority) {

plugins/net_api_plugin/net.swagger.yaml

@@ -1,12 +1,13 @@
 openapi: 3.0.0
 info:
   title: Net API
+  description: Nodeos Net API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eos.io
+    url: https://eosnetwork.com
 servers:
   - url: '{protocol}://{host}:{port}/v1/'
     variables:

plugins/net_plugin/net_plugin.cpp

@@ -26,6 +26,7 @@
 #include <boost/asio/steady_timer.hpp>
 
 #include <atomic>
+#include <cmath>
 #include <shared_mutex>
 
 using namespace eosio::chain::plugin_interface;
@@ -634,6 +635,7 @@ namespace eosio {
       string                  local_endpoint_port;
 
       std::atomic<uint32_t>   trx_in_progress_size{0};
+      fc::time_point          last_dropped_trx_msg_time;
       const uint32_t          connection_id;
       int16_t                 sent_handshake_count = 0;
       std::atomic<bool>       connecting{true};
@@ -784,7 +786,7 @@ namespace eosio {
       void handle_message( const packed_transaction& msg ) = delete; // packed_transaction_ptr overload used instead
       void handle_message( packed_transaction_ptr msg );
 
-      void process_signed_block( const block_id_type& id, signed_block_ptr msg );
+      void process_signed_block( const block_id_type& id, signed_block_ptr msg, block_state_ptr bsp );
 
       fc::variant_object get_logger_variant() const {
          fc::mutable_variant_object mvo;
@@ -1775,7 +1777,6 @@ namespace eosio {
       if( c->is_transactions_only_connection() ) return;
 
       uint32_t lib_num = 0;
-      uint32_t peer_lib = msg.last_irreversible_block_num;
       uint32_t head = 0;
       block_id_type head_id;
       std::tie( lib_num, std::ignore, head,
@@ -1789,10 +1790,10 @@ namespace eosio {
          peer_wlog(c, "Peer sent a handshake with a timestamp skewed by at least ${t}ms", ("t", network_latency_ns/1000000));
          network_latency_ns = 0;
       }
-      // number of blocks syncing node is behind from a peer node
-      uint32_t nblk_behind_by_net_latency = static_cast<uint32_t>(network_latency_ns / block_interval_ns);
-      // 2x for time it takes for message to reach back to peer node, +1 to compensate for integer division truncation
-      uint32_t nblk_combined_latency = 2 * nblk_behind_by_net_latency + 1;
+      // number of blocks syncing node is behind from a peer node, round up
+      uint32_t nblk_behind_by_net_latency = std::lround( static_cast<double>(network_latency_ns) / static_cast<double>(block_interval_ns) );
+      // 2x for time it takes for message to reach back to peer node
+      uint32_t nblk_combined_latency = 2 * nblk_behind_by_net_latency;
       // message in the log below is used in p2p_high_latency_test.py test
       peer_dlog(c, "Network latency is ${lat}ms, ${num} blocks discrepancy by network latency, ${tot_num} blocks discrepancy expected once message received",
                 ("lat", network_latency_ns/1000000)("num", nblk_behind_by_net_latency)("tot_num", nblk_combined_latency));
@@ -1812,8 +1813,8 @@ namespace eosio {
       //-----------------------------
 
       if (head_id == msg.head_id) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 0",
-                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 0, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))("l", lib_num) );
          c->syncing = false;
          notice_message note;
          note.known_blocks.mode = none;
@@ -1822,9 +1823,10 @@ namespace eosio {
          c->enqueue( note );
          return;
       }
-      if (head < peer_lib) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 1",
-                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+      if (head < msg.last_irreversible_block_num) {
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 1, head ${h}, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))
+                    ("h", head)("l", lib_num) );
          c->syncing = false;
          if (c->sent_handshake_count > 0) {
             c->send_handshake();
@@ -1832,8 +1834,9 @@ namespace eosio {
          return;
       }
       if (lib_num > msg.head_num + nblk_combined_latency) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 2",
-                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 2, head ${h}, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))
+                    ("h", head)("l", lib_num) );
          if (msg.generation > 1 || c->protocol_version > proto_base) {
             notice_message note;
             note.known_trx.pending = lib_num;
@@ -1847,14 +1850,16 @@ namespace eosio {
       }
 
       if (head + nblk_combined_latency < msg.head_num ) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 3",
-                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 3, head ${h}, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))
+                    ("h", head)("l", lib_num) );
          c->syncing = false;
          verify_catchup(c, msg.head_num, msg.head_id);
          return;
       } else if(head >= msg.head_num + nblk_combined_latency) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 4",
-                  ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 4, head ${h}, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))
+                    ("h", head)("l", lib_num) );
          if (msg.generation > 1 ||  c->protocol_version > proto_base) {
             notice_message note;
             note.known_trx.mode = none;
@@ -2340,14 +2345,6 @@ namespace eosio {
 
    // called from connection strand
    void connection::connect( const std::shared_ptr<tcp::resolver>& resolver, tcp::resolver::results_type endpoints ) {
-      switch ( no_retry ) {
-         case no_reason:
-         case wrong_version:
-         case benign_other:
-            break;
-         default:
-            return;
-      }
       connecting = true;
       pending_message_buffer.reset();
       buffer_queue.clear_out_queue();
@@ -2683,6 +2680,10 @@ namespace eosio {
          char reason[72];
          snprintf(reason, 72, "Dropping trx, too many trx in progress %lu bytes", trx_in_progress_sz);
          my_impl->producer_plug->log_failed_transaction(ptr->id(), ptr, reason);
+         if (fc::time_point::now() - fc::seconds(1) >= last_dropped_trx_msg_time) {
+            last_dropped_trx_msg_time = fc::time_point::now();
+            peer_wlog(this, reason);
+         }
          return true;
       }
       bool have_trx = my_impl->dispatcher->have_txn( ptr->id() );
@@ -3138,13 +3139,44 @@ namespace eosio {
    // called from connection strand
    void connection::handle_message( const block_id_type& id, signed_block_ptr ptr ) {
       peer_dlog( this, "received signed_block ${num}, id ${id}", ("num", ptr->block_num())("id", id) );
-      app().post(priority::medium, [ptr{std::move(ptr)}, id, c = shared_from_this()]() mutable {
-         c->process_signed_block( id, std::move( ptr ) );
+
+      controller& cc = my_impl->chain_plug->chain();
+      block_state_ptr bsp;
+      bool exception = false;
+      try {
+         if( cc.fetch_block_state_by_id( id ) ) {
+            my_impl->dispatcher->add_peer_block( id, connection_id );
+            my_impl->sync_master->sync_recv_block( shared_from_this(), id, ptr->block_num(), false );
+            return;
+         }
+         // this may return null if block is not immediately ready to be processed
+         bsp = cc.create_block_state( id, ptr );
+      } catch( const fc::exception& ex ) {
+         exception = true;
+         peer_elog(this, "bad block exception: #${n} ${id}...: ${m}",
+                   ("n", ptr->block_num())("id", id.str().substr(8,16))("m",ex.to_string()));
+      } catch( ... ) {
+         exception = true;
+         peer_elog(this, "bad block: #${n} ${id}...: unknown exception",
+                   ("n", ptr->block_num())("id", id.str().substr(8,16)));
+      }
+      if( exception ) {
+         my_impl->sync_master->rejected_block( shared_from_this(), ptr->block_num() );
+         my_impl->dispatcher->rejected_block( id );
+         return;
+      }
+
+      bool signal_producer = !!bsp; // ready to process immediately, so signal producer to interrupt start_block
+      app().post(priority::medium, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c = shared_from_this()]() mutable {
+         c->process_signed_block( id, std::move(ptr), std::move(bsp) );
       });
+
+      if( signal_producer )
+         my_impl->producer_plug->received_block();
    }
 
    // called from application thread
-   void connection::process_signed_block( const block_id_type& blk_id, signed_block_ptr msg ) {
+   void connection::process_signed_block( const block_id_type& blk_id, signed_block_ptr msg, block_state_ptr bsp ) {
       controller& cc = my_impl->chain_plug->chain();
       uint32_t blk_num = msg->block_num();
       // use c in this method instead of this to highlight that all methods called on c-> must be thread safe
@@ -3174,7 +3206,7 @@ namespace eosio {
 
       go_away_reason reason = fatal_other;
       try {
-         bool accepted = my_impl->chain_plug->accept_block(msg, blk_id);
+         bool accepted = my_impl->chain_plug->accept_block(msg, blk_id, bsp);
          my_impl->update_chain_info();
          if( !accepted ) return;
          reason = no_reason;

plugins/producer_api_plugin/producer.swagger.yaml

@@ -1,16 +1,18 @@
 openapi: 3.0.0
 info:
   title: Producer API
+  description: Nodeos Producer API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eos.io
+    url: https://antelope.io
 tags:
-  - name: eosio
+  - name: Protocol Version 3.1
+    description: The release tag for Leap binaries is also the protocol version
 servers:
-  - url: "{protocol}://{host}:{port}/v1/"
+  - url: "{protocol}://{host}:{port}/v1"
     variables:
       protocol:
         enum:
@@ -21,285 +23,223 @@ servers:
         default: localhost
       port:
         default: "8080"
-components:
-  securitySchemes: {}
-  schemas: {}
 security:
   - {}
 paths:
   /producer/pause:
     post:
       summary: pause
-      description: Pause producer node
+      description: Pause producer node. Takes no arguments and returns no values.
       operationId: pause
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                type: boolean
-                description: "returns status"
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/resume:
     post:
       summary: resume
-      description: Resume producer node
+      description: Resume producer node. Takes no arguments and returns no values.
       operationId: resume
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              description: Resumes activity for producer
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/paused:
     post:
       summary: paused
-      description: Retreives paused status for producer node
+      description: Retrieves paused status for producer node. Takes no arguments and returns no values.
       operationId: paused
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                type: boolean
-                description: True if producer is paused, false otherwise
+                $ref: '#/components/schemas/OK_PAUSED'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/get_runtime_options:
     post:
       summary: get_runtime_options
-      description: Retreives run time options for producer node
+      description: Retrieves runtime options for producer node.
       operationId: get_runtime_options
-      parameters: []
+      responses:
+        "201":
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Runtime_Options'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
+  /producer/update_runtime_options:
+    post:
+      summary: update_runtime_options
+      description: Update runtime options for producer node. May post any of the runtime options in combination or alone.
+      operationId: update_runtime_options
       requestBody:
         content:
           application/json:
             schema:
-              type: object
-              properties: {}
+              $ref: '#/components/schemas/Runtime_Options'
       responses:
-        "200":
+        "201":
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
+  /producer/get_greylist:
+    post:
+      summary: get_greylist
+      description: Retrieves the greylist for producer node.
+      operationId: get_greylist
+      responses:
+        "201":
           description: OK
           content:
             application/json:
               schema:
                 type: object
-                description: Returns run time options set for the producer
                 properties:
-                  max_transaction_time:
-                    type: integer
-                    description: Max transaction time
-                  max_irreversible_block_age:
-                    type: integer
-                    description: Max irreversible block age
-                  produce_time_offset_us:
-                    type: integer
-                    description: Time offset
-                  last_block_time_offset_us:
-                    type: integer
-                    description: Last block time offset
-                  max_scheduled_transaction_time_per_block_ms:
-                    type: integer
-                    description: Max scheduled transaction time per block in ms
-                  subjective_cpu_leeway_us:
-                    type: integer
-                    description: Subjective CPU leeway
-                  incoming_defer_ratio:
-                    type: integer
-                    description: Incoming defer ration
-  /producer/update_runtime_options:
-    post:
-      summary: update_runtime_options
-      description: Update run time options for producer node
-      operationId: update_runtime_options
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              required:
-                - options
-              properties:
-                options:
-                  type: object
-                  description: Defines the run time options to set for the producer
-                  properties:
-                    max_transaction_time:
-                      type: integer
-                      description: Max transaction time
-                    max_irreversible_block_age:
-                      type: integer
-                      description: Max irreversible block age
-                    produce_time_offset_us:
-                      type: integer
-                      description: Time offset
-                    last_block_time_offset_us:
-                      type: integer
-                      description: Last block time offset
-                    max_scheduled_transaction_time_per_block_ms:
-                      type: integer
-                      description: Max scheduled transaction time per block in ms
-                    subjective_cpu_leeway_us:
-                      type: integer
-                      description: Subjective CPU leeway
-                    incoming_defer_ratio:
-                      type: integer
-                      description: Incoming defer ration
-
-      responses:
-        "200":
-          description: OK
+                  accounts:
+                    type: array
+                    description: Array of account names stored in the greylist
+                    items:
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+        "400":
+          description: client error
           content:
             application/json:
               schema:
-                description: Returns Nothing
-  /producer/get_greylist:
-    post:
-      summary: get_greylist
-      description: Retreives the greylist for producer node
-      operationId: get_greylist
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                type: array
-                description: List of account names stored in the greylist
-                items:
-                  $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                $ref: '#/components/schemas/Error'
 
   /producer/add_greylist_accounts:
     post:
       summary: add_greylist_accounts
-      description: Adds accounts to grey list for producer node
+      description: Adds accounts to greylist for producer node. At least one account is required.
       operationId: add_greylist_accounts
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - params
               properties:
-                params:
-                  type: object
-                  properties:
-                    accounts:
-                      type: array
-                      description: List of account names to add
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-
+                  accounts:
+                    type: array
+                    description: List of account names to add
+                    items:
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/remove_greylist_accounts:
     post:
       summary: remove_greylist_accounts
-      description: Removes accounts from greylist for producer node
+      description: Removes accounts from greylist for producer node. At least one account is required.
       operationId: remove_greylist_accounts
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - params
               properties:
-                params:
-                  type: object
-                  properties:
-                    accounts:
-                      type: array
-                      description: List of account names to remove
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-
+                accounts:
+                  type: array
+                  description: List of account names to remove
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                type: array
-                description: List of account names stored in the greylist
-                items:
-                  $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/get_whitelist_blacklist:
     post:
       summary: get_whitelist_blacklist
-      description: Retreives the white list and black list for producer node
+      description: Retrieves the whitelist and blacklist for producer node. A JSON object containing whitelist and blacklist information.
       operationId: get_whitelist_blacklist
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
-
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
                 type: object
-                description: Defines the actor whitelist and blacklist, the contract whitelist and blacklist, the action blacklist and key blacklist
                 properties:
                   actor_whitelist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
                   actor_blacklist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
                   contract_whitelist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
                   contract_blacklist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
                   action_blacklist:
                     type: array
                     items:
@@ -307,116 +247,118 @@ paths:
                       description: Array of two string values, the account name as the first and action name as the second
                       items:
                         allOf:
-                          - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                          - $ref: "https://eosio.github.io/schemata/v2.0/oas/CppSignature.yaml"
+                          - $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                          - $ref: "https://docs.eosnetwork.com/openapi/v2.0/CppSignature.yaml"
                   key_blacklist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/KeyType.yaml"
+                      - $ref: "https://docs.eosnetwork.com/openapi/v2.0/KeyType.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/set_whitelist_blacklist:
     post:
       summary: set_whitelist_blacklist
-      description: Sets the white list and black list for producer node
+      description: Defines the whitelist and blacklist for a producer node. Takes a JSON object containing whitelist and blacklist information.
       operationId: set_whitelist_blacklist
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - params
               properties:
-                params:
-                  type: object
-                  description: Defines the actor whitelist and blacklist, the contract whitelist and blacklist, the action blacklist and key blacklist
-                  properties:
-                    actor_whitelist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                    actor_blacklist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                    contract_whitelist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                    contract_blacklist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                    action_blacklist:
-                      type: array
-                      items:
-                        type: array
-                        description: Array of two string values, the account name as the first and action name as the second
-                        items:
-                          allOf:
-                            - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                            - $ref: "https://eosio.github.io/schemata/v2.0/oas/CppSignature.yaml"
-                    key_blacklist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/KeyType.yaml"
-
+                actor_whitelist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                actor_blacklist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                contract_whitelist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                contract_blacklist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                action_blacklist:
+                  type: array
+                  items:
+                    type: array
+                    description: Array of two string values, the account name as the first and action name as the second
+                    items:
+                      anyOf:
+                        - $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                        - $ref: "https://docs.eosnetwork.com/openapi/v2.0/CppSignature.yaml"
+                key_blacklist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/KeyType.yaml"
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/create_snapshot:
     post:
       summary: create_snapshot
-      description: Creates a snapshot for producer node
+      description: Creates a snapshot for producer node. Returns error when unable to create a snapshot.
       operationId: create_snapshot
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              required:
-                - next
-              properties:
-                next:
-                  type: object
-                  description: Defines the snapshot to be created
-                  properties:
-                    snapshot_name:
-                      type: string
-                      description: The name of the snapshot
-                    head_block_id:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml"
-
-
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                type: object
+                properties:
+                  head_block_id:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                  head_block_num:
+                    type: integer
+                    description: Highest block number on the chain
+                    example: 5102
+                  head_block_time:
+                    type: string
+                    description: Highest block unix timestamp
+                    example: 2020-11-16T00:00:00.000
+                  version:
+                    type: integer
+                    description: version number
+                    example: 6
+                  snapshot_name:
+                    type: string
+                    description: The path and file name of the snapshot
+                    example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/get_integrity_hash:
     post:
       summary: get_integrity_hash
-      description: Retreives the integrity hash for producer node
+      description: Retrieves the integrity hash for producer node
       operationId: get_integrity_hash
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
-
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
@@ -424,69 +366,67 @@ paths:
                 type: object
                 description: Defines the integrity hash information details
                 properties:
-                  integrity_hash:
-                    $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml"
                   head_block_id:
-                    $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml"
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                  integrity_hash:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/schedule_protocol_feature_activations:
     post:
       summary: schedule_protocol_feature_activations
-      description: Schedule protocol feature activation for producer node
+      description: Schedule protocol feature activation for producer node. Note some features may require pre-activation. Will return error for duplicate requests or when feature required pre-activation.
       operationId: schedule_protocol_feature_activations
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - schedule
               properties:
-                schedule:
-                  type: object
-                  properties:
-                    protocol_features_to_activate:
-                      type: array
-                      description: List of protocol features to activate
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml"
-
+                protocol_features_to_activate:
+                  type: array
+                  description: List of protocol features to activate
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/get_supported_protocol_features:
     post:
       summary: get_supported_protocol_features
-      description: Retreives supported protocol features for producer node
+      description: Retrieves supported protocol features for producer node. Pass filters in as part of the request body.
       operationId: get_supported_protocol_features
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - params
               properties:
-                params:
-                  type: object
-                  description: Defines filters based on which to return the supported protocol features
-                  properties:
-                    exclude_disabled:
-                      type: boolean
-                      description: Exclude disabled protocol features
-                    exclude_unactivatable:
-                      type: boolean
-                      description: Exclude unactivatable protocol features
-
+                exclude_disabled:
+                  type: boolean
+                  description: Exclude disabled protocol features
+                exclude_unactivatable:
+                  type: boolean
+                  description: Exclude unactivatable protocol features
+                  example: false
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
@@ -494,4 +434,196 @@ paths:
                 type: array
                 description: Variant type, an array of strings with the supported protocol features
                 items:
-                  type: string
+                  type: object
+                  properties:
+                    feature_digest:
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                    subjective_restrictions:
+                      type: object
+                      properties:
+                        enabled:
+                          type: boolean
+                          example: true
+                        preactivation_required:
+                          type: boolean
+                          example: true
+                        earliest_allowed_activation_time:
+                          type: string
+                          example: "1970-01-01T00:00:00.000"
+                        description_digest:
+                          $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                        dependencies:
+                          type: array
+                          items:
+                            $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                        protocol_feature_type:
+                          type: string
+                          example: "builtin"
+                        specification:
+                          type: array
+                          items:
+                            type: object
+                            properties:
+                              name:
+                                type: string
+                              value:
+                                type: string
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
+
+  /producer/get_account_ram_corrections:
+    post:
+      summary: get_account_ram_corrections
+      description: Retrieves accounts with ram corrections.
+      operationId: get_account_ram_corrections
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                lower_bound:
+                  type: integer
+                  description: lowest account key
+                upper_bound:
+                  type: integer
+                  description: highest account key
+                limit:
+                  type: integer
+                  description: number of rows to scan
+                  example: 10
+                reverse:
+                  type: boolean
+                  description: direction of search
+                  example: false
+      responses:
+        "201":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                  - rows
+                properties:
+                  rows:
+                    type: array
+                    items:
+                      type: string
+                  more:
+                    type: array
+                    items:
+                      - $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
+components:
+  securitySchemes: {}
+  schemas:
+    Error:
+      type: object
+      properties:
+        code:
+          type: integer
+          description: http return code
+          example: 400
+        message:
+          type: string
+          description: summary of error
+          example: Invalid Request
+        error:
+          type: object
+          description: details on the error
+          properties:
+            code:
+              type: integer
+              description: internal error code
+              example: 3200006
+            name:
+              type: string
+              description: name of error
+              example: invalid_http_request
+            what:
+              type: string
+              description: prettier version of error name
+              example: invalid http request
+            details:
+              type: array
+              description: list of additional information for debugging
+              items:
+                type: object
+                properties:
+                  message:
+                    type: string
+                    description: debugging message
+                    example: Unable to parse valid input from POST body
+                  file:
+                    type: string
+                    description: file where error was thrown
+                    example: http_plugin.hpp
+                  line_number:
+                    type: integer
+                    description: line number in file where error was thrown
+                    example: 246
+                  method:
+                    type: string
+                    description: function executed when error occurred
+                    example: parse_params
+    OK:
+      type: object
+      properties:
+        result:
+          type: string
+          description: status
+          example: ok
+    OK_PAUSED:
+      type: object
+      properties:
+        json:
+          type: boolean
+          description: true/false indicating paused state
+          example: true
+    Runtime_Options:
+      type: object
+      properties:
+        max_transaction_time:
+          type: integer
+          description: Max transaction time
+          example: 100
+        max_irreversible_block_age:
+          type: integer
+          description: Max irreversible block age
+          example: -1
+        produce_time_offset_us:
+          type: integer
+          description: Time offset
+          example: -100000
+        last_block_time_offset_us:
+          type: integer
+          description: Last block time offset
+          example: -200000
+        max_scheduled_transaction_time_per_block_ms:
+          type: integer
+          description: Max scheduled transaction time per block in ms
+          example: 100
+        subjective_cpu_leeway_us:
+          type: integer
+          description: in micro seconds
+          example: 10
+        incoming_defer_ratio:
+          type: string
+          description: Incoming defer ratio, parsed to double
+          example: "1.00000000000000000"
+        greylist_limit:
+          type: integer
+          description: limit on number of Names supported by greylist
+          example: 1000

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -114,7 +114,10 @@ public:
 
    get_account_ram_corrections_result  get_account_ram_corrections( const get_account_ram_corrections_params& params ) const;
 
-    void log_failed_transaction(const transaction_id_type& trx_id, const chain::packed_transaction_ptr& packed_trx_ptr, const char* reason) const;
+   void log_failed_transaction(const transaction_id_type& trx_id, const chain::packed_transaction_ptr& packed_trx_ptr, const char* reason) const;
+
+   // thread-safe, called when a new block is received
+   void received_block();
 
  private:
    std::shared_ptr<class producer_plugin_impl> my;

plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp

@@ -180,7 +180,8 @@ public:
       fc_dlog( log, "Subjective billed accounts ${n} removed ${r}", ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
    }
 
-   bool remove_expired( fc::logger& log, const fc::time_point& pending_block_time, const fc::time_point& now, const fc::time_point& deadline ) {
+   template <typename Yield>
+   bool remove_expired( fc::logger& log, const fc::time_point& pending_block_time, const fc::time_point& now, Yield&& yield ) {
       bool exhausted = false;
       auto& idx = _trx_cache_index.get<by_expiry>();
       if( !idx.empty() ) {
@@ -189,7 +190,7 @@ public:
          uint32_t num_expired = 0;
 
          while( !idx.empty() ) {
-            if( deadline <= fc::time_point::now() ) {
+            if( yield() ) {
                exhausted = true;
                break;
             }

plugins/producer_plugin/producer_plugin.cpp

@@ -41,6 +41,13 @@ using std::vector;
 using std::deque;
 using boost::signals2::scoped_connection;
 
+// in release/3.2+ move to fc logger.hpp
+#define fc_tlog( LOGGER, FORMAT, ... ) \
+  FC_MULTILINE_MACRO_BEGIN \
+   if( (LOGGER).is_enabled( fc::log_level::all ) ) \
+      (LOGGER).log( FC_LOG_MESSAGE( all, FORMAT, __VA_ARGS__ ) ); \
+  FC_MULTILINE_MACRO_END
+
 #undef FC_LOG_AND_DROP
 #define LOG_AND_DROP()  \
    catch ( const guard_exception& e ) { \
@@ -197,8 +204,9 @@ public:
    {
    }
 
-   void set_max_failures_per_account( uint32_t max_failures ) {
-       max_failures_per_account = max_failures;
+   void set_max_failures_per_account( uint32_t max_failures, uint32_t size ) {
+      max_failures_per_account = max_failures;
+      reset_window_size_in_num_blocks = size;
    }
 
    void add( const account_name& n, int64_t exception_code ) {
@@ -218,10 +226,24 @@ public:
       return false;
    }
 
-   void report() const {
-      if( _log.is_enabled( fc::log_level::debug ) ) {
+   void report_and_clear(uint32_t block_num) {
+      if (last_reset_block_num != block_num && (block_num % reset_window_size_in_num_blocks == 0) ) {
+         report(block_num);
+         failed_accounts.clear();
+         last_reset_block_num = block_num;
+      }
+   }
+
+   fc::time_point next_reset_timepoint(uint32_t current_block_num, fc::time_point current_block_time) const {
+      auto num_blocks_to_reset = reset_window_size_in_num_blocks - (current_block_num % reset_window_size_in_num_blocks);
+      return current_block_time + fc::milliseconds(num_blocks_to_reset * eosio::chain::config::block_interval_ms);
+   }
+
+private:
+   void report(uint32_t block_num) const {
+      if( _log.is_enabled(fc::log_level::debug)) {
          auto now = fc::time_point::now();
-         for( const auto& e : failed_accounts ) {
+         for ( const auto& e : failed_accounts ) {
             std::string reason;
             if( e.second.is_deadline() ) reason += "deadline";
             if( e.second.is_tx_cpu_usage() ) {
@@ -242,12 +264,6 @@ public:
          }
       }
    }
-
-   void clear() {
-      failed_accounts.clear();
-   }
-
-private:
    struct account_failure {
       enum class ex_fields : uint8_t {
          ex_deadline_exception = 1,
@@ -282,6 +298,8 @@ private:
 
    std::map<account_name, account_failure> failed_accounts;
    uint32_t max_failures_per_account = 3;
+   uint32_t last_reset_block_num = 0;
+   uint32_t reset_window_size_in_num_blocks = 1;
    const eosio::subjective_billing& subjective_billing;
 };
 
@@ -322,6 +340,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       std::optional<named_thread_pool>                          _thread_pool;
 
       std::atomic<int32_t>                                      _max_transaction_time_ms; // modified by app thread, read by net_plugin thread pool
+      std::atomic<bool>                                         _received_block{false}; // modified by net_plugin thread pool and app thread
       fc::microseconds                                          _max_irreversible_block_age_us;
       int32_t                                                   _produce_time_offset_us = 0;
       int32_t                                                   _last_block_time_offset_us = 0;
@@ -339,9 +358,6 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
       chain_plugin* chain_plug = nullptr;
 
-      incoming::channels::block::channel_type::handle         _incoming_block_subscription;
-      incoming::channels::transaction::channel_type::handle   _incoming_transaction_subscription;
-
       compat::channels::transaction_ack::channel_type&        _transaction_ack_channel;
 
       incoming::methods::block_sync::method_type::handle        _incoming_block_sync_provider;
@@ -431,7 +447,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          _subjective_billing.abort_block();
       }
 
-      bool on_incoming_block(const signed_block_ptr& block, const std::optional<block_id_type>& block_id) {
+      bool on_incoming_block(const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
          auto& chain = chain_plug->chain();
          if ( _pending_block_mode == pending_block_mode::producing ) {
             fc_wlog( _log, "dropped incoming block #${num} id: ${id}",
@@ -439,6 +455,11 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
             return false;
          }
 
+         // start a new speculative block, speculative start_block may have been interrupted
+         auto ensure = fc::make_scoped_exit([this](){
+            schedule_production_loop();
+         });
+
          const auto& id = block_id ? *block_id : block->calculate_id();
          auto blk_num = block->block_num();
 
@@ -452,16 +473,14 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          if( existing ) { return false; }
 
          // start processing of block
-         auto bsf = chain.create_block_state_future( id, block );
+         std::future<block_state_ptr> bsf;
+         if( !bsp ) {
+            bsf = chain.create_block_state_future( id, block );
+         }
 
          // abort the pending block
          abort_block();
 
-         // exceptions throw out, make sure we restart our loop
-         auto ensure = fc::make_scoped_exit([this](){
-            schedule_production_loop();
-         });
-
          // push the new block
          auto handle_error = [&](const auto& e)
          {
@@ -471,7 +490,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          };
 
          try {
-            chain.push_block( bsf, [this]( const branch_type& forked_branch ) {
+            const block_state_ptr& bspr = bsp ? bsp : bsf.get();
+            chain.push_block( bspr, [this]( const branch_type& forked_branch ) {
                _unapplied_transactions.add_forked( forked_branch );
             }, [this]( const transaction_id_type& id ) {
                return _unapplied_transactions.get_trx( id );
@@ -673,8 +693,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
             auto first_auth = trx->packed_trx()->get_transaction().first_authorizer();
             if( _account_fails.failure_limit( first_auth ) ) {
                send_response( std::static_pointer_cast<fc::exception>( std::make_shared<tx_cpu_usage_exceeded>(
-                     FC_LOG_MESSAGE( error, "transaction ${id} exceeded failure limit for account ${a}",
-                                     ("id", trx->id())("a", first_auth) ) ) ) );
+                     FC_LOG_MESSAGE( error, "transaction ${id} exceeded failure limit for account ${a} until ${next_reset_time}",
+                                     ("id", trx->id())("a", first_auth)
+                                     ("next_reset_time", _account_fails.next_reset_timepoint(chain.head_block_num(),chain.head_block_time())) ) ) ) );
                return true;
             }
 
@@ -694,7 +715,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
             auto prev_billed_cpu_time_us = trx->billed_cpu_time_us;
             auto trace = chain.push_transaction( trx, block_deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );
-            fc_dlog( _trx_failed_trace_log, "Subjective bill for ${a}: ${b} elapsed ${t}us", ("a",first_auth)("b",sub_bill)("t",trace->elapsed));
+            fc_tlog( _log, "Subjective bill for ${a}: ${b} elapsed ${t}us", ("a",first_auth)("b",sub_bill)("t",trace->elapsed));
             if( trace->except ) {
                if( exception_is_exhausted( *trace->except ) ) {
                   _unapplied_transactions.add_incoming( trx, persist_until_expired, return_failure_traces, next );
@@ -786,6 +807,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          exhausted
       };
 
+      inline bool should_interrupt_start_block( const fc::time_point& deadline ) const;
       start_block_result start_block();
 
       fc::time_point calculate_pending_block_time() const;
@@ -876,7 +898,9 @@ void producer_plugin::set_program_options(
          ("subjective-cpu-leeway-us", boost::program_options::value<int32_t>()->default_value( config::default_subjective_cpu_leeway_us ),
           "Time in microseconds allowed for a transaction that starts with insufficient CPU quota to complete and cover its CPU usage.")
          ("subjective-account-max-failures", boost::program_options::value<uint32_t>()->default_value(3),
-          "Sets the maximum amount of failures that are allowed for a given account per block.")
+          "Sets the maximum amount of failures that are allowed for a given account per window size.")
+         ("subjective-account-max-failures-window-size", boost::program_options::value<uint32_t>()->default_value(1),
+          "Sets the window size in number of blocks for subjective-account-max-failures.")
          ("subjective-account-decay-time-minutes", bpo::value<uint32_t>()->default_value( config::account_cpu_usage_average_window_ms / 1000 / 60 ),
           "Sets the time to return full subjective cpu for accounts")
          ("incoming-defer-ratio", bpo::value<double>()->default_value(1.0),
@@ -1026,7 +1050,13 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
 
    my->_keosd_provider_timeout_us = fc::milliseconds(options.at("keosd-provider-timeout").as<int32_t>());
 
-   my->_account_fails.set_max_failures_per_account( options.at("subjective-account-max-failures").as<uint32_t>() );
+   auto subjective_account_max_failures_window_size = options.at("subjective-account-max-failures-window-size").as<uint32_t>();
+   EOS_ASSERT( subjective_account_max_failures_window_size > 0, plugin_config_exception,
+               "subjective-account-max-failures-window-size ${s} must be greater than 0", ("s", subjective_account_max_failures_window_size) );
+
+   my->_account_fails.set_max_failures_per_account( options.at("subjective-account-max-failures").as<uint32_t>(),
+                                                    subjective_account_max_failures_window_size );
+
 
    my->_produce_time_offset_us = options.at("produce-time-offset-us").as<int32_t>();
    EOS_ASSERT( my->_produce_time_offset_us <= 0 && my->_produce_time_offset_us >= -config::block_interval_us, plugin_config_exception,
@@ -1128,23 +1158,9 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
-   my->_incoming_block_subscription = app().get_channel<incoming::channels::block>().subscribe(
-         [this](const signed_block_ptr& block) {
-      try {
-         my->on_incoming_block(block, {});
-      } LOG_AND_DROP();
-   });
-
-   my->_incoming_transaction_subscription = app().get_channel<incoming::channels::transaction>().subscribe(
-         [this](const packed_transaction_ptr& trx) {
-      try {
-         my->on_incoming_transaction_async(trx, false, false, false, [](const auto&){});
-      } LOG_AND_DROP();
-   });
-
    my->_incoming_block_sync_provider = app().get_method<incoming::methods::block_sync>().register_provider(
-         [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id) {
-      return my->on_incoming_block(block, block_id);
+         [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
+      return my->on_incoming_block(block, block_id, bsp);
    });
 
    my->_incoming_transaction_async_provider = app().get_method<incoming::methods::transaction_async>().register_provider(
@@ -1242,6 +1258,7 @@ void producer_plugin::plugin_shutdown() {
    }
 
    app().post( 0, [me = my](){} ); // keep my pointer alive until queue is drained
+   fc_ilog(_log, "exit shutdown");
 }
 
 void producer_plugin::handle_sighup() {
@@ -1679,6 +1696,14 @@ fc::time_point producer_plugin_impl::calculate_block_deadline( const fc::time_po
    }
 }
 
+bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline ) const {
+   if( _pending_block_mode == pending_block_mode::producing ) {
+      return deadline <= fc::time_point::now();
+   }
+   // if we can produce then honor deadline so production starts on time
+   return (!_producers.empty() && deadline <= fc::time_point::now()) || _received_block;
+}
+
 producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    chain::controller& chain = chain_plug->chain();
 
@@ -1687,7 +1712,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
    const auto& hbs = chain.head_block_state();
 
-   if( chain.get_terminate_at_block() > 0 && chain.get_terminate_at_block() < chain.head_block_num() ) {
+   if( chain.get_terminate_at_block() > 0 && chain.get_terminate_at_block() <= chain.head_block_num() ) {
       ilog("Reached configured maximum block ${num}; terminating", ("num", chain.get_terminate_at_block()));
       app().quit();
       return start_block_result::failed;
@@ -1854,16 +1879,19 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
       }
 
       try {
+         _account_fails.report_and_clear(hbs->block_num);
+
          if( !remove_expired_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
          if( !remove_expired_blacklisted_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
-         if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(), preprocess_deadline ) )
+         if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(),
+                                                  [&](){ return should_interrupt_start_block( preprocess_deadline ); } ) ) {
             return start_block_result::exhausted;
+         }
 
          // limit execution of pending incoming to once per block
          size_t pending_incoming_process_limit = _unapplied_transactions.incoming_size();
-         _account_fails.clear();
 
          if( !process_unapplied_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
@@ -1882,7 +1910,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
          if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit()
             return start_block_result::failed;
-         if (preprocess_deadline <= fc::time_point::now() || block_is_exhausted()) {
+         if ( should_interrupt_start_block( preprocess_deadline ) || block_is_exhausted() ) {
             return start_block_result::exhausted;
          } else {
             if( !process_incoming_trxs( preprocess_deadline, pending_incoming_process_limit ) )
@@ -1913,7 +1941,7 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
    size_t num_expired_persistent = 0;
    size_t num_expired_other = 0;
    size_t orig_count = _unapplied_transactions.size();
-   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, deadline,
+   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block( deadline ); },
                   [chain_plug = chain_plug, &num_expired_persistent, &num_expired_other, pbm = _pending_block_mode,
                    &chain, has_producers = !_producers.empty()]( const packed_transaction_ptr& packed_trx_ptr, trx_enum_type trx_type ) {
             if( trx_type == trx_enum_type::persisted ) {
@@ -1955,7 +1983,7 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
             }
    });
 
-   if( exhausted ) {
+   if( exhausted && _pending_block_mode == pending_block_mode::producing ) {
       fc_wlog( _log, "Unable to process all expired transactions in unapplied queue before deadline, "
                      "Persistent expired ${persistent_expired}, Other expired ${other_expired}",
                ("persistent_expired", num_expired_persistent)("other_expired", num_expired_other) );
@@ -1981,7 +2009,7 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point
       int orig_count = _blacklisted_transactions.size();
 
       while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= lib_time) {
-         if (deadline <= fc::time_point::now()) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2009,7 +2037,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
       auto end_itr = (_pending_block_mode == pending_block_mode::producing) ?
                      _unapplied_transactions.unapplied_end()   : _unapplied_transactions.persisted_end();
       while( itr != end_itr ) {
-         if( deadline <= fc::time_point::now() ) {
+         if( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2024,8 +2052,9 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
                ++num_failed;
                if( itr->next ) {
                   itr->next( std::make_shared<tx_cpu_usage_exceeded>(
-                        FC_LOG_MESSAGE( error, "transaction ${id} exceeded failure limit for account ${a}",
-                                        ("id", trx->id())("a", first_auth) ) ) );
+                        FC_LOG_MESSAGE( error, "transaction ${id} exceeded failure limit for account ${a} until ${next_reset_time}",
+                                        ("id", trx->id())("a", first_auth)
+                                        ("next_reset_time", _account_fails.next_reset_timepoint(chain.head_block_num(),chain.head_block_time())) ) ) );
                }
                itr = _unapplied_transactions.erase( itr );
                continue;
@@ -2048,7 +2077,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
                || trx->read_only;
 
             auto trace = chain.push_transaction( trx, deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );
-            fc_dlog( _trx_failed_trace_log, "Subjective unapplied bill for ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
+            fc_tlog( _log, "Subjective unapplied bill for ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
             if( trace->except ) {
                if( exception_is_exhausted( *trace->except ) ) {
                   if( block_is_exhausted() ) {
@@ -2057,7 +2086,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
                      break;
                   }
                } else {
-                  fc_dlog( _trx_failed_trace_log, "Subjective unapplied bill for failed ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
+                  fc_tlog( _log, "Subjective unapplied bill for failed ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
                   auto failure_code = trace->except->code();
                   if( failure_code != tx_duplicate::code_value ) {
                      // this failed our configured maximum transaction time, we don't want to replay it
@@ -2076,7 +2105,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
                   continue;
                }
             } else {
-               fc_dlog( _trx_successful_trace_log, "Subjective unapplied bill for success ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
+               fc_tlog( _log, "Subjective unapplied bill for success ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
                // if db_read_mode SPECULATIVE then trx is in the pending block and not immediately reverted
                if (!disable_subjective_billing)
                   _subjective_billing.subjective_bill( trx->id(), trx->packed_trx()->expiration(), first_auth, trace->elapsed,
@@ -2210,7 +2239,7 @@ bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline
       auto itr = _unapplied_transactions.incoming_begin();
       auto end = _unapplied_transactions.incoming_end();
       while( pending_incoming_process_limit && itr != end ) {
-         if (deadline <= fc::time_point::now()) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2250,6 +2279,7 @@ bool producer_plugin_impl::block_is_exhausted() const {
 // -> Idle
 // --> Start block B (block time y.000) at time x.500
 void producer_plugin_impl::schedule_production_loop() {
+   _received_block = false;
    _timer.cancel();
 
    auto result = start_block();
@@ -2437,9 +2467,6 @@ void producer_plugin_impl::produce_block() {
 
    block_state_ptr new_bs = chain.head_block_state();
 
-   _account_fails.report();
-   _account_fails.clear();
-
    ilog("Produced block ${id}... #${n} @ ${t} signed by ${p} [trxs: ${count}, lib: ${lib}, confirmed: ${confs}]",
         ("p",new_bs->header.producer)("id",new_bs->id.str().substr(8,16))
         ("n",new_bs->block_num)("t",new_bs->header.timestamp)
@@ -2447,6 +2474,10 @@ void producer_plugin_impl::produce_block() {
 
 }
 
+void producer_plugin::received_block() {
+   my->_received_block = true;
+}
+
 void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id, const packed_transaction_ptr& packed_trx_ptr, const char* reason) const {
    fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Speculative execution is REJECTING tx: ${txid} : ${why}",
             ("txid", trx_id)("why", reason));

plugins/producer_plugin/test/test_subjective_billing.cpp

@@ -50,7 +50,7 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) {
       BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) );
 
       // expires transactions but leaves them in the decay at full value
-      sub_bill.remove_expired( log, now + fc::microseconds(1), now, fc::time_point::maximum() );
+      sub_bill.remove_expired( log, now + fc::microseconds(1), now, [](){ return false; } );
 
       BOOST_CHECK_EQUAL( 13+11, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) );
@@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) {
       BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) );
 
-      sub_bill.remove_expired( log, now, now, fc::time_point::maximum() );
+      sub_bill.remove_expired( log, now, now, [](){ return false; } );
       BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) );
 
@@ -148,7 +148,7 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) {
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(a, endtime) );
       BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(b, endtime) );
 
-      sub_bill.remove_expired( log, now + fc::microseconds(1), now, fc::time_point::maximum() );
+      sub_bill.remove_expired( log, now + fc::microseconds(1), now, [](){ return false; } );
       BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) );
 

plugins/resource_monitor_plugin/include/eosio/resource_monitor_plugin/file_space_handler.hpp

@@ -59,7 +59,8 @@ namespace eosio::resource_monitor {
 
             if ( info.available < fs.shutdown_available ) {
                if (output_threshold_warning) {
-                  wlog("Space usage warning: ${path}'s file system exceeded threshold ${threshold}%, available: ${available}, Capacity: ${capacity}, shutdown_available: ${shutdown_available}", ("path", fs.path_name.string()) ("threshold", shutdown_threshold) ("available", info.available) ("capacity", info.capacity) ("shutdown_available", fs.shutdown_available));
+                  elog("Space usage warning: ${path}'s file system exceeded threshold ${threshold}%, available: ${available}, Capacity: ${capacity}, shutdown_available: ${shutdown_available}",
+                       ("path", fs.path_name.string())("threshold", shutdown_threshold)("available", info.available)("capacity", info.capacity)("shutdown_available", fs.shutdown_available));
                }
                return true;
             } else if ( info.available < fs.warning_available && output_threshold_warning ) {
@@ -114,7 +115,7 @@ namespace eosio::resource_monitor {
    
    void space_monitor_loop() {
       if ( is_threshold_exceeded() && shutdown_on_exceeded ) {
-         wlog("Shutting down");
+         elog("Shutting down, file system exceeded threshold");
          appbase::app().quit(); // This will gracefully stop Nodeos
          return;
       }

plugins/test_control_api_plugin/test_control.swagger.yaml

@@ -1,12 +1,13 @@
 openapi: 3.0.0
 info:
   title: Test Control API
+  description: Nodeos Test Control API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eos.io
+    url: https://eosnetwork.com
 tags:
   - name: eosio
 servers:

plugins/test_control_plugin/test_control_plugin.cpp

@@ -1,5 +1,4 @@
 #include <eosio/test_control_plugin/test_control_plugin.hpp>
-#include <atomic>
 
 namespace fc { class variant; }
 
@@ -9,7 +8,7 @@ static appbase::abstract_plugin& _test_control_plugin = app().register_plugin<te
 
 class test_control_plugin_impl {
 public:
-   test_control_plugin_impl(chain::controller& c) : _chain(c) {}
+   explicit test_control_plugin_impl(chain::controller& c) : _chain(c) {}
    void connect();
    void disconnect();
    void kill_on_lib(account_name prod, uint32_t where_in_seq);
@@ -24,12 +23,11 @@ private:
    std::optional<boost::signals2::scoped_connection> _irreversible_block_connection;
    chain::controller&  _chain;
    account_name        _producer;
-   int32_t             _where_in_sequence{-1};
-   int32_t             _producer_sequence{-1};
-   uint32_t            _first_sequence_timeslot{0};
+   uint32_t            _where_in_sequence{};
    bool                _clean_producer_sequence{false};
-   std::atomic_bool    _track_lib;
-   std::atomic_bool    _track_head;
+   bool                _started_production_round{false};
+   bool                _track_lib{false};
+   bool                _track_head{false};
 };
 
 void test_control_plugin_impl::connect() {
@@ -59,68 +57,54 @@ void test_control_plugin_impl::accepted_block(const chain::block_state_ptr& bsp)
 }
 
 void test_control_plugin_impl::process_next_block_state(const chain::block_state_ptr& bsp) {
+   // Tests expect the shutdown only after signaling a producer shutdown and seeing a full production cycle
    const auto block_time = _chain.head_block_time() + fc::microseconds(chain::config::block_interval_us);
    const auto& producer_authority = bsp->get_scheduled_producer(block_time);
    const auto producer_name = producer_authority.producer_name;
-   if (_producer != account_name())
-      ilog("producer ${cprod}, looking for ${lprod}", ("cprod", producer_name.to_string())("lprod", _producer.to_string()));
-
-   // start counting sequences for this producer (once we have a sequence that we saw the initial block for that producer)
-   if (producer_name == _producer && _clean_producer_sequence) {
-      auto slot = bsp->block->timestamp.slot;
-      _producer_sequence += 1;
-      ilog("producer ${prod} seq: ${seq} slot: ${slot}", 
-           ("prod", producer_name.to_string())
-           ("seq", _producer_sequence+1) // _producer_sequence is index, aligning it with slot number
-           ("slot", slot - _first_sequence_timeslot));
-
-      bool last_slot = false;
-      if (_where_in_sequence + 1 == chain::config::producer_repetitions){
-         auto last_slot_time = _first_sequence_timeslot + chain::config::producer_repetitions;
-         last_slot = slot == last_slot_time;
+   const auto slot = bsp->block->timestamp.slot % chain::config::producer_repetitions;
+   if (_producer != account_name()) {
+      if( _producer != producer_name ) _clean_producer_sequence = true;
+      if( _clean_producer_sequence ) {
+         ilog( "producer ${cprod} slot ${pslot}, looking for ${lprod} slot ${slot}",
+               ("cprod", producer_name)("pslot", slot)("lprod", _producer)("slot", _where_in_sequence) );
+      } else {
+         ilog( "producer ${cprod} slot ${pslot}, looking for start of ${lprod} production round",
+               ("cprod", producer_name)("pslot", slot)("lprod", _producer) );
       }
+   }
 
-      if (_producer_sequence >= _where_in_sequence || last_slot) {
-         int32_t slot_index = slot - _first_sequence_timeslot;
-         if (last_slot && slot_index > _producer_sequence + 1){
-            wlog("Producer produced less than ${n} blocks, ${l}th block is last in sequence. Likely performance issue, check timing",
-                 ("n", chain::config::producer_repetitions)("l", _producer_sequence + 1));
-         }
+   // check started_production_round in case where producer does not produce a full round, still want to shut down
+   if( _clean_producer_sequence && (producer_name == _producer || _started_production_round) ) {
+      _started_production_round = true;
+      const auto current_slot = chain::block_timestamp_type( block_time ).slot % chain::config::producer_repetitions;
+      ilog( "producer ${prod} slot: ${slot}", ("prod", producer_name)("slot", slot) );
+
+      if( current_slot >= _where_in_sequence || producer_name != _producer ) {
          ilog("shutting down");
          app().quit();
       }
-   } else if (producer_name != _producer) {
-      if (_producer_sequence != -1)
-         ilog("producer changed, restarting");
-      _producer_sequence = -1;
-      // can now guarantee we are at the start of the producer
-      _clean_producer_sequence = true;
-
-      _first_sequence_timeslot = bsp->block->timestamp.slot;
    }
 }
 
 void test_control_plugin_impl::kill_on_lib(account_name prod, uint32_t where_in_seq) {
    _track_head = false;
    _producer = prod;
-   _where_in_sequence = static_cast<uint32_t>(where_in_seq);
-   _producer_sequence = -1;
+   _where_in_sequence = where_in_seq;
    _clean_producer_sequence = false;
+   _started_production_round = false;
    _track_lib = true;
 }
 
 void test_control_plugin_impl::kill_on_head(account_name prod, uint32_t where_in_seq) {
    _track_lib = false;
    _producer = prod;
-   _where_in_sequence = static_cast<uint32_t>(where_in_seq);
-   _producer_sequence = -1;
+   _where_in_sequence = where_in_seq;
    _clean_producer_sequence = false;
+   _started_production_round = false;
    _track_head = true;
 }
 
-test_control_plugin::test_control_plugin()
-{
-}
+test_control_plugin::test_control_plugin() = default;
 
 void test_control_plugin::set_program_options(options_description& cli, options_description& cfg) {
 }

plugins/trace_api_plugin/store_provider.cpp

@@ -307,8 +307,11 @@ namespace eosio::trace_api {
                _maintenance_condition.wait(lock);
             }
 
+            if (_maintenance_shutdown) {
+               break;
+            }
+
             uint32_t best_known_lib = _best_known_lib;
-            bool shutdown = _maintenance_shutdown;
             lock.unlock();
 
             log(std::string("Waking up to handle lib: ") + std::to_string(best_known_lib));
@@ -319,10 +322,6 @@ namespace eosio::trace_api {
                   last_lib = best_known_lib;
                } FC_LOG_AND_DROP();
             }
-
-            if (shutdown) {
-               break;
-            }
          }
       });
    }

plugins/trace_api_plugin/trace_api.swagger.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.0
 info:
   title: Trace API
-  description: See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
+  description: Nodeos Trace API Specification. See developer documentation at https://docs.eosnetwork.com for information on enabling this plugin.
   version: 1.0.0
   license:
     name: MIT

plugins/trace_api_plugin/trace_api_plugin.cpp

@@ -455,6 +455,7 @@ void trace_api_plugin::plugin_startup() {
 void trace_api_plugin::plugin_shutdown() {
    my->plugin_shutdown();
    rpc->plugin_shutdown();
+   fc_ilog( _log, "exit shutdown");
 }
 
 void trace_api_plugin::handle_sighup() {

programs/keosd/main.cpp

@@ -71,6 +71,17 @@ bfs::path determine_home_directory()
    return home;
 }
 
+enum return_codes {
+   OTHER_FAIL        = -2,
+   INITIALIZE_FAIL   = -1,
+   SUCCESS           = 0,
+   BAD_ALLOC         = 1,
+   DATABASE_DIRTY    = 2,
+   FIXED_REVERSIBLE  = SUCCESS,
+   EXTRACTED_GENESIS = SUCCESS,
+   NODE_MANAGEMENT_SUCCESS = 5
+};
+
 int main(int argc, char** argv)
 {
    try {
@@ -90,6 +101,7 @@ int main(int argc, char** argv)
              opts.count("print-default-config")) {
             return 0;
          }
+         return INITIALIZE_FAIL;
       }
       initialize_logging();
       auto& http = app().get_plugin<http_plugin>();

scripts/pinned_build.sh

@@ -132,7 +132,7 @@ pushdir ${LEAP_DIR}
 
 # build Leap
 echo "Building Leap ${SCRIPT_DIR}"
-try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=${LLVM_DIR}/lib/cmake -DCMAKE_PREFIX_PATH=${BOOST_DIR}/bin ${SCRIPT_DIR}/..
+try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=${LEAP_PINNED_INSTALL_PREFIX:-/usr/local} -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=${LLVM_DIR}/lib/cmake -DCMAKE_PREFIX_PATH=${BOOST_DIR}/bin ${SCRIPT_DIR}/..
 
 try make -j${JOBS}
 try cpack

tests/CMakeLists.txt

@@ -120,11 +120,11 @@ add_test(NAME restart-scenarios-test-hard_replay COMMAND tests/restart-scenarios
 set_property(TEST restart-scenarios-test-hard_replay PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME restart-scenarios-test-none COMMAND tests/restart-scenarios-test.py -c none --kill-sig term -p4 -v --clean-run --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST restart-scenarios-test-none PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME terminate-scenarios-test-resync COMMAND tests/terminate-scenarios-test.py -p 1 -c resync --kill-count 1 --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME terminate-scenarios-test-resync COMMAND tests/terminate-scenarios-test.py -c resync --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST terminate-scenarios-test-resync PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME terminate-scenarios-test-replay COMMAND tests/terminate-scenarios-test.py -p 1 -c replay --kill-count 1 --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME terminate-scenarios-test-replay COMMAND tests/terminate-scenarios-test.py -c replay --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST terminate-scenarios-test-replay PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME terminate-scenarios-test-hard_replay COMMAND tests/terminate-scenarios-test.py -p 1 -c hardReplay --kill-count 1 --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME terminate-scenarios-test-hard_replay COMMAND tests/terminate-scenarios-test.py -c hardReplay --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST terminate-scenarios-test-hard_replay PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST validate_dirty_db_test PROPERTY LABELS nonparallelizable_tests)

tests/Cluster.py

@@ -1430,14 +1430,14 @@ class Cluster(object):
         time.sleep(1) # Give processes time to stand down
         return True
 
-    def relaunchEosInstances(self, cachePopen=False, nodeArgs=""):
+    def relaunchEosInstances(self, cachePopen=False, nodeArgs="", waitForTerm=False):
 
         chainArg=self.__chainSyncStrategy.arg + " " + nodeArgs
 
         newChain= False if self.__chainSyncStrategy.name in [Utils.SyncHardReplayTag, Utils.SyncNoneTag] else True
         for i in range(0, len(self.nodes)):
             node=self.nodes[i]
-            if node.killed and not node.relaunch(chainArg, newChain=newChain, cachePopen=cachePopen):
+            if node.killed and not node.relaunch(chainArg, newChain=newChain, cachePopen=cachePopen, waitForTerm=waitForTerm):
                 return False
 
         return True

tests/Node.py

@@ -1210,7 +1210,7 @@ class Node(object):
 
     # pylint: disable=too-many-locals
     # If nodeosPath is equal to None, it will use the existing nodeos path
-    def relaunch(self, chainArg=None, newChain=False, skipGenesis=True, timeout=Utils.systemWaitTimeout, addSwapFlags=None, cachePopen=False, nodeosPath=None, waitForTerm=True):
+    def relaunch(self, chainArg=None, newChain=False, skipGenesis=True, timeout=Utils.systemWaitTimeout, addSwapFlags=None, cachePopen=False, nodeosPath=None, waitForTerm=False):
 
         assert(self.pid is None)
         assert(self.killed)
@@ -1265,15 +1265,16 @@ class Node(object):
             except subprocess.TimeoutExpired:
                 return False
             with open(popen.errfile.name, 'r') as f:
-                if "Reached configured maximum block 10; terminating" in f.read():
+                if "successfully exiting" in f.read():
                     return True
                 else:
                     return False
 
-        if "terminate-at-block" not in cmd or not waitForTerm:
-            isAlive=Utils.waitForBool(isNodeAlive, timeout, sleepTime=1)
-        else:
+        if waitForTerm:
             isAlive=Utils.waitForBoolWithArg(didNodeExitGracefully, self.popenProc, timeout, sleepTime=1)
+        else:
+            isAlive=Utils.waitForBool(isNodeAlive, timeout, sleepTime=1)
+
         if isAlive:
             Utils.Print("Node relaunch was successful.")
         else:

tests/nodeos_forked_chain_test.py

@@ -235,7 +235,7 @@ try:
         transferAmount="100000000.0000 {0}".format(CORE_SYMBOL)
         Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, account.name))
         node.transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer", waitForTransBlock=True)
-        trans=node.delegatebw(account, 20000000.0000, 20000000.0000, waitForTransBlock=True, exitOnError=True)
+        trans=node.delegatebw(account, 20000000.0000, 20000000.0000, waitForTransBlock=False, exitOnError=True)
 
 
     # ***   vote using accounts   ***
@@ -245,7 +245,7 @@ try:
     index=0
     for account in accounts:
         Print("Vote for producers=%s" % (producers))
-        trans=prodNodes[index % len(prodNodes)].vote(account, producers, waitForTransBlock=True)
+        trans=prodNodes[index % len(prodNodes)].vote(account, producers, waitForTransBlock=False)
         index+=1
 
 
@@ -372,6 +372,11 @@ try:
     # block number to start expecting node killed after
     preKillBlockNum=nonProdNode.getBlockNum()
     preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum)
+    if preKillBlockProducer == "defproducerj" or preKillBlockProducer == "defproducerk":
+        # wait for defproduceri so there is plenty of time to send kill before defproducerk
+        nonProdNode.waitForProducer("defproduceri")
+        preKillBlockNum=nonProdNode.getBlockNum()
+        preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum)
     Print("preKillBlockProducer = {}".format(preKillBlockProducer))
     # kill at last block before defproducerl, since the block it is killed on will get propagated
     killAtProducer="defproducerk"

tests/nodeos_read_terminate_at_block_test.py

@@ -25,8 +25,6 @@ relaunchTimeout = 10
 numOfProducers = 1
 totalNodes = 4
 
-termAtFutureBlockNum = 30
-
 # Parse command line arguments
 args = TestHelper.parse_args({
     "-v",
@@ -47,7 +45,7 @@ keepLogs = args.keep_logs
 # Wrapper function to execute test
 # This wrapper function will resurrect the node to be tested, and shut
 # it down by the end of the test
-def executeTest(cluster, producerNode, testNodeId, testNodeArgs, resultMsgs):
+def executeTest(cluster, testNodeId, testNodeArgs, resultMsgs):
     testNode = None
     testResult = False
     resultDesc = "!!!BUG IS CONFIRMED ON TEST CASE #{} ({})".format(
@@ -63,17 +61,12 @@ def executeTest(cluster, producerNode, testNodeId, testNodeArgs, resultMsgs):
             )
         )
 
-        # Get current information from the producer.
-        producerNode.waitForHeadToAdvance()
-
-        # Launch the node with the terminate-at-block option specified
-        # in testNodeArgs. The option for each node has already
-        # been set via specificExtraNodeosArg in the cluster launch.
-        cluster.launchUnstarted(cachePopen=True)
         testNode = cluster.getNode(testNodeId)
+        assert not testNode.verifyAlive() # resets pid so reluanch works
+        testNode.relaunch(addSwapFlags={"--terminate-at-block": "9999999"}, cachePopen=True)
 
         # Wait for node to start up.
-        time.sleep(10)
+        time.sleep(3)
 
         # Check the node stops at the correct block.
         checkStatus(testNode, testNodeArgs)
@@ -107,8 +100,7 @@ def checkStatus(testNode, testNodeArgs):
     """Test --terminate-at-block stops at the correct block."""
     Print(" ".join([
         "The test node has begun receiving from the producing node and",
-        "is expected to stop at or little bigger than the block number",
-        "specified here: ",
+        "is expected to stop at the block number specified here: ",
         testNodeArgs
     ]))
 
@@ -122,6 +114,12 @@ def checkStatus(testNode, testNodeArgs):
     else:
         checkHeadOrSpeculative(head, lib)
 
+    # Check for the terminate at block message.
+    match = re.search(r"--terminate-at-block (\d+)", testNodeArgs)
+    termAtBlock = int(match.group(1))
+
+    assert head == termAtBlock, f"head {head} termAtBlock {termAtBlock}"
+
 
 def checkReplay(testNode, testNodeArgs):
     """Test --terminate-at-block with --replay-blockchain."""
@@ -135,20 +133,18 @@ def checkReplay(testNode, testNodeArgs):
         testNodeArgs
     ]))
 
-    testNode.relaunch(chainArg="--replay-blockchain", cachePopen=True)
+    assert not testNode.verifyAlive()
+    testNode.relaunch(chainArg="--replay-blockchain", addSwapFlags={"--terminate-at-block": "9999999"}, cachePopen=True)
 
     # Wait for node to finish up.
-    time.sleep(10)
+    time.sleep(3)
 
     # Check for the terminate at block message.
-    match = re.search(r"--terminate-at-block=(\d+)", testNodeArgs)
+    match = re.search(r"--terminate-at-block (\d+)", testNodeArgs)
     termAtBlock = int(match.group(1))
 
-    termMsg = "Reached configured maximum block {}; terminating".format(
-        termAtBlock
-    )
-
-    assert checkLog(testNode.popenProc.errfile.name, termMsg)
+    head, lib = getBlockNumInfo(testNode)
+    assert head == termAtBlock, f"head {head} termAtBlock {termAtBlock}"
 
 
 def getBlockNumInfo(testNode):
@@ -159,11 +155,12 @@ def getBlockNumInfo(testNode):
         info = testNode.getInfo()
 
         if not info:
-            break
+            continue
 
         try:
             head = info["head_block_num"]
             lib = info["last_irreversible_block_num"]
+            break
 
         except KeyError:
             pass
@@ -172,26 +169,6 @@ def getBlockNumInfo(testNode):
     return head, lib
 
 
-def checkLog(log, message, sleepDuration=1, maxAttempt=10):
-    attemptCnt = 0
-    found = False
-
-    while not found and attemptCnt < maxAttempt:
-        try:
-            with open(log, "r") as f:
-                found = message in f.read()
-
-        except OSError:
-            pass
-
-        if not found:
-            time.sleep(sleepDuration)
-
-        attemptCnt = attemptCnt + 1
-
-    return found
-
-
 def checkIrreversible(head, lib):
     assert head == lib, (
         "Head ({}) should be equal to lib ({})".format(head, lib)
@@ -215,9 +192,9 @@ testSuccessful = False
 try:
     specificNodeosArgs = {
         0 : "--enable-stale-production",
-        1 : "--read-mode irreversible --terminate-at-block=250",
-        2 : "--read-mode speculative --terminate-at-block=550",
-        3 : "--read-mode head --terminate-at-block=850",
+        1 : "--read-mode irreversible --terminate-at-block 75",
+        2 : "--read-mode speculative --terminate-at-block 100",
+        3 : "--read-mode head --terminate-at-block 125",
     }
     traceNodeosArgs = " --plugin eosio::trace_api_plugin --trace-no-abis "
 
@@ -229,7 +206,6 @@ try:
         prodCount=numOfProducers,
         totalProducers=numOfProducers,
         totalNodes=totalNodes,
-        unstartedNodes=totalNodes - numOfProducers,
         pnodes=1,
         useBiosBootFile=False,
         topo="mesh",
@@ -240,6 +216,11 @@ try:
     producingNodeId = 0
     producingNode = cluster.getNode(producingNodeId)
 
+    # wait for all to terminate, needs to be larger than largest terminate-at-block
+    producingNode.waitForBlock( 150, timeout=150 )
+    cluster.biosNode.kill(signal.SIGTERM)
+    producingNode.kill(signal.SIGTERM)
+
     # Start executing test cases here
     Utils.Print("Script Begin .............................")
 
@@ -250,7 +231,6 @@ try:
 
         success = executeTest(
             cluster,
-            producingNode,
             nodeId,
             nodeArgs,
             testResultMsgs

tests/terminate-scenarios-test.py

@@ -20,16 +20,15 @@ import random
 Print=Utils.Print
 errorExit=Utils.errorExit
 
-args=TestHelper.parse_args({"-p","-d","-s","-c","--kill-sig","--kill-count","--keep-logs"
+args=TestHelper.parse_args({"-d","-s","-c","--kill-sig","--keep-logs"
                             ,"--dump-error-details","-v","--leave-running","--clean-run"
                             ,"--terminate-at-block"})
-pnodes=args.p
+pnodes=1
 topo=args.s
 delay=args.d
 chainSyncStrategyStr=args.c
 debug=args.v
 total_nodes = pnodes
-killCount=args.kill_count if args.kill_count > 0 else 1
 killSignal=args.kill_sig
 killEosInstances= not args.leave_running
 dumpErrorDetails=args.dump_error_details
@@ -70,19 +69,20 @@ try:
     if not cluster.waitOnClusterBlockNumSync(3):
         errorExit("Cluster never stabilized")
 
-    Print("Kill %d cluster node instances." % (killCount))
-    if cluster.killSomeEosInstances(killCount, killSignal) is False:
+    Print("Kill cluster node instance.")
+    if cluster.killSomeEosInstances(1, killSignal) is False:
         errorExit("Failed to kill Eos instances")
+    assert not cluster.getNode(0).verifyAlive()
     Print("nodeos instances killed.")
 
-    Print ("Relaunch dead cluster nodes instances.")
+    Print ("Relaunch dead cluster node instance.")
     nodeArg = "--terminate-at-block %d" % terminate if terminate > 0 else ""
     if nodeArg != "":
         if chainSyncStrategyStr == "hardReplay":
             nodeArg += " --truncate-at-block %d" % terminate
-    if cluster.relaunchEosInstances(cachePopen=True, nodeArgs=nodeArg) is False:
-        errorExit("Failed to relaunch Eos instances")
-    Print("nodeos instances relaunched.")
+    if cluster.relaunchEosInstances(cachePopen=True, nodeArgs=nodeArg, waitForTerm=(terminate > 0)) is False:
+        errorExit("Failed to relaunch Eos instance")
+    Print("nodeos instance relaunched.")
 
     testSuccessful=True
 finally:

tests/trx_finality_status_forked_test.py

@@ -93,10 +93,10 @@ try:
     nonProdNode=None
     prodNodes=[]
     producers=[]
-    for node in cluster.getNodes():
+    for i, node in enumerate(cluster.getNodes()):
         node.producers=Cluster.parseProducers(node.nodeId)
         numProducers=len(node.producers)
-        Print(f"node has producers={node.producers}")
+        Print(f"node {i} has producers={node.producers}")
         if numProducers==0:
             if nonProdNode is None:
                 nonProdNode=node
@@ -106,9 +106,8 @@ try:
             prodNodes.append(node)
             producers.extend(node.producers)
 
-
-    node=prodNodes[0]
-    node1=prodNodes[1]
+    prodAB=prodNodes[0]  # defproducera, defproducerb
+    prodC=prodNodes[1]   # defproducerc
 
     # ***   Identify a block where production is stable   ***
 
@@ -122,24 +121,15 @@ try:
     account1.activePublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
     cluster.createAccountAndVerify(account1, cluster.eosioAccount, stakedDeposit=1000)
 
-    Print("Creating account2")
-    account2 = Account('account2')
-    account2.ownerPublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
-    account2.activePublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
-    cluster.createAccountAndVerify(account2, cluster.eosioAccount, stakedDeposit=1000)
-
     Print("Validating accounts after bootstrap")
-    cluster.validateAccounts([account1, account2])
+    cluster.validateAccounts([account1])
 
     # ***   Killing the "bridge" node   ***
     Print("Sending command to kill \"bridge\" node to separate the 2 producer groups.")
-    # block number to start expecting node killed after
-    preKillBlockNum=nonProdNode.getBlockNum()
-    preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum)
-    # kill at the end of the production window for defproducera, so there is still time for the fork for
+    # kill at the beginning of the production window for defproducera, so there is time for the fork for
     # defproducerc to grow before it would overtake the fork for defproducera and defproducerb
-    killAtProducer="defproducerc"
-    nonProdNode.killNodeOnProducer(producer=killAtProducer, whereInSequence=11)
+    killAtProducer="defproducera"
+    nonProdNode.killNodeOnProducer(producer=killAtProducer, whereInSequence=1)
 
     #verify that the non producing node is not alive (and populate the producer nodes with current getInfo data to report if
     #an error occurs)
@@ -150,7 +140,7 @@ try:
     while nonProdNode.verifyAlive() and count > 0:
         # wait on prodNode 0 since it will continue to advance, since defproducera and defproducerb are its producers
         Print("Wait for next block")
-        assert prodNodes[0].waitForNextBlock(timeout=6), "Production node 0 should continue to advance, even after bridge node is killed"
+        assert prodAB.waitForNextBlock(timeout=6), "Production node AB should continue to advance, even after bridge node is killed"
         count -= 1
 
     assert not nonProdNode.verifyAlive(), "Bridge node should have been killed if test was functioning correctly."
@@ -162,9 +152,9 @@ try:
         return status["state"]
 
     transferAmount = 10
-    prodNodes[1].transferFunds(cluster.eosioAccount, account1, f"{transferAmount}.0000 {CORE_SYMBOL}", "fund account")
-    transId = prodNodes[1].getLastTrackedTransactionId()
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    prodC.transferFunds(cluster.eosioAccount, account1, f"{transferAmount}.0000 {CORE_SYMBOL}", "fund account")
+    transId = prodC.getLastTrackedTransactionId()
+    retStatus = prodC.getTransactionStatus(transId)
     state = getState(retStatus)
 
     localState = "LOCALLY_APPLIED"
@@ -176,21 +166,21 @@ try:
     assert state == localState, \
         f"ERROR: getTransactionStatus didn't return \"{localState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}"
 
-    assert prodNodes[1].waitForNextBlock(), "Production node 1 should continue to advance, even after bridge node is killed"
+    assert prodC.waitForNextBlock(), "Production node C should continue to advance, even after bridge node is killed"
 
     # since the Bridge node is killed when this producer is producing its last block in its window, there is plenty of time for the transfer to be
     # sent before the first block is created, but adding this to ensure it is in one of these blocks
     numTries = 2
-    preInfo = prodNodes[1].getInfo()
+    preInfo = prodC.getInfo()
     while numTries > 0:
-        retStatus = prodNodes[1].getTransactionStatus(transId)
+        retStatus = prodC.getTransactionStatus(transId)
         state = getState(retStatus)
         if state == inBlockState:
             break
         numTries -= 1
-        assert prodNodes[1].waitForNextBlock(), "Production node 1 should continue to advance, even after bridge node is killed"
+        assert prodC.waitForNextBlock(), "Production node C should continue to advance, even after bridge node is killed"
 
-    postInfo = prodNodes[1].getInfo()
+    postInfo = prodC.getInfo()
 
     Print(f"getTransactionStatus returned status: {json.dumps(retStatus, indent=1)}")
     assert state == inBlockState, \
@@ -202,42 +192,31 @@ try:
     if not nonProdNode.relaunch():
         errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
 
-    Print("Wait for LIB to move, which indicates prodNode[1] has forked out the branch")
-    assert prodNodes[1].waitForLibToAdvance(), \
-        "ERROR: Network did not reach concensus after bridge node was restarted."
+    Print("Wait for LIB to move, which indicates prodC has forked out the branch")
+    assert prodC.waitForLibToAdvance(), \
+        "ERROR: Network did not reach consensus after bridge node was restarted."
 
-    Print("Wait till prodNodes[1] is reporting at least the same head block number as the forked out block")
-    assert prodNodes[1].waitForBlock(originalInBlockState["block_number"]), \
-        f"Production node 1 should continue to advance after LIB starts advancing. \n\originalInBlockState: {json.dumps(originalInBlockState, indent=1)}"
-
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    retStatus = prodC.getTransactionStatus(transId)
     state = getState(retStatus)
 
     assert state == forkedOutState, \
         f"ERROR: getTransactionStatus didn't return \"{forkedOutState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
+        f"\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}"
 
     for prodNode in prodNodes:
         info=prodNode.getInfo()
         Print(f"node info: {json.dumps(info, indent=1)}")
 
-    retStatus = prodNodes[1].getTransactionStatus(transId)
-    state = getState(retStatus)
+    assert prodC.waitForProducer("defproducerc"), \
+        f"Waiting for prodC to produce, but it never happened" + \
+        f"\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}"
 
-    assert state == forkedOutState, \
-        f"ERROR: getTransactionStatus didn't return \"{forkedOutState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
-
-    assert prodNodes[1].waitForProducer("defproducerc"), \
-        f"Waiting for prodNode 1 to produce, but it never happened" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
-
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    retStatus = prodC.getTransactionStatus(transId)
     state = getState(retStatus)
 
     assert state == inBlockState, \
         f"ERROR: getTransactionStatus didn't return \"{inBlockState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
+        f"\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}"
 
     afterForkInBlockState = retStatus
     afterForkBlockId = retStatus["block_id"]
@@ -245,22 +224,22 @@ try:
         "ERROR: The way the test is designed, the transaction should be added to a block that has a higher number than it was in originally before it was forked out." + \
        f"\n\noriginal in block state: {json.dumps(originalInBlockState, indent=1)}\n\nafter fork in block state: {json.dumps(afterForkInBlockState, indent=1)}"
 
-    assert prodNodes[1].waitForBlock(afterForkInBlockState["block_number"], timeout=120, blockType=BlockType.lib), \
-        f"ERROR: Block never finalized.\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}" + \
+    assert prodC.waitForBlock(afterForkInBlockState["block_number"], timeout=120, blockType=BlockType.lib), \
+        f"ERROR: Block never finalized.\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}" + \
         f"\n\nafter fork in block state: {json.dumps(afterForkInBlockState, indent=1)}"
 
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    retStatus = prodC.getTransactionStatus(transId)
     if afterForkBlockId != retStatus["block_id"]: # might have been forked out, if so wait for new block to become LIB
-        assert prodNodes[1].waitForBlock(retStatus["block_number"], timeout=120, blockType=BlockType.lib), \
-            f"ERROR: Block never finalized.\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}" + \
+        assert prodC.waitForBlock(retStatus["block_number"], timeout=120, blockType=BlockType.lib), \
+            f"ERROR: Block never finalized.\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}" + \
             f"\n\nafter fork in block state: {json.dumps(afterForkInBlockState, indent=1)}"
 
-    retStatus = prodNodes[1].getTransactionStatus(transId)
+    retStatus = prodC.getTransactionStatus(transId)
     state = getState(retStatus)
 
     assert state == irreversibleState, \
         f"ERROR: getTransactionStatus didn't return \"{irreversibleState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}" + \
-        f"\n\nprod 0 info: {json.dumps(prodNodes[0].getInfo(), indent=1)}\n\nprod 1 info: {json.dumps(prodNodes[1].getInfo(), indent=1)}"
+        f"\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}"
 
     testSuccessful=True
 finally:

tests/trx_finality_status_test.py

@@ -146,7 +146,7 @@ try:
     state = getState(retStatus)
 
     assert (state == localState or state == inBlockState), \
-        f"ERROR: getTransactionStatus didn't return \"{localState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}"
+        f"ERROR: getTransactionStatus didn't return \"{localState}\" or \"{inBlockState}\" state.\n\nstatus: {json.dumps(retStatus, indent=1)}"
     status.append(copy.copy(retStatus))
     startingBlockNum=testNode.getInfo()["head_block_num"]
 
@@ -154,12 +154,13 @@ try:
         bnPresent = "block_number" in status
         biPresent = "block_id" in status
         btPresent = "block_timestamp" in status
-        desc = "" if present else "not "
+        desc = "" if present else " not"
         group = bnPresent and biPresent and btPresent if present else not bnPresent and not biPresent and not btPresent
         assert group, \
             f"ERROR: getTransactionStatus should{desc} contain \"block_number\", \"block_id\", or \"block_timestamp\" since state was \"{getState(status)}\".\nstatus: {json.dumps(status, indent=1)}"
 
-    validateTrxState(status[0], present=False)
+    present = True if state == inBlockState else False
+    validateTrxState(status[0], present)
 
     def validate(status, knownTrx=True):
         assert "head_number" in status and "head_id" in status and "head_timestamp" in status, \

tutorials/bios-boot-tutorial/bios-boot-tutorial.py

@@ -332,9 +332,9 @@ def stepSetSystemContract():
     retry(args.cleos + 'push action eosio activate \'["ad9e3d8f650687709fd68f4b90b41f7d825a365b02c23a636cef88ac2ac00c43"]\' -p eosio@active')
     # DISALLOW_EMPTY_PRODUCER_SCHEDULE
     retry(args.cleos + 'push action eosio activate \'["68dcaa34c0517d19666e6b33add67351d8c5f69e999ca1e37931bc410a297428"]\' -p eosio@active')
-     # FIX_LINKAUTH_RESTRICTION
+    # FIX_LINKAUTH_RESTRICTION
     retry(args.cleos + 'push action eosio activate \'["e0fb64b1085cc5538970158d05a009c24e276fb94e1a0bf6a528b48fbc4ff526"]\' -p eosio@active')
-     # REPLACE_DEFERRED
+    # REPLACE_DEFERRED
     retry(args.cleos + 'push action eosio activate \'["ef43112c6543b88db2283a2e077278c315ae2c84719a8b25f25cc88565fbea99"]\' -p eosio@active')
     # NO_DUPLICATE_DEFERRED_ID
     retry(args.cleos + 'push action eosio activate \'["4a90c00d55454dc5b059055ca213579c6ea856967712a56017487886a4d4cc0f"]\' -p eosio@active')
@@ -346,6 +346,12 @@ def stepSetSystemContract():
     retry(args.cleos + 'push action eosio activate \'["4fca8bd82bbd181e714e283f83e1b45d95ca5af40fb89ad3977b653c448f78c2"]\' -p eosio@active')
     # WTMSIG_BLOCK_SIGNATURES
     retry(args.cleos + 'push action eosio activate \'["299dcb6af692324b899b39f16d5a530a33062804e41f09dc97e9f156b4476707"]\' -p eosio@active')
+    # GET_CODE_HASH
+    retry(args.cleos + 'push action eosio activate \'["bcd2a26394b36614fd4894241d3c451ab0f6fd110958c3423073621a70826e99"]\' -p eosio@active')
+    # GET_BLOCK_NUM
+    retry(args.cleos + 'push action eosio activate \'["35c2186cc36f7bb4aeaf4487b36e57039ccf45a9136aa856a5d569ecca55ef2b"]\' -p eosio@active')
+    # CRYPTO_PRIMITIVES
+    retry(args.cleos + 'push action eosio activate \'["6bcb40a24e49c26d0a60513b6aeb8551d264e4717f306b81a37a5afb3b47cedc"]\' -p eosio@active')
     sleep(1)
 
     # install eosio.system latest version

unittests/api_tests.cpp

@@ -1072,8 +1072,8 @@ BOOST_AUTO_TEST_CASE(checktime_pause_block_deadline_not_extended_test) { try {
    dlog("elapsed ${e}us", ("e", dur) );
    BOOST_CHECK( dur >= 75'000 ); // should never fail
    // If this check fails but duration is >= 75'000 (previous check did not fail), then the check here is likely
-   // because it took longer than 2 ms for checktime to trigger, trace to be created, and to get to the now() call.
-   BOOST_CHECK_MESSAGE( dur < 77'000, "elapsed " << dur << "us" );
+   // because it took longer than 10 ms for checktime to trigger, trace to be created, and to get to the now() call.
+   BOOST_CHECK_MESSAGE( dur < 85'000, "elapsed " << dur << "us" );
 
    BOOST_REQUIRE_EQUAL( t.validate(), true );
 } FC_LOG_AND_RETHROW() }

unittests/block_tests.cpp

@@ -44,9 +44,9 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test)
 
    // Push block with invalid transaction to other chain
    tester validator;
-   auto bs = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
+   auto bsf = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
    validator.control->abort_block();
-   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
+   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
    [] (const fc::exception &e)->bool {
       return e.code() == account_name_exists_exception::code_value ;
    }) ;
@@ -82,9 +82,9 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_mroot_test)
 
    // Push block with invalid transaction to other chain
    tester validator;
-   auto bs = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
+   auto bsf = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
    validator.control->abort_block();
-   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
+   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
                            [] (const fc::exception &e)->bool {
                               return e.code() == block_validate_exception::code_value &&
                                      e.to_detail_string().find("invalid block transaction merkle root") != std::string::npos;

unittests/forked_tests.cpp

@@ -267,9 +267,9 @@ BOOST_AUTO_TEST_CASE( forking ) try {
    signed_block bad_block = std::move(*b);
    bad_block.action_mroot = bad_block.previous;
    auto bad_id = bad_block.calculate_id();
-   auto bad_block_bs = c.control->create_block_state_future( bad_id, std::make_shared<signed_block>(std::move(bad_block)) );
+   auto bad_block_bsf = c.control->create_block_state_future( bad_id, std::make_shared<signed_block>(std::move(bad_block)) );
    c.control->abort_block();
-   BOOST_REQUIRE_EXCEPTION(c.control->push_block( bad_block_bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception,
+   BOOST_REQUIRE_EXCEPTION(c.control->push_block( bad_block_bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception,
       [] (const fc::exception &ex)->bool {
          return ex.to_detail_string().find("block signed by unexpected key") != std::string::npos;
       });

unittests/snapshot_tests.cpp

@@ -159,7 +159,7 @@ namespace {
    }
 
    template <typename SNAPSHOT_SUITE>
-   void verify_integrity_hash(const controller& lhs, const controller& rhs) {
+   void verify_integrity_hash(controller& lhs, controller& rhs) {
       const auto lhs_integrity_hash = lhs.calculate_integrity_hash();
       const auto rhs_integrity_hash = rhs.calculate_integrity_hash();
       if (std::is_same_v<SNAPSHOT_SUITE, variant_snapshot_suite> && lhs_integrity_hash.str() != rhs_integrity_hash.str()) {

unittests/unapplied_transaction_queue_tests.cpp

@@ -305,7 +305,7 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try {
    q.add_aborted( { trx20, trx22 } );
    q.add_persisted( trx21 );
    q.add_persisted( trx23 );
-   q.clear_expired( fc::time_point::now(), fc::time_point::now() + fc::seconds( 300 ), [](auto, auto){} );
+   q.clear_expired( fc::time_point::now(), [](){ return false; }, [](auto, auto){} );
    BOOST_CHECK( q.size() == 2 );
    BOOST_REQUIRE( next( q ) == trx23 );
    BOOST_REQUIRE( next( q ) == trx22 );