Bump Leap to release/3.1.5

[3.1] Pinned Builds manual dispatch workflow in CI

.github/workflows/pinned_build.yaml

@@ -0,0 +1,58 @@
+name: "Pinned Build"
+
+on:
+  workflow_dispatch:
+
+permissions:
+  packages: read
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  Build:
+    name: Build
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu18, ubuntu20, ubuntu22]
+    runs-on: ["self-hosted", "enf-x86-beefy-long"]
+    container: ${{ matrix.platform == 'ubuntu18' && 'ubuntu:bionic' || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }}
+    steps:
+      - name: Conditionally update git repo
+        if: ${{ matrix.platform == 'ubuntu18' }}
+        run: |
+          apt-get update
+          apt-get install -y software-properties-common
+          apt-get update
+          add-apt-repository ppa:git-core/ppa
+      - name: Update and Install git
+        run: |
+          apt-get update
+          apt-get install -y git
+          git --version
+      - name: Clone leap
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Install dependencies
+        run: |
+          # https://github.com/actions/runner/issues/2033
+          chown -R $(id -u):$(id -g) $PWD
+          ./scripts/install_deps.sh
+      - name: Build Pinned Build
+        env:
+          LEAP_PINNED_INSTALL_PREFIX: /usr
+        run: |
+          ./scripts/pinned_build.sh deps build "$(nproc)"
+      - name: Upload package
+        uses: actions/upload-artifact@v3
+        with:
+          name: leap-${{matrix.platform}}-pinned-amd64
+          path: build/leap-3*.deb
+      - name: Run Parallel Tests
+        run: |
+          cd build
+          ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)" --timeout 420

CMakeLists.txt

@@ -15,7 +15,7 @@ set( CXX_STANDARD_REQUIRED ON)
 
 set(VERSION_MAJOR 3)
 set(VERSION_MINOR 1)
-set(VERSION_PATCH 3)
+set(VERSION_PATCH 5)
 #set(VERSION_SUFFIX rc4)
 
 if(VERSION_SUFFIX)

docs/01_nodeos/03_plugins/chain_plugin/index.md

@@ -21,33 +21,35 @@ These can only be specified from the `nodeos` command-line:
 ```console
 Command Line Options for eosio::chain_plugin:
   --genesis-json arg                    File to read Genesis State from
-  --genesis-timestamp arg               override the initial timestamp in the 
+  --genesis-timestamp arg               override the initial timestamp in the
                                         Genesis State file
-  --print-genesis-json                  extract genesis_state from blocks.log 
+  --print-genesis-json                  extract genesis_state from blocks.log
                                         as JSON, print to console, and exit
-  --extract-genesis-json arg            extract genesis_state from blocks.log 
+  --extract-genesis-json arg            extract genesis_state from blocks.log
                                         as JSON, write into specified file, and
                                         exit
-  --print-build-info                    print build environment information to 
+  --print-build-info                    print build environment information to
                                         console as JSON and exit
-  --extract-build-info arg              extract build environment information 
+  --extract-build-info arg              extract build environment information
                                         as JSON, write into specified file, and
                                         exit
-  --force-all-checks                    do not skip any checks that can be 
-                                        skipped while replaying irreversible 
-                                        blocks
+  --force-all-checks                    do not skip any validation checks while
+                                        replaying blocks (useful for replaying
+                                        blocks from untrusted source)
   --disable-replay-opts                 disable optimizations that specifically
                                         target replay
-  --replay-blockchain                   clear chain state database and replay 
+  --replay-blockchain                   clear chain state database and replay
                                         all blocks
-  --hard-replay-blockchain              clear chain state database, recover as 
-                                        many blocks as possible from the block 
+  --hard-replay-blockchain              clear chain state database, recover as
+                                        many blocks as possible from the block
                                         log, and then replay those blocks
-  --delete-all-blocks                   clear chain state database and block 
+  --delete-all-blocks                   clear chain state database and block
                                         log
-  --truncate-at-block arg (=0)          stop hard replay / block log recovery 
-                                        at this block number (if set to 
+  --truncate-at-block arg (=0)          stop hard replay / block log recovery
+                                        at this block number (if set to
                                         non-zero number)
+  --terminate-at-block arg (=0)         terminate after reaching this block
+                                        number (if set to a non-zero number)
   --snapshot arg                        File to read Snapshot State from
 
 ```
@@ -58,162 +60,170 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::chain_plugin:
-  --blocks-dir arg (="blocks")          the location of the blocks directory 
-                                        (absolute path or relative to 
+  --blocks-dir arg (="blocks")          the location of the blocks directory
+                                        (absolute path or relative to
                                         application data dir)
   --protocol-features-dir arg (="protocol_features")
-                                        the location of the protocol_features 
+                                        the location of the protocol_features
                                         directory (absolute path or relative to
                                         application config dir)
-  --checkpoint arg                      Pairs of [BLOCK_NUM,BLOCK_ID] that 
+  --checkpoint arg                      Pairs of [BLOCK_NUM,BLOCK_ID] that
                                         should be enforced as checkpoints.
-  --wasm-runtime runtime (=eos-vm-jit)  Override default WASM runtime ( 
+  --wasm-runtime runtime (=eos-vm-jit)  Override default WASM runtime (
                                         "eos-vm-jit", "eos-vm")
-                                        "eos-vm-jit" : A WebAssembly runtime 
-                                        that compiles WebAssembly code to 
+                                        "eos-vm-jit" : A WebAssembly runtime
+                                        that compiles WebAssembly code to
                                         native x86 code prior to execution.
                                         "eos-vm" : A WebAssembly interpreter.
-                                        
-  --profile-account arg                 The name of an account whose code will 
+
+  --profile-account arg                 The name of an account whose code will
                                         be profiled
   --abi-serializer-max-time-ms arg (=15)
-                                        Override default maximum ABI 
+                                        Override default maximum ABI
                                         serialization time allowed in ms
-  --chain-state-db-size-mb arg (=1024)  Maximum size (in MiB) of the chain 
+  --chain-state-db-size-mb arg (=1024)  Maximum size (in MiB) of the chain
                                         state database
   --chain-state-db-guard-size-mb arg (=128)
-                                        Safely shut down node when free space 
-                                        remaining in the chain state database 
+                                        Safely shut down node when free space
+                                        remaining in the chain state database
                                         drops below this size (in MiB).
   --signature-cpu-billable-pct arg (=50)
                                         Percentage of actual signature recovery
-                                        cpu to bill. Whole number percentages, 
+                                        cpu to bill. Whole number percentages,
                                         e.g. 50 for 50%
-  --chain-threads arg (=2)              Number of worker threads in controller 
+  --chain-threads arg (=2)              Number of worker threads in controller
                                         thread pool
   --contracts-console                   print contract's output to console
-  --deep-mind                           print deeper information about chain 
+  --deep-mind                           print deeper information about chain
                                         operations
-  --actor-whitelist arg                 Account added to actor whitelist (may 
+  --actor-whitelist arg                 Account added to actor whitelist (may
                                         specify multiple times)
-  --actor-blacklist arg                 Account added to actor blacklist (may 
+  --actor-blacklist arg                 Account added to actor blacklist (may
                                         specify multiple times)
-  --contract-whitelist arg              Contract account added to contract 
+  --contract-whitelist arg              Contract account added to contract
                                         whitelist (may specify multiple times)
-  --contract-blacklist arg              Contract account added to contract 
+  --contract-blacklist arg              Contract account added to contract
                                         blacklist (may specify multiple times)
   --action-blacklist arg                Action (in the form code::action) added
-                                        to action blacklist (may specify 
+                                        to action blacklist (may specify
                                         multiple times)
-  --key-blacklist arg                   Public key added to blacklist of keys 
-                                        that should not be included in 
-                                        authorities (may specify multiple 
+  --key-blacklist arg                   Public key added to blacklist of keys
+                                        that should not be included in
+                                        authorities (may specify multiple
                                         times)
-  --sender-bypass-whiteblacklist arg    Deferred transactions sent by accounts 
-                                        in this list do not have any of the 
-                                        subjective whitelist/blacklist checks 
-                                        applied to them (may specify multiple 
+  --sender-bypass-whiteblacklist arg    Deferred transactions sent by accounts
+                                        in this list do not have any of the
+                                        subjective whitelist/blacklist checks
+                                        applied to them (may specify multiple
                                         times)
-  --read-mode arg (=speculative)        Database read mode ("speculative", 
+  --read-mode arg (=speculative)        Database read mode ("speculative",
                                         "head", "read-only", "irreversible").
-                                        In "speculative" mode: database 
-                                        contains state changes by transactions 
-                                        in the blockchain up to the head block 
-                                        as well as some transactions not yet 
+                                        In "speculative" mode: database
+                                        contains state changes by transactions
+                                        in the blockchain up to the head block
+                                        as well as some transactions not yet
                                         included in the blockchain.
                                         In "head" mode: database contains state
-                                        changes by only transactions in the 
-                                        blockchain up to the head block; 
-                                        transactions received by the node are 
+                                        changes by only transactions in the
+                                        blockchain up to the head block;
+                                        transactions received by the node are
                                         relayed if valid.
-                                        In "read-only" mode: (DEPRECATED: see 
-                                        p2p-accept-transactions & 
-                                        api-accept-transactions) database 
-                                        contains state changes by only 
-                                        transactions in the blockchain up to 
-                                        the head block; transactions received 
+                                        In "read-only" mode: (DEPRECATED: see
+                                        p2p-accept-transactions &
+                                        api-accept-transactions) database
+                                        contains state changes by only
+                                        transactions in the blockchain up to
+                                        the head block; transactions received
                                         via the P2P network are not relayed and
-                                        transactions cannot be pushed via the 
+                                        transactions cannot be pushed via the
                                         chain API.
-                                        In "irreversible" mode: database 
-                                        contains state changes by only 
-                                        transactions in the blockchain up to 
-                                        the last irreversible block; 
-                                        transactions received via the P2P 
-                                        network are not relayed and 
-                                        transactions cannot be pushed via the 
+                                        In "irreversible" mode: database
+                                        contains state changes by only
+                                        transactions in the blockchain up to
+                                        the last irreversible block;
+                                        transactions received via the P2P
+                                        network are not relayed and
+                                        transactions cannot be pushed via the
                                         chain API.
-                                        
-  --api-accept-transactions arg (=1)    Allow API transactions to be evaluated 
+
+  --api-accept-transactions arg (=1)    Allow API transactions to be evaluated
                                         and relayed if valid.
-  --validation-mode arg (=full)         Chain validation mode ("full" or 
+  --validation-mode arg (=full)         Chain validation mode ("full" or
                                         "light").
                                         In "full" mode all incoming blocks will
                                         be fully validated.
-                                        In "light" mode all incoming blocks 
-                                        headers will be fully validated; 
-                                        transactions in those validated blocks 
-                                        will be trusted 
-                                        
-  --disable-ram-billing-notify-checks   Disable the check which subjectively 
+                                        In "light" mode all incoming blocks
+                                        headers will be fully validated;
+                                        transactions in those validated blocks
+                                        will be trusted
+
+  --disable-ram-billing-notify-checks   Disable the check which subjectively
                                         fails a transaction if a contract bills
-                                        more RAM to another account within the 
+                                        more RAM to another account within the
                                         context of a notification handler (i.e.
-                                        when the receiver is not the code of 
+                                        when the receiver is not the code of
                                         the action).
   --maximum-variable-signature-length arg (=16384)
-                                        Subjectively limit the maximum length 
-                                        of variable components in a variable 
+                                        Subjectively limit the maximum length
+                                        of variable components in a variable
                                         legnth signature to this size in bytes
-  --trusted-producer arg                Indicate a producer whose blocks 
-                                        headers signed by it will be fully 
-                                        validated, but transactions in those 
+  --trusted-producer arg                Indicate a producer whose blocks
+                                        headers signed by it will be fully
+                                        validated, but transactions in those
                                         validated blocks will be trusted.
   --database-map-mode arg (=mapped)     Database map mode ("mapped", "heap", or
                                         "locked").
-                                        In "mapped" mode database is memory 
+                                        In "mapped" mode database is memory
                                         mapped as a file.
                                         In "heap" mode database is preloaded in
-                                        to swappable memory and will use huge 
+                                        to swappable memory and will use huge
                                         pages if available.
                                         In "locked" mode database is preloaded,
-                                        locked in to memory, and will use huge 
+                                        locked in to memory, and will use huge
                                         pages if available.
-                                        
-  --enable-account-queries arg (=0)     enable queries to find accounts by 
+
+  --eos-vm-oc-cache-size-mb arg (=1024) Maximum size (in MiB) of the EOS VM OC
+                                        code cache
+  --eos-vm-oc-compile-threads arg (=1)  Number of threads to use for EOS VM OC
+                                        tier-up
+  --eos-vm-oc-enable                    Enable EOS VM OC tier-up runtime
+  --enable-account-queries arg (=0)     enable queries to find accounts by
                                         various metadata.
   --max-nonprivileged-inline-action-size arg (=4096)
-                                        maximum allowed size (in bytes) of an 
-                                        inline action for a nonprivileged 
+                                        maximum allowed size (in bytes) of an
+                                        inline action for a nonprivileged
                                         account
   --transaction-retry-max-storage-size-gb arg
-                                        Maximum size (in GiB) allowed to be 
-                                        allocated for the Transaction Retry 
-                                        feature. Setting above 0 enables this 
+                                        Maximum size (in GiB) allowed to be
+                                        allocated for the Transaction Retry
+                                        feature. Setting above 0 enables this
                                         feature.
   --transaction-retry-interval-sec arg (=20)
-                                        How often, in seconds, to resend an 
-                                        incoming transaction to network if not 
+                                        How often, in seconds, to resend an
+                                        incoming transaction to network if not
                                         seen in a block.
-  --transaction-retry-max-expiration-sec arg (=90)
-                                        Maximum allowed transaction expiration 
-                                        for retry transactions, will retry 
+  --transaction-retry-max-expiration-sec arg (=120)
+                                        Maximum allowed transaction expiration
+                                        for retry transactions, will retry
                                         transactions up to this value.
   --transaction-finality-status-max-storage-size-gb arg
-                                        Maximum size (in GiB) allowed to be 
-                                        allocated for the Transaction Finality 
+                                        Maximum size (in GiB) allowed to be
+                                        allocated for the Transaction Finality
                                         Status feature. Setting above 0 enables
                                         this feature.
   --transaction-finality-status-success-duration-sec arg (=180)
-                                        Duration (in seconds) a successful 
-                                        transaction's Finality Status will 
-                                        remain available from being first 
+                                        Duration (in seconds) a successful
+                                        transaction's Finality Status will
+                                        remain available from being first
                                         identified.
   --transaction-finality-status-failure-duration-sec arg (=180)
-                                        Duration (in seconds) a failed 
-                                        transaction's Finality Status will 
-                                        remain available from being first 
+                                        Duration (in seconds) a failed
+                                        transaction's Finality Status will
+                                        remain available from being first
                                         identified.
+  --block-log-retain-blocks arg         if set, periodically prune the block
+                                        log to store only configured number of
+                                        most recent blocks
 
 ```
 

docs/01_nodeos/03_plugins/history_api_plugin/index.md

@@ -1,59 +0,0 @@
-[[warning | Deprecation Notice]]
-| The `history_plugin` that the `history_api_plugin` depends upon is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead.
-
-## Description
-
-The `history_api_plugin` exposes functionality from the [`history_plugin`](../history_plugin/index.md) to the RPC API interface managed by the [`http_plugin`](../http_plugin/index.md), providing read-only access to blockchain data.
-
-It provides four RPC API endpoints:
-
-* get_actions
-* get_transaction
-* get_key_accounts
-* get_controlled_accounts
-
-The four actions listed above are used by the following `cleos` commands (matching order):
-
-* get actions
-* get transaction
-* get accounts
-* get servants
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::history_api_plugin
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_api_plugin
-```
-
-## Options
-
-None
-
-## Dependencies
-
-* [`history_plugin`](../history_plugin/index.md)
-* [`chain_plugin`](../chain_plugin/index.md)
-* [`http_plugin`](../http_plugin/index.md)
-
-### Load Dependency Examples
-
-```console
-# config.ini
-plugin = eosio::history_plugin
-[options]
-plugin = eosio::chain_plugin
-[options]
-plugin = eosio::http_plugin
-[options]
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_plugin [options]  \
-           --plugin eosio::chain_plugin [operations] [options]  \
-           --plugin eosio::http_plugin [options]
-```

docs/01_nodeos/03_plugins/history_plugin/index.md

@@ -1,41 +0,0 @@
-[[warning | Deprecation Notice]]
-| The `history_plugin` is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead.
-
-## Description
-
-The `history_plugin` provides a cache layer to obtain historical data about the blockchain objects. It depends on [`chain_plugin`](../chain_plugin/index.md) for the data.
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::history_plugin
-[options]
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_plugin [options]
-```
-
-## Options
-
-These can be specified from both the `nodeos` command-line or the `config.ini` file:
-
-```console
-Config Options for eosio::history_plugin:
-  -f [ --filter-on ] arg                Track actions which match 
-                                        receiver:action:actor. Actor may be 
-                                        blank to include all. Action and Actor 
-                                        both blank allows all from Recieiver. 
-                                        Receiver may not be blank.
-  -F [ --filter-out ] arg               Do not track actions which match 
-                                        receiver:action:actor. Action and Actor
-                                        both blank excludes all from Reciever. 
-                                        Actor blank excludes all from 
-                                        reciever:action. Receiver may not be 
-                                        blank.
-```
-
-## Dependencies
-
-* [`chain_plugin`](../chain_plugin/index.md)

docs/01_nodeos/03_plugins/http_plugin/index.md

@@ -22,46 +22,47 @@ These can be specified from both the command-line or the `config.ini` file:
 
 ```console
 Config Options for eosio::http_plugin:
-  --unix-socket-path arg                The filename (relative to data-dir) to 
-                                        create a unix socket for HTTP RPC; set 
-                                        blank to disable (=keosd.sock for keosd)
-  --http-server-address arg (=127.0.0.1:8888 for nodeos)
-                                        The local IP and port to listen for 
+  --unix-socket-path arg                The filename (relative to data-dir) to
+                                        create a unix socket for HTTP RPC; set
+                                        blank to disable.
+  --http-server-address arg (=127.0.0.1:8888)
+                                        The local IP and port to listen for
                                         incoming http connections; set blank to
                                         disable.
-  --https-server-address arg            The local IP and port to listen for 
+  --https-server-address arg            The local IP and port to listen for
                                         incoming https connections; leave blank
                                         to disable.
-  --https-certificate-chain-file arg    Filename with the certificate chain to 
-                                        present on https connections. PEM 
+  --https-certificate-chain-file arg    Filename with the certificate chain to
+                                        present on https connections. PEM
                                         format. Required for https.
-  --https-private-key-file arg          Filename with https private key in PEM 
+  --https-private-key-file arg          Filename with https private key in PEM
                                         format. Required for https
-  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use: 
+  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use:
                                         secp384r1 or prime256v1
   --access-control-allow-origin arg     Specify the Access-Control-Allow-Origin
                                         to be returned on each request.
   --access-control-allow-headers arg    Specify the Access-Control-Allow-Header
                                         s to be returned on each request.
-  --access-control-max-age arg          Specify the Access-Control-Max-Age to 
+  --access-control-max-age arg          Specify the Access-Control-Max-Age to
                                         be returned on each request.
   --access-control-allow-credentials    Specify if Access-Control-Allow-Credent
-                                        ials: true should be returned on each 
+                                        ials: true should be returned on each
                                         request.
-  --max-body-size arg (=1048576)        The maximum body size in bytes allowed 
+  --max-body-size arg (=2097152)        The maximum body size in bytes allowed
                                         for incoming RPC requests
   --http-max-bytes-in-flight-mb arg (=500)
-                                        Maximum size in megabytes http_plugin 
-                                        should use for processing http 
-                                        requests. 503 error response when 
-                                        exceeded.
+                                        Maximum size in megabytes http_plugin
+                                        should use for processing http
+                                        requests. -1 for unlimited. 429 error
+                                        response when exceeded.
+  --http-max-response-time-ms arg (=30) Maximum time for processing a request.
   --verbose-http-errors                 Append the error log to HTTP responses
-  --http-validate-host arg (=1)         If set to false, then any incoming 
+  --http-validate-host arg (=1)         If set to false, then any incoming
                                         "Host" header is considered valid
-  --http-alias arg                      Additionaly acceptable values for the 
-                                        "Host" header of incoming HTTP 
-                                        requests, can be specified multiple 
-                                        times.  Includes http/s_server_address 
+  --http-alias arg                      Additionaly acceptable values for the
+                                        "Host" header of incoming HTTP
+                                        requests, can be specified multiple
+                                        times.  Includes http/s_server_address
                                         by default.
   --http-threads arg (=2)               Number of worker threads in http thread
                                         pool

docs/01_nodeos/03_plugins/net_plugin/index.md

@@ -44,7 +44,7 @@ Config Options for eosio::net_plugin:
   --p2p-accept-transactions arg (=1)    Allow transactions received over p2p
                                         network to be evaluated and relayed if
                                         valid.
-  --agent-name arg (="EOS Test Agent")  The name supplied to identify this node
+  --agent-name arg (=EOS Test Agent)    The name supplied to identify this node
                                         amongst the peers.
   --allowed-connection arg (=any)       Can be 'any' or 'producers' or
                                         'specified' or 'none'. If 'specified',
@@ -73,13 +73,15 @@ Config Options for eosio::net_plugin:
                                         synchronization
   --use-socket-read-watermark arg (=0)  Enable experimental socket read
                                         watermark optimization
-  --peer-log-format arg (=["${_name}" ${_ip}:${_port}])
+  --peer-log-format arg (=["${_name}" - ${_cid} ${_ip}:${_port}] )
                                         The string used to format peers when
                                         logging messages about them.  Variables
                                         are escaped with ${<variable name>}.
                                         Available Variables:
                                            _name  self-reported name
 
+                                           _cid   assigned connection id
+
                                            _id    self-reported ID (64 hex
                                                   characters)
 
@@ -94,7 +96,10 @@ Config Options for eosio::net_plugin:
                                                   peer
 
                                            _lport local port number connected
-                                                  to peer                                        
+                                                  to peer
+  --p2p-keepalive-interval-ms arg (=10000)
+                                        peer heartbeat keepalive message
+                                        interval in milliseconds
 ```
 
 ## Dependencies

docs/01_nodeos/03_plugins/producer_plugin/index.md

@@ -23,7 +23,6 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::producer_plugin:
-
   -e [ --enable-stale-production ]      Enable block production, even if the
                                         chain is stale.
   -x [ --pause-on-startup ]             Start this node in a state where
@@ -49,7 +48,7 @@ Config Options for eosio::producer_plugin:
                                         <public-key>=<provider-spec>
                                         Where:
                                            <public-key>    is a string form of
-                                                           a vaild Antelope public
+                                                           a valid EOS public
                                                            key
 
                                            <provider-spec> is a string in the
@@ -59,7 +58,7 @@ Config Options for eosio::producer_plugin:
                                            <provider-type> is KEY, or KEOSD
 
                                            KEY:<data>      is a string form of
-                                                           a valid Antelope
+                                                           a valid EOS
                                                            private key which
                                                            maps to the provided
                                                            public key
@@ -119,11 +118,6 @@ Config Options for eosio::producer_plugin:
                                         Sets the maximum amount of failures
                                         that are allowed for a given account
                                         per block.
-                                        Disregarded for accounts that have been
-                                        whitelisted by disabling subjective
-                                        billing for the account using the
-                                        disable-subjective-account-billing
-                                        configuration option.
   --subjective-account-decay-time-minutes arg (=1440)
                                         Sets the time to return full subjective
                                         cpu for accounts
@@ -142,9 +136,6 @@ Config Options for eosio::producer_plugin:
   --disable-subjective-account-billing arg
                                         Account which is excluded from
                                         subjective CPU billing
-                                        Account is considered whitelisted and
-                                        will not be subject to enforcement of
-                                        subjective-account-max-failures.
   --disable-subjective-p2p-billing arg (=1)
                                         Disable subjective CPU billing for P2P
                                         transactions

docs/01_nodeos/03_plugins/resource_monitor_plugin/index.md

@@ -1,19 +1,10 @@
 
 ## Overview
 
-The `resource_monitor_plugin` monitors space usage in the computing system where `nodeos` is running. Specifically, every `resource-monitor-interval-seconds` seconds,
-it measures the individual space used by each of the file systems mounted
-by `data-dir`, `state-dir`, `blocks-log-dir`, `snapshots-dir`,
-`state-history-dir`, and `trace-dir`.
-When space usage in any of the monitored file system is within `5%` of the threshold
-specified by `resource-monitor-space-threshold`, a warning containing the file system
-path and percentage of space has used is printed out.
-When space usage exceeds the threshold,
-if `resource-monitor-not-shutdown-on-threshold-exceeded` is not set,
-`nodeos` gracefully shuts down; if `resource-monitor-not-shutdown-on-threshold-exceeded` is set, `nodeos` prints out warnings periodically
-until space usage goes under the threshold.
+The `resource_monitor_plugin` monitors space usage in the computing system where `nodeos` is running. Specifically, every `resource-monitor-interval-seconds` seconds, it measures the individual space used by each of the file systems mounted by `data-dir`, `state-dir`, `blocks-log-dir`, `snapshots-dir`, `state-history-dir`, and `trace-dir`. When space usage in any of the monitored file system is within `5%` of the threshold specified by `resource-monitor-space-threshold`, a warning containing the file system path and percentage of space has used is printed out. When space usage exceeds the threshold, if `resource-monitor-not-shutdown-on-threshold-exceeded` is not set, `nodeos` gracefully shuts down; if `resource-monitor-not-shutdown-on-threshold-exceeded` is set, `nodeos` prints out warnings periodically until space usage goes under the threshold.
 
 `resource_monitor_plugin` is always loaded.
+
 ## Usage
 
 ```console
@@ -32,33 +23,28 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::resource_monitor_plugin:
-
   --resource-monitor-interval-seconds arg (=2)
-                                        Time in seconds between two consecutive checks
-                                        of space usage. Should be between 1 and 300.
+                                        Time in seconds between two consecutive
+                                        checks of resource usage. Should be
+                                        between 1 and 300
   --resource-monitor-space-threshold arg (=90)
-                                        Threshold in terms of percentage of used space
-                                        vs total space. If the used space is within
-                                        `5%` of the threshold, a warning is generated.
-                                        If the used space is above the threshold and
-                                        `resource-monitor-not-shutdown-on-threshold-exceeded`
-                                        is enabled, a shutdown is initiated; otherwise
-                                        a warning will be continuously printed out.
-                                        The value should be between 6 and 99.
+                                        Threshold in terms of percentage of
+                                        used space vs total space. If used
+                                        space is above (threshold - 5%), a
+                                        warning is generated.  Unless
+                                        resource-monitor-not-shutdown-on-thresh
+                                        old-exceeded is enabled, a graceful
+                                        shutdown is initiated if used space is
+                                        above the threshold. The value should
+                                        be between 6 and 99
   --resource-monitor-not-shutdown-on-threshold-exceeded
-                                        A switch used to indicate `nodeos` will "not"
-                                        shutdown when threshold is exceeded. When not
-                                        set, `nodeos` will shutdown.
+                                        Used to indicate nodeos will not
+                                        shutdown when threshold is exceeded.
   --resource-monitor-warning-interval arg (=30)
-                                        Number of monitor intervals between which a
-                                        warning is displayed.  For example, if
-                                        `resource-monitor-warning-interval` is to 10
-                                        and `resource-monitor-interval-seconds` is 2,
-                                        a warning will be displayed every 20 seconds,
-                                        even though the space usage is checked every
-                                        2 seconds.  This is used to throttle the
-                                        number of warnings in the `nodeos` log file.
-                                        Should be between 1 and 450.
+                                        Number of resource monitor intervals
+                                        between two consecutive warnings when
+                                        the threshold is hit. Should be between
+                                        1 and 450
 ```
 
 ## Plugin Dependencies

docs/01_nodeos/03_plugins/state_history_plugin/index.md

@@ -31,19 +31,21 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::state_history_plugin:
-
   --state-history-dir arg (="state-history")
-                                        the location of the state-history 
+                                        the location of the state-history
                                         directory (absolute path or relative to
                                         application data dir)
   --trace-history                       enable trace history
   --chain-state-history                 enable chain state history
   --state-history-endpoint arg (=127.0.0.1:8080)
-                                        the endpoint upon which to listen for 
-                                        incoming connections. Caution: only 
-                                        expose this port to your internal 
+                                        the endpoint upon which to listen for
+                                        incoming connections. Caution: only
+                                        expose this port to your internal
                                         network.
   --trace-history-debug-mode            enable debug mode for trace history
+  --state-history-log-retain-blocks arg if set, periodically prune the state
+                                        history files to store only configured
+                                        number of most recent blocks
 ```
 
 ## Examples

libraries/chain/controller.cpp

@@ -2056,43 +2056,64 @@ struct controller_impl {
       }
    } FC_CAPTURE_AND_RETHROW() } /// apply_block
 
+
+   // thread safe, expected to be called from thread other than the main thread
+   block_state_ptr create_block_state_i( const block_id_type& id, const signed_block_ptr& b, const block_header_state& prev ) {
+      auto trx_mroot = calculate_trx_merkle( b->transactions );
+      EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception,
+                  "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) );
+
+      const bool skip_validate_signee = false;
+      auto bsp = std::make_shared<block_state>(
+            prev,
+            b,
+            protocol_features.get_protocol_feature_set(),
+            [this]( block_timestamp_type timestamp,
+                    const flat_set<digest_type>& cur_features,
+                    const vector<digest_type>& new_features )
+            { check_protocol_features( timestamp, cur_features, new_features ); },
+            skip_validate_signee
+      );
+
+      EOS_ASSERT( id == bsp->id, block_validate_exception,
+                  "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id) );
+      return bsp;
+   }
+
    std::future<block_state_ptr> create_block_state_future( const block_id_type& id, const signed_block_ptr& b ) {
       EOS_ASSERT( b, block_validate_exception, "null block" );
 
+      return async_thread_pool( thread_pool.get_executor(), [b, id, control=this]() {
+         // no reason for a block_state if fork_db already knows about block
+         auto existing = control->fork_db.get_block( id );
+         EOS_ASSERT( !existing, fork_database_exception, "we already know about this block: ${id}", ("id", id) );
+
+         auto prev = control->fork_db.get_block_header( b->previous );
+         EOS_ASSERT( prev, unlinkable_block_exception,
+                     "unlinkable block ${id}", ("id", id)("previous", b->previous) );
+
+         return control->create_block_state_i( id, b, *prev );
+      } );
+   }
+
+   // thread safe, expected to be called from thread other than the main thread
+   block_state_ptr create_block_state( const block_id_type& id, const signed_block_ptr& b ) {
+      EOS_ASSERT( b, block_validate_exception, "null block" );
+
       // no reason for a block_state if fork_db already knows about block
       auto existing = fork_db.get_block( id );
       EOS_ASSERT( !existing, fork_database_exception, "we already know about this block: ${id}", ("id", id) );
 
+      // previous not found could mean that previous block not applied yet
       auto prev = fork_db.get_block_header( b->previous );
-      EOS_ASSERT( prev, unlinkable_block_exception,
-                  "unlinkable block ${id}", ("id", id)("previous", b->previous) );
+      if( !prev ) return {};
 
-      return async_thread_pool( thread_pool.get_executor(), [b, prev, id, control=this]() {
-         const bool skip_validate_signee = false;
-
-         auto trx_mroot = calculate_trx_merkle( b->transactions );
-         EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception,
-                     "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) );
-
-         auto bsp = std::make_shared<block_state>(
-                        *prev,
-                        move( b ),
-                        control->protocol_features.get_protocol_feature_set(),
-                        [control]( block_timestamp_type timestamp,
-                                   const flat_set<digest_type>& cur_features,
-                                   const vector<digest_type>& new_features )
-                        { control->check_protocol_features( timestamp, cur_features, new_features ); },
-                        skip_validate_signee
-         );
-
-         EOS_ASSERT( id == bsp->id, block_validate_exception,
-                     "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id) );
-         return bsp;
-      } );
+      return create_block_state_i( id, b, *prev );
    }
 
-   void push_block( std::future<block_state_ptr>& block_state_future,
-                    const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup )
+   void push_block( const block_state_ptr& bsp,
+                    const forked_branch_callback& forked_branch_cb,
+                    const trx_meta_cache_lookup& trx_lookup )
    {
       controller::block_status s = controller::block_status::complete;
       EOS_ASSERT(!pending, block_validate_exception, "it is not valid to push a block when there is a pending block");
@@ -2101,10 +2122,10 @@ struct controller_impl {
          trusted_producer_light_validation = old_value;
       });
       try {
-         block_state_ptr bsp = block_state_future.get();
+         EOS_ASSERT( bsp, block_validate_exception, "null block" );
          const auto& b = bsp->block;
 
-         if( conf.terminate_at_block > 0 && conf.terminate_at_block < self.head_block_num()) {
+         if( conf.terminate_at_block > 0 && conf.terminate_at_block <= self.head_block_num()) {
             ilog("Reached configured maximum block ${num}; terminating", ("num", conf.terminate_at_block) );
             shutdown();
             return;
@@ -2139,7 +2160,7 @@ struct controller_impl {
          EOS_ASSERT( (s == controller::block_status::irreversible || s == controller::block_status::validated),
                      block_validate_exception, "invalid block status for replay" );
 
-         if( conf.terminate_at_block > 0 && conf.terminate_at_block < self.head_block_num() ) {
+         if( conf.terminate_at_block > 0 && conf.terminate_at_block <= self.head_block_num() ) {
             ilog("Reached configured maximum block ${num}; terminating", ("num", conf.terminate_at_block) );
             shutdown();
             return;
@@ -2816,11 +2837,16 @@ std::future<block_state_ptr> controller::create_block_state_future( const block_
    return my->create_block_state_future( id, b );
 }
 
-void controller::push_block( std::future<block_state_ptr>& block_state_future,
-                             const forked_branch_callback& forked_branch_cb, const trx_meta_cache_lookup& trx_lookup )
+block_state_ptr controller::create_block_state( const block_id_type& id, const signed_block_ptr& b ) const {
+   return my->create_block_state( id, b );
+}
+
+void controller::push_block( const block_state_ptr& bsp,
+                             const forked_branch_callback& forked_branch_cb,
+                             const trx_meta_cache_lookup& trx_lookup )
 {
    validate_db_available_size();
-   my->push_block( block_state_future, forked_branch_cb, trx_lookup );
+   my->push_block( bsp, forked_branch_cb, trx_lookup );
 }
 
 transaction_trace_ptr controller::push_transaction( const transaction_metadata_ptr& trx,
@@ -3468,10 +3494,14 @@ std::optional<chain_id_type> controller::extract_chain_id_from_db( const path& s
 
       if( db.revision() < 1 ) return {};
 
-      return db.get<global_property_object>().chain_id;
-   } catch( const bad_database_version_exception& ) {
-      throw;
-   } catch( ... ) {
+      auto * gpo = db.find<global_property_object>();
+      if (gpo==nullptr) return {};
+
+      return gpo->chain_id;
+   } catch( const std::system_error& e ) {
+      // do not propagate db_error_code::not_found for absent db, so it will be created
+      if( e.code().value() != chainbase::db_error_code::not_found )
+         throw;
    }
 
    return {};

libraries/chain/fork_database.cpp

@@ -9,6 +9,7 @@
 #include <boost/multi_index/composite_key.hpp>
 #include <fc/io/fstream.hpp>
 #include <fstream>
+#include <shared_mutex>
 
 namespace eosio { namespace chain {
    using boost::multi_index_container;
@@ -60,27 +61,44 @@ namespace eosio { namespace chain {
    }
 
    struct fork_database_impl {
-      fork_database_impl( fork_database& self, const fc::path& data_dir )
-      :self(self)
-      ,datadir(data_dir)
+      explicit fork_database_impl( const fc::path& data_dir )
+      :datadir(data_dir)
       {}
 
-      fork_database&        self;
+      std::shared_mutex     mtx;
       fork_multi_index_type index;
       block_state_ptr       root; // Only uses the block_header_state portion
       block_state_ptr       head;
       fc::path              datadir;
 
-      void add( const block_state_ptr& n,
-                bool ignore_duplicate, bool validate,
-                const std::function<void( block_timestamp_type,
-                                          const flat_set<digest_type>&,
-                                          const vector<digest_type>& )>& validator );
+      void open_impl( const std::function<void( block_timestamp_type,
+                                                const flat_set<digest_type>&,
+                                                const vector<digest_type>& )>& validator );
+      void close_impl();
+
+
+      block_header_state_ptr  get_block_header_impl( const block_id_type& id )const;
+      block_state_ptr         get_block_impl( const block_id_type& id )const;
+      void            reset_impl( const block_header_state& root_bhs );
+      void            rollback_head_to_root_impl();
+      void            advance_root_impl( const block_id_type& id );
+      void            remove_impl( const block_id_type& id );
+      branch_type     fetch_branch_impl( const block_id_type& h, uint32_t trim_after_block_num )const;
+      block_state_ptr search_on_branch_impl( const block_id_type& h, uint32_t block_num )const;
+      pair<branch_type, branch_type> fetch_branch_from_impl( const block_id_type& first,
+                                                             const block_id_type& second )const;
+      void mark_valid_impl( const block_state_ptr& h );
+
+      void add_impl( const block_state_ptr& n,
+                     bool ignore_duplicate, bool validate,
+                     const std::function<void( block_timestamp_type,
+                                               const flat_set<digest_type>&,
+                                               const vector<digest_type>& )>& validator );
    };
 
 
    fork_database::fork_database( const fc::path& data_dir )
-   :my( new fork_database_impl( *this, data_dir ) )
+   :my( new fork_database_impl( data_dir ) )
    {}
 
 
@@ -88,10 +106,18 @@ namespace eosio { namespace chain {
                                                        const flat_set<digest_type>&,
                                                        const vector<digest_type>& )>& validator )
    {
-      if (!fc::is_directory(my->datadir))
-         fc::create_directories(my->datadir);
+      std::lock_guard g( my->mtx );
+      my->open_impl( validator );
+   }
 
-      auto fork_db_dat = my->datadir / config::forkdb_filename;
+   void fork_database_impl::open_impl( const std::function<void( block_timestamp_type,
+                                                                 const flat_set<digest_type>&,
+                                                                 const vector<digest_type>& )>& validator )
+   {
+      if (!fc::is_directory(datadir))
+         fc::create_directories(datadir);
+
+      auto fork_db_dat = datadir / config::forkdb_filename;
       if( fc::exists( fork_db_dat ) ) {
          try {
             string content;
@@ -102,29 +128,29 @@ namespace eosio { namespace chain {
             // validate totem
             uint32_t totem = 0;
             fc::raw::unpack( ds, totem );
-            EOS_ASSERT( totem == magic_number, fork_database_exception,
+            EOS_ASSERT( totem == fork_database::magic_number, fork_database_exception,
                         "Fork database file '${filename}' has unexpected magic number: ${actual_totem}. Expected ${expected_totem}",
                         ("filename", fork_db_dat.generic_string())
                         ("actual_totem", totem)
-                        ("expected_totem", magic_number)
+                        ("expected_totem", fork_database::magic_number)
             );
 
             // validate version
             uint32_t version = 0;
             fc::raw::unpack( ds, version );
-            EOS_ASSERT( version >= min_supported_version && version <= max_supported_version,
+            EOS_ASSERT( version >= fork_database::min_supported_version && version <= fork_database::max_supported_version,
                         fork_database_exception,
                        "Unsupported version of fork database file '${filename}'. "
                        "Fork database version is ${version} while code supports version(s) [${min},${max}]",
                        ("filename", fork_db_dat.generic_string())
                        ("version", version)
-                       ("min", min_supported_version)
-                       ("max", max_supported_version)
+                       ("min", fork_database::min_supported_version)
+                       ("max", fork_database::max_supported_version)
             );
 
             block_header_state bhs;
             fc::raw::unpack( ds, bhs );
-            reset( bhs );
+            reset_impl( bhs );
 
             unsigned_int size; fc::raw::unpack( ds, size );
             for( uint32_t i = 0, n = size.value; i < n; ++i ) {
@@ -132,27 +158,27 @@ namespace eosio { namespace chain {
                fc::raw::unpack( ds, s );
                // do not populate transaction_metadatas, they will be created as needed in apply_block with appropriate key recovery
                s.header_exts = s.block->validate_and_extract_header_extensions();
-               my->add( std::make_shared<block_state>( move( s ) ), false, true, validator );
+               add_impl( std::make_shared<block_state>( std::move( s ) ), false, true, validator );
             }
             block_id_type head_id;
             fc::raw::unpack( ds, head_id );
 
-            if( my->root->id == head_id ) {
-               my->head = my->root;
+            if( root->id == head_id ) {
+               head = root;
             } else {
-               my->head = get_block( head_id );
-               EOS_ASSERT( my->head, fork_database_exception,
+               head = get_block_impl( head_id );
+               EOS_ASSERT( head, fork_database_exception,
                            "could not find head while reconstructing fork database from file; '${filename}' is likely corrupted",
                            ("filename", fork_db_dat.generic_string()) );
             }
 
-            auto candidate = my->index.get<by_lib_block_num>().begin();
-            if( candidate == my->index.get<by_lib_block_num>().end() || !(*candidate)->is_valid() ) {
-               EOS_ASSERT( my->head->id == my->root->id, fork_database_exception,
+            auto candidate = index.get<by_lib_block_num>().begin();
+            if( candidate == index.get<by_lib_block_num>().end() || !(*candidate)->is_valid() ) {
+               EOS_ASSERT( head->id == root->id, fork_database_exception,
                            "head not set to root despite no better option available; '${filename}' is likely corrupted",
                            ("filename", fork_db_dat.generic_string()) );
             } else {
-               EOS_ASSERT( !first_preferred( **candidate, *my->head ), fork_database_exception,
+               EOS_ASSERT( !first_preferred( **candidate, *head ), fork_database_exception,
                            "head not set to best available option available; '${filename}' is likely corrupted",
                            ("filename", fork_db_dat.generic_string()) );
             }
@@ -163,10 +189,15 @@ namespace eosio { namespace chain {
    }
 
    void fork_database::close() {
-      auto fork_db_dat = my->datadir / config::forkdb_filename;
+      std::lock_guard g( my->mtx );
+      my->close_impl();
+   }
 
-      if( !my->root ) {
-         if( my->index.size() > 0 ) {
+   void fork_database_impl::close_impl() {
+      auto fork_db_dat = datadir / config::forkdb_filename;
+
+      if( !root ) {
+         if( index.size() > 0 ) {
             elog( "fork_database is in a bad state when closing; not writing out '${filename}'",
                   ("filename", fork_db_dat.generic_string()) );
          }
@@ -174,13 +205,13 @@ namespace eosio { namespace chain {
       }
 
       std::ofstream out( fork_db_dat.generic_string().c_str(), std::ios::out | std::ios::binary | std::ofstream::trunc );
-      fc::raw::pack( out, magic_number );
-      fc::raw::pack( out, max_supported_version ); // write out current version which is always max_supported_version
-      fc::raw::pack( out, *static_cast<block_header_state*>(&*my->root) );
-      uint32_t num_blocks_in_fork_db = my->index.size();
+      fc::raw::pack( out, fork_database::magic_number );
+      fc::raw::pack( out, fork_database::max_supported_version ); // write out current version which is always max_supported_version
+      fc::raw::pack( out, *static_cast<block_header_state*>(&*root) );
+      uint32_t num_blocks_in_fork_db = index.size();
       fc::raw::pack( out, unsigned_int{num_blocks_in_fork_db} );
 
-      const auto& indx = my->index.get<by_lib_block_num>();
+      const auto& indx = index.get<by_lib_block_num>();
 
       auto unvalidated_itr = indx.rbegin();
       auto unvalidated_end = boost::make_reverse_iterator( indx.lower_bound( false ) );
@@ -215,30 +246,40 @@ namespace eosio { namespace chain {
          fc::raw::pack( out, *(*itr) );
       }
 
-      if( my->head ) {
-         fc::raw::pack( out, my->head->id );
+      if( head ) {
+         fc::raw::pack( out, head->id );
       } else {
          elog( "head not set in fork database; '${filename}' will be corrupted",
                ("filename", fork_db_dat.generic_string()) );
       }
 
-      my->index.clear();
+      index.clear();
    }
 
    fork_database::~fork_database() {
-      close();
+      my->close_impl();
    }
 
    void fork_database::reset( const block_header_state& root_bhs ) {
-      my->index.clear();
-      my->root = std::make_shared<block_state>();
-      static_cast<block_header_state&>(*my->root) = root_bhs;
-      my->root->validated = true;
-      my->head = my->root;
+      std::lock_guard g( my->mtx );
+      my->reset_impl(root_bhs);
+   }
+
+   void fork_database_impl::reset_impl( const block_header_state& root_bhs ) {
+      index.clear();
+      root = std::make_shared<block_state>();
+      static_cast<block_header_state&>(*root) = root_bhs;
+      root->validated = true;
+      head = root;
    }
 
    void fork_database::rollback_head_to_root() {
-      auto& by_id_idx = my->index.get<by_block_id>();
+      std::lock_guard g( my->mtx );
+      my->rollback_head_to_root_impl();
+   }
+
+   void fork_database_impl::rollback_head_to_root_impl() {
+      auto& by_id_idx = index.get<by_block_id>();
       auto itr = by_id_idx.begin();
       while (itr != by_id_idx.end()) {
          by_id_idx.modify( itr, [&]( block_state_ptr& bsp ) {
@@ -246,13 +287,18 @@ namespace eosio { namespace chain {
          } );
          ++itr;
       }
-      my->head = my->root;
+      head = root;
    }
 
    void fork_database::advance_root( const block_id_type& id ) {
-      EOS_ASSERT( my->root, fork_database_exception, "root not yet set" );
+      std::lock_guard g( my->mtx );
+      my->advance_root_impl( id );
+   }
 
-      auto new_root = get_block( id );
+   void fork_database_impl::advance_root_impl( const block_id_type& id ) {
+      EOS_ASSERT( root, fork_database_exception, "root not yet set" );
+
+      auto new_root = get_block_impl( id );
       EOS_ASSERT( new_root, fork_database_exception,
                   "cannot advance root to a block that does not exist in the fork database" );
       EOS_ASSERT( new_root->is_valid(), fork_database_exception,
@@ -262,48 +308,53 @@ namespace eosio { namespace chain {
       deque<block_id_type> blocks_to_remove;
       for( auto b = new_root; b; ) {
          blocks_to_remove.emplace_back( b->header.previous );
-         b = get_block( blocks_to_remove.back() );
-         EOS_ASSERT( b || blocks_to_remove.back() == my->root->id, fork_database_exception, "invariant violation: orphaned branch was present in forked database" );
+         b = get_block_impl( blocks_to_remove.back() );
+         EOS_ASSERT( b || blocks_to_remove.back() == root->id, fork_database_exception, "invariant violation: orphaned branch was present in forked database" );
       }
 
       // The new root block should be erased from the fork database index individually rather than with the remove method,
       // because we do not want the blocks branching off of it to be removed from the fork database.
-      my->index.erase( my->index.find( id ) );
+      index.erase( index.find( id ) );
 
       // The other blocks to be removed are removed using the remove method so that orphaned branches do not remain in the fork database.
       for( const auto& block_id : blocks_to_remove ) {
-         remove( block_id );
+         remove_impl( block_id );
       }
 
       // Even though fork database no longer needs block or trxs when a block state becomes a root of the tree,
       // avoid mutating the block state at all, for example clearing the block shared pointer, because other
       // parts of the code which run asynchronously may later expect it remain unmodified.
 
-      my->root = new_root;
+      root = new_root;
    }
 
    block_header_state_ptr fork_database::get_block_header( const block_id_type& id )const {
-      if( my->root->id == id ) {
-         return my->root;
+      std::shared_lock g( my->mtx );
+      return my->get_block_header_impl( id );
+   }
+
+   block_header_state_ptr fork_database_impl::get_block_header_impl( const block_id_type& id )const {
+      if( root->id == id ) {
+         return root;
       }
 
-      auto itr = my->index.find( id );
-      if( itr != my->index.end() )
+      auto itr = index.find( id );
+      if( itr != index.end() )
          return *itr;
 
       return block_header_state_ptr();
    }
 
-   void fork_database_impl::add( const block_state_ptr& n,
-                                 bool ignore_duplicate, bool validate,
-                                 const std::function<void( block_timestamp_type,
-                                                           const flat_set<digest_type>&,
-                                                           const vector<digest_type>& )>& validator )
+   void fork_database_impl::add_impl( const block_state_ptr& n,
+                                      bool ignore_duplicate, bool validate,
+                                      const std::function<void( block_timestamp_type,
+                                                                const flat_set<digest_type>&,
+                                                                const vector<digest_type>& )>& validator )
    {
       EOS_ASSERT( root, fork_database_exception, "root not yet set" );
       EOS_ASSERT( n, fork_database_exception, "attempt to add null block state" );
 
-      auto prev_bh = self.get_block_header( n->header.previous );
+      auto prev_bh = get_block_header_impl( n->header.previous );
 
       EOS_ASSERT( prev_bh, unlinkable_block_exception,
                   "unlinkable block", ("id", n->id)("previous", n->header.previous) );
@@ -332,19 +383,27 @@ namespace eosio { namespace chain {
    }
 
    void fork_database::add( const block_state_ptr& n, bool ignore_duplicate ) {
-      my->add( n, ignore_duplicate, false,
-               []( block_timestamp_type timestamp,
-                   const flat_set<digest_type>& cur_features,
-                   const vector<digest_type>& new_features )
-               {}
+      std::lock_guard g( my->mtx );
+      my->add_impl( n, ignore_duplicate, false,
+                    []( block_timestamp_type timestamp,
+                        const flat_set<digest_type>& cur_features,
+                        const vector<digest_type>& new_features )
+                    {}
       );
    }
 
-   const block_state_ptr& fork_database::root()const { return my->root; }
+   block_state_ptr fork_database::root()const {
+      std::shared_lock g( my->mtx );
+      return my->root;
+   }
 
-   const block_state_ptr& fork_database::head()const { return my->head; }
+   block_state_ptr fork_database::head()const {
+      std::shared_lock g( my->mtx );
+      return my->head;
+   }
 
    block_state_ptr fork_database::pending_head()const {
+      std::shared_lock g( my->mtx );
       const auto& indx = my->index.get<by_lib_block_num>();
 
       auto itr = indx.lower_bound( false );
@@ -357,8 +416,13 @@ namespace eosio { namespace chain {
    }
 
    branch_type fork_database::fetch_branch( const block_id_type& h, uint32_t trim_after_block_num )const {
+      std::shared_lock g( my->mtx );
+      return my->fetch_branch_impl( h, trim_after_block_num );
+   }
+
+   branch_type fork_database_impl::fetch_branch_impl( const block_id_type& h, uint32_t trim_after_block_num )const {
       branch_type result;
-      for( auto s = get_block(h); s; s = get_block( s->header.previous ) ) {
+      for( auto s = get_block_impl(h); s; s = get_block_impl( s->header.previous ) ) {
          if( s->block_num <= trim_after_block_num )
              result.push_back( s );
       }
@@ -367,7 +431,12 @@ namespace eosio { namespace chain {
    }
 
    block_state_ptr fork_database::search_on_branch( const block_id_type& h, uint32_t block_num )const {
-      for( auto s = get_block(h); s; s = get_block( s->header.previous ) ) {
+      std::shared_lock g( my->mtx );
+      return my->search_on_branch_impl( h, block_num );
+   }
+
+   block_state_ptr fork_database_impl::search_on_branch_impl( const block_id_type& h, uint32_t block_num )const {
+      for( auto s = get_block_impl(h); s; s = get_block_impl( s->header.previous ) ) {
          if( s->block_num == block_num )
              return s;
       }
@@ -381,9 +450,15 @@ namespace eosio { namespace chain {
     */
    pair< branch_type, branch_type >  fork_database::fetch_branch_from( const block_id_type& first,
                                                                        const block_id_type& second )const {
+      std::shared_lock g( my->mtx );
+      return my->fetch_branch_from_impl( first, second );
+   }
+
+   pair< branch_type, branch_type >  fork_database_impl::fetch_branch_from_impl( const block_id_type& first,
+                                                                                 const block_id_type& second )const {
       pair<branch_type,branch_type> result;
-      auto first_branch = (first == my->root->id) ? my->root : get_block(first);
-      auto second_branch = (second == my->root->id) ? my->root : get_block(second);
+      auto first_branch = (first == root->id) ? root : get_block_impl(first);
+      auto second_branch = (second == root->id) ? root : get_block_impl(second);
 
       EOS_ASSERT(first_branch, fork_db_block_not_found, "block ${id} does not exist", ("id", first));
       EOS_ASSERT(second_branch, fork_db_block_not_found, "block ${id} does not exist", ("id", second));
@@ -392,7 +467,7 @@ namespace eosio { namespace chain {
       {
          result.first.push_back(first_branch);
          const auto& prev = first_branch->header.previous;
-         first_branch = (prev == my->root->id) ? my->root : get_block( prev );
+         first_branch = (prev == root->id) ? root : get_block_impl( prev );
          EOS_ASSERT( first_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", prev)
@@ -403,7 +478,7 @@ namespace eosio { namespace chain {
       {
          result.second.push_back( second_branch );
          const auto& prev = second_branch->header.previous;
-         second_branch = (prev == my->root->id) ? my->root : get_block( prev );
+         second_branch = (prev == root->id) ? root : get_block_impl( prev );
          EOS_ASSERT( second_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", prev)
@@ -417,9 +492,9 @@ namespace eosio { namespace chain {
          result.first.push_back(first_branch);
          result.second.push_back(second_branch);
          const auto &first_prev = first_branch->header.previous;
-         first_branch = get_block( first_prev );
+         first_branch = get_block_impl( first_prev );
          const auto &second_prev = second_branch->header.previous;
-         second_branch = get_block( second_prev );
+         second_branch = get_block_impl( second_prev );
          EOS_ASSERT( first_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", first_prev)
@@ -436,13 +511,18 @@ namespace eosio { namespace chain {
          result.second.push_back(second_branch);
       }
       return result;
-   } /// fetch_branch_from
+   } /// fetch_branch_from_impl
 
    /// remove all of the invalid forks built off of this id including this id
    void fork_database::remove( const block_id_type& id ) {
+      std::lock_guard g( my->mtx );
+      return my->remove_impl( id );
+   }
+
+   void fork_database_impl::remove_impl( const block_id_type& id ) {
       deque<block_id_type> remove_queue{id};
-      const auto& previdx = my->index.get<by_prev>();
-      const auto& head_id = my->head->id;
+      const auto& previdx = index.get<by_prev>();
+      const auto& head_id = head->id;
 
       for( uint32_t i = 0; i < remove_queue.size(); ++i ) {
          EOS_ASSERT( remove_queue[i] != head_id, fork_database_exception,
@@ -456,16 +536,19 @@ namespace eosio { namespace chain {
       }
 
       for( const auto& block_id : remove_queue ) {
-         auto itr = my->index.find( block_id );
-         if( itr != my->index.end() )
-            my->index.erase(itr);
+         index.erase( block_id );
       }
    }
 
    void fork_database::mark_valid( const block_state_ptr& h ) {
+      std::lock_guard g( my->mtx );
+      my->mark_valid_impl( h );
+   }
+
+   void fork_database_impl::mark_valid_impl( const block_state_ptr& h ) {
       if( h->validated ) return;
 
-      auto& by_id_idx = my->index.get<by_block_id>();
+      auto& by_id_idx = index.get<by_block_id>();
 
       auto itr = by_id_idx.find( h->id );
       EOS_ASSERT( itr != by_id_idx.end(), fork_database_exception,
@@ -476,15 +559,20 @@ namespace eosio { namespace chain {
          bsp->validated = true;
       } );
 
-      auto candidate = my->index.get<by_lib_block_num>().begin();
-      if( first_preferred( **candidate, *my->head ) ) {
-         my->head = *candidate;
+      auto candidate = index.get<by_lib_block_num>().begin();
+      if( first_preferred( **candidate, *head ) ) {
+         head = *candidate;
       }
    }
 
-   block_state_ptr   fork_database::get_block(const block_id_type& id)const {
-      auto itr = my->index.find( id );
-      if( itr != my->index.end() )
+   block_state_ptr fork_database::get_block(const block_id_type& id)const {
+      std::shared_lock g( my->mtx );
+      return my->get_block_impl(id);
+   }
+
+   block_state_ptr fork_database_impl::get_block_impl(const block_id_type& id)const {
+      auto itr = index.find( id );
+      if( itr != index.end() )
          return *itr;
       return block_state_ptr();
    }

libraries/chain/include/eosio/chain/controller.hpp

@@ -84,7 +84,7 @@ namespace eosio { namespace chain {
             bool                     allow_ram_billing_in_notify = false;
             uint32_t                 maximum_variable_signature_length = chain::config::default_max_variable_signature_length;
             bool                     disable_all_subjective_mitigations = false; //< for developer & testing purposes, can be configured using `disable-all-subjective-mitigations` when `EOSIO_DEVELOPER` build option is provided
-            uint32_t                 terminate_at_block     = 0; //< primarily for testing purposes
+            uint32_t                 terminate_at_block     = 0;
 
             wasm_interface::vm_type  wasm_runtime = chain::config::default_wasm_runtime;
             eosvmoc::config          eosvmoc_config;
@@ -166,14 +166,17 @@ namespace eosio { namespace chain {
          void sign_block( const signer_callback_type& signer_callback );
          void commit_block();
 
+         // thread-safe
          std::future<block_state_ptr> create_block_state_future( const block_id_type& id, const signed_block_ptr& b );
+         // thread-safe
+         block_state_ptr create_block_state( const block_id_type& id, const signed_block_ptr& b ) const;
 
          /**
-          * @param block_state_future provide from call to create_block_state_future
+          * @param bsp block to push
           * @param cb calls cb with forked applied transactions for each forked block
           * @param trx_lookup user provided lookup function for externally cached transaction_metadata
           */
-         void push_block( std::future<block_state_ptr>& block_state_future,
+         void push_block( const block_state_ptr& bsp,
                           const forked_branch_callback& cb,
                           const trx_meta_cache_lookup& trx_lookup );
 
@@ -244,6 +247,7 @@ namespace eosio { namespace chain {
          signed_block_ptr fetch_block_by_id( block_id_type id )const;
 
          block_state_ptr fetch_block_state_by_number( uint32_t block_num )const;
+         // return block_state from forkdb, thread-safe
          block_state_ptr fetch_block_state_by_id( block_id_type id )const;
 
          block_id_type get_block_id_for_num( uint32_t block_num )const;

libraries/chain/include/eosio/chain/fork_database.hpp

@@ -16,6 +16,8 @@ namespace eosio { namespace chain {
     * database tracks the longest chain and the last irreversible block number. All
     * blocks older than the last irreversible block are freed after emitting the
     * irreversible signal.
+    *
+    * An internal mutex is used to provide thread-safety.
     */
    class fork_database {
       public:
@@ -55,9 +57,9 @@ namespace eosio { namespace chain {
 
          void            remove( const block_id_type& id );
 
-         const block_state_ptr& root()const;
-         const block_state_ptr& head()const;
-         block_state_ptr        pending_head()const;
+         block_state_ptr root()const;
+         block_state_ptr head()const;
+         block_state_ptr pending_head()const;
 
          /**
           *  Returns the sequence of block states resulting from trimming the branch from the

libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp

@@ -96,15 +96,15 @@ public:
       return itr->trx_meta;
    }
 
-   template <typename Func>
-   bool clear_expired( const time_point& pending_block_time, const time_point& deadline, Func&& callback ) {
+   template <typename Yield, typename Callback>
+   bool clear_expired( const time_point& pending_block_time, Yield&& yield, Callback&& callback ) {
       auto& persisted_by_expiry = queue.get<by_expiry>();
       while( !persisted_by_expiry.empty() ) {
          const auto& itr = persisted_by_expiry.begin();
          if( itr->expiry > pending_block_time ) {
             break;
          }
-         if( deadline <= fc::time_point::now() ) {
+         if( yield() ) {
             return false;
          }
          callback( itr->trx_meta->packed_trx(), itr->trx_type );

libraries/chain/webassembly/runtimes/eos-vm.cpp

@@ -247,7 +247,7 @@ std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instan
       wasm_code_ptr code((uint8_t*)code_bytes, code_size);
       apply_options options = { .max_pages = 65536,
                                 .max_call_depth = 0 };
-      std::unique_ptr<backend_t> bkend = std::make_unique<backend_t>(code, code_size, nullptr, options);
+      std::unique_ptr<backend_t> bkend = std::make_unique<backend_t>(code, code_size, nullptr, options, false); // uses 2-passes parsing
       eos_vm_host_functions_t::resolve(bkend->get_module());
       return std::make_unique<eos_vm_instantiated_module<Impl>>(this, std::move(bkend));
    } catch(eosio::vm::exception& e) {
@@ -273,7 +273,7 @@ std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::inst
       wasm_code_ptr code((uint8_t*)code_bytes, code_size);
       apply_options options = { .max_pages = 65536,
                                 .max_call_depth = 0 };
-      std::unique_ptr<backend_t> bkend = std::make_unique<backend_t>(code, code_size, nullptr, options);
+      std::unique_ptr<backend_t> bkend = std::make_unique<backend_t>(code, code_size, nullptr, options, false); // uses 2-passes parsing
       eos_vm_host_functions_t::resolve(bkend->get_module());
       return std::make_unique<eos_vm_profiling_module>(std::move(bkend), code_bytes, code_size);
    } catch(eosio::vm::exception& e) {

libraries/testing/include/eosio/testing/tester.hpp

@@ -597,7 +597,7 @@ namespace eosio { namespace testing {
       signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override {
          auto sb = _produce_block(skip_time, false);
          auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb );
-         validating_node->push_block( bsf, forked_branch_callback{}, trx_meta_cache_lookup{} );
+         validating_node->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} );
 
          return sb;
       }
@@ -607,15 +607,15 @@ namespace eosio { namespace testing {
       }
 
       void validate_push_block(const signed_block_ptr& sb) {
-         auto bs = validating_node->create_block_state_future( sb->calculate_id(), sb );
-         validating_node->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} );
+         auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb );
+         validating_node->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} );
       }
 
       signed_block_ptr produce_empty_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override {
          unapplied_transactions.add_aborted( control->abort_block() );
          auto sb = _produce_block(skip_time, true);
          auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb );
-         validating_node->push_block( bsf, forked_branch_callback{}, trx_meta_cache_lookup{} );
+         validating_node->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} );
 
          return sb;
       }

libraries/testing/tester.cpp

@@ -322,7 +322,7 @@ namespace eosio { namespace testing {
    void base_tester::push_block(signed_block_ptr b) {
       auto bsf = control->create_block_state_future(b->calculate_id(), b);
       unapplied_transactions.add_aborted( control->abort_block() );
-      control->push_block( bsf, [this]( const branch_type& forked_branch ) {
+      control->push_block( bsf.get(), [this]( const branch_type& forked_branch ) {
          unapplied_transactions.add_forked( forked_branch );
       }, [this]( const transaction_id_type& id ) {
          return unapplied_transactions.get_trx( id );
@@ -1050,7 +1050,7 @@ namespace eosio { namespace testing {
             if( block ) { //&& !b.control->is_known_block(block->id()) ) {
                auto bsf = b.control->create_block_state_future( block->calculate_id(), block );
                b.control->abort_block();
-               b.control->push_block(bsf, forked_branch_callback{}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block);
+               b.control->push_block(bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block);
             }
          }
       };

plugins/chain_interface/include/eosio/chain/plugin_interface.hpp

@@ -37,14 +37,9 @@ namespace eosio { namespace chain { namespace plugin_interface {
    }
 
    namespace incoming {
-      namespace channels {
-         using block                 = channel_decl<struct block_tag, signed_block_ptr>;
-         using transaction           = channel_decl<struct transaction_tag, packed_transaction_ptr>;
-      }
-
       namespace methods {
-         // synchronously push a block/trx to a single provider
-         using block_sync            = method_decl<chain_plugin_interface, bool(const signed_block_ptr&, const std::optional<block_id_type>&), first_provider_policy>;
+         // synchronously push a block/trx to a single provider, block_state_ptr may be null
+         using block_sync            = method_decl<chain_plugin_interface, bool(const signed_block_ptr&, const std::optional<block_id_type>&, const block_state_ptr&), first_provider_policy>;
          using transaction_async     = method_decl<chain_plugin_interface, void(const packed_transaction_ptr&, bool, bool, bool, next_function<transaction_trace_ptr>), first_provider_policy>;
       }
    }

plugins/chain_plugin/chain_plugin.cpp

@@ -142,7 +142,6 @@ public:
    ,irreversible_block_channel(app().get_channel<channels::irreversible_block>())
    ,accepted_transaction_channel(app().get_channel<channels::accepted_transaction>())
    ,applied_transaction_channel(app().get_channel<channels::applied_transaction>())
-   ,incoming_block_channel(app().get_channel<incoming::channels::block>())
    ,incoming_block_sync_method(app().get_method<incoming::methods::block_sync>())
    ,incoming_transaction_async_method(app().get_method<incoming::methods::transaction_async>())
    {}
@@ -170,7 +169,6 @@ public:
    channels::irreversible_block::channel_type&     irreversible_block_channel;
    channels::accepted_transaction::channel_type&   accepted_transaction_channel;
    channels::applied_transaction::channel_type&    applied_transaction_channel;
-   incoming::channels::block::channel_type&        incoming_block_channel;
 
    // retained references to methods for easy calling
    incoming::methods::block_sync::method_type&        incoming_block_sync_method;
@@ -1333,8 +1331,8 @@ chain_apis::read_only chain_plugin::get_read_only_api() const {
 }
 
 
-bool chain_plugin::accept_block(const signed_block_ptr& block, const block_id_type& id ) {
-   return my->incoming_block_sync_method(block, id);
+bool chain_plugin::accept_block(const signed_block_ptr& block, const block_id_type& id, const block_state_ptr& bsp ) {
+   return my->incoming_block_sync_method(block, id, bsp);
 }
 
 void chain_plugin::accept_transaction(const chain::packed_transaction_ptr& trx, next_function<chain::transaction_trace_ptr> next) {
@@ -2130,7 +2128,7 @@ fc::variant read_only::get_block_header_state(const get_block_header_state_param
 
 void read_write::push_block(read_write::push_block_params&& params, next_function<read_write::push_block_results> next) {
    try {
-      app().get_method<incoming::methods::block_sync>()(std::make_shared<signed_block>(std::move(params)), {});
+      app().get_method<incoming::methods::block_sync>()(std::make_shared<signed_block>( std::move(params) ), std::optional<block_id_type>{}, block_state_ptr{});
    } catch ( boost::interprocess::bad_alloc& ) {
       chain_plugin::handle_db_exhaustion();
    } catch ( const std::bad_alloc& ) {

plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp

@@ -777,7 +777,7 @@ public:
    chain_apis::read_write get_read_write_api();
    chain_apis::read_only get_read_only_api() const;
 
-   bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id );
+   bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id, const chain::block_state_ptr& bsp );
    void accept_transaction(const chain::packed_transaction_ptr& trx, chain::plugin_interface::next_function<chain::transaction_trace_ptr> next);
 
    // Only call this after plugin_initialize()!

plugins/http_plugin/http_plugin.cpp

@@ -912,6 +912,7 @@ class http_plugin_impl : public std::enable_shared_from_this<http_plugin_impl> {
       my->url_handlers.clear();
 
       app().post( 0, [me = my](){} ); // keep my pointer alive until queue is drained
+      fc_ilog( logger, "exit shutdown");
    }
 
    void http_plugin::add_handler(const string& url, const url_handler& handler, int priority) {

plugins/net_plugin/net_plugin.cpp

@@ -26,6 +26,7 @@
 #include <boost/asio/steady_timer.hpp>
 
 #include <atomic>
+#include <cmath>
 #include <shared_mutex>
 
 using namespace eosio::chain::plugin_interface;
@@ -785,7 +786,7 @@ namespace eosio {
       void handle_message( const packed_transaction& msg ) = delete; // packed_transaction_ptr overload used instead
       void handle_message( packed_transaction_ptr msg );
 
-      void process_signed_block( const block_id_type& id, signed_block_ptr msg );
+      void process_signed_block( const block_id_type& id, signed_block_ptr msg, block_state_ptr bsp );
 
       fc::variant_object get_logger_variant() const {
          fc::mutable_variant_object mvo;
@@ -1776,7 +1777,6 @@ namespace eosio {
       if( c->is_transactions_only_connection() ) return;
 
       uint32_t lib_num = 0;
-      uint32_t peer_lib = msg.last_irreversible_block_num;
       uint32_t head = 0;
       block_id_type head_id;
       std::tie( lib_num, std::ignore, head,
@@ -1790,10 +1790,10 @@ namespace eosio {
          peer_wlog(c, "Peer sent a handshake with a timestamp skewed by at least ${t}ms", ("t", network_latency_ns/1000000));
          network_latency_ns = 0;
       }
-      // number of blocks syncing node is behind from a peer node
-      uint32_t nblk_behind_by_net_latency = static_cast<uint32_t>(network_latency_ns / block_interval_ns);
-      // 2x for time it takes for message to reach back to peer node, +1 to compensate for integer division truncation
-      uint32_t nblk_combined_latency = 2 * nblk_behind_by_net_latency + 1;
+      // number of blocks syncing node is behind from a peer node, round up
+      uint32_t nblk_behind_by_net_latency = std::lround( static_cast<double>(network_latency_ns) / static_cast<double>(block_interval_ns) );
+      // 2x for time it takes for message to reach back to peer node
+      uint32_t nblk_combined_latency = 2 * nblk_behind_by_net_latency;
       // message in the log below is used in p2p_high_latency_test.py test
       peer_dlog(c, "Network latency is ${lat}ms, ${num} blocks discrepancy by network latency, ${tot_num} blocks discrepancy expected once message received",
                 ("lat", network_latency_ns/1000000)("num", nblk_behind_by_net_latency)("tot_num", nblk_combined_latency));
@@ -1813,8 +1813,8 @@ namespace eosio {
       //-----------------------------
 
       if (head_id == msg.head_id) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 0",
-                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 0, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))("l", lib_num) );
          c->syncing = false;
          notice_message note;
          note.known_blocks.mode = none;
@@ -1823,9 +1823,10 @@ namespace eosio {
          c->enqueue( note );
          return;
       }
-      if (head < peer_lib) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 1",
-                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+      if (head < msg.last_irreversible_block_num) {
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 1, head ${h}, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))
+                    ("h", head)("l", lib_num) );
          c->syncing = false;
          if (c->sent_handshake_count > 0) {
             c->send_handshake();
@@ -1833,8 +1834,9 @@ namespace eosio {
          return;
       }
       if (lib_num > msg.head_num + nblk_combined_latency) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 2",
-                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 2, head ${h}, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))
+                    ("h", head)("l", lib_num) );
          if (msg.generation > 1 || c->protocol_version > proto_base) {
             notice_message note;
             note.known_trx.pending = lib_num;
@@ -1848,14 +1850,16 @@ namespace eosio {
       }
 
       if (head + nblk_combined_latency < msg.head_num ) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 3",
-                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 3, head ${h}, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))
+                    ("h", head)("l", lib_num) );
          c->syncing = false;
          verify_catchup(c, msg.head_num, msg.head_id);
          return;
       } else if(head >= msg.head_num + nblk_combined_latency) {
-         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 4",
-                  ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16)) );
+         peer_ilog( c, "handshake lib ${lib}, head ${head}, head id ${id}.. sync 4, head ${h}, lib ${l}",
+                    ("lib", msg.last_irreversible_block_num)("head", msg.head_num)("id", msg.head_id.str().substr(8,16))
+                    ("h", head)("l", lib_num) );
          if (msg.generation > 1 ||  c->protocol_version > proto_base) {
             notice_message note;
             note.known_trx.mode = none;
@@ -2341,14 +2345,6 @@ namespace eosio {
 
    // called from connection strand
    void connection::connect( const std::shared_ptr<tcp::resolver>& resolver, tcp::resolver::results_type endpoints ) {
-      switch ( no_retry ) {
-         case no_reason:
-         case wrong_version:
-         case benign_other:
-            break;
-         default:
-            return;
-      }
       connecting = true;
       pending_message_buffer.reset();
       buffer_queue.clear_out_queue();
@@ -3143,13 +3139,44 @@ namespace eosio {
    // called from connection strand
    void connection::handle_message( const block_id_type& id, signed_block_ptr ptr ) {
       peer_dlog( this, "received signed_block ${num}, id ${id}", ("num", ptr->block_num())("id", id) );
-      app().post(priority::medium, [ptr{std::move(ptr)}, id, c = shared_from_this()]() mutable {
-         c->process_signed_block( id, std::move( ptr ) );
+
+      controller& cc = my_impl->chain_plug->chain();
+      block_state_ptr bsp;
+      bool exception = false;
+      try {
+         if( cc.fetch_block_state_by_id( id ) ) {
+            my_impl->dispatcher->add_peer_block( id, connection_id );
+            my_impl->sync_master->sync_recv_block( shared_from_this(), id, ptr->block_num(), false );
+            return;
+         }
+         // this may return null if block is not immediately ready to be processed
+         bsp = cc.create_block_state( id, ptr );
+      } catch( const fc::exception& ex ) {
+         exception = true;
+         peer_elog(this, "bad block exception: #${n} ${id}...: ${m}",
+                   ("n", ptr->block_num())("id", id.str().substr(8,16))("m",ex.to_string()));
+      } catch( ... ) {
+         exception = true;
+         peer_elog(this, "bad block: #${n} ${id}...: unknown exception",
+                   ("n", ptr->block_num())("id", id.str().substr(8,16)));
+      }
+      if( exception ) {
+         my_impl->sync_master->rejected_block( shared_from_this(), ptr->block_num() );
+         my_impl->dispatcher->rejected_block( id );
+         return;
+      }
+
+      bool signal_producer = !!bsp; // ready to process immediately, so signal producer to interrupt start_block
+      app().post(priority::medium, [ptr{std::move(ptr)}, bsp{std::move(bsp)}, id, c = shared_from_this()]() mutable {
+         c->process_signed_block( id, std::move(ptr), std::move(bsp) );
       });
+
+      if( signal_producer )
+         my_impl->producer_plug->received_block();
    }
 
    // called from application thread
-   void connection::process_signed_block( const block_id_type& blk_id, signed_block_ptr msg ) {
+   void connection::process_signed_block( const block_id_type& blk_id, signed_block_ptr msg, block_state_ptr bsp ) {
       controller& cc = my_impl->chain_plug->chain();
       uint32_t blk_num = msg->block_num();
       // use c in this method instead of this to highlight that all methods called on c-> must be thread safe
@@ -3179,7 +3206,7 @@ namespace eosio {
 
       go_away_reason reason = fatal_other;
       try {
-         bool accepted = my_impl->chain_plug->accept_block(msg, blk_id);
+         bool accepted = my_impl->chain_plug->accept_block(msg, blk_id, bsp);
          my_impl->update_chain_info();
          if( !accepted ) return;
          reason = no_reason;

plugins/producer_api_plugin/producer.swagger.yaml

@@ -7,11 +7,12 @@ info:
     name: MIT
     url: https://opensource.org/licenses/MIT
   contact:
-    url: https://eosnetwork.com
+    url: https://antelope.io
 tags:
-  - name: eosio
+  - name: Protocol Version 3.1
+    description: The release tag for Leap binaries is also the protocol version
 servers:
-  - url: "{protocol}://{host}:{port}/v1/"
+  - url: "{protocol}://{host}:{port}/v1"
     variables:
       protocol:
         enum:
@@ -22,285 +23,223 @@ servers:
         default: localhost
       port:
         default: "8080"
-components:
-  securitySchemes: {}
-  schemas: {}
 security:
   - {}
 paths:
   /producer/pause:
     post:
       summary: pause
-      description: Pause producer node
+      description: Pause producer node. Takes no arguments and returns no values.
       operationId: pause
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                type: boolean
-                description: "returns status"
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/resume:
     post:
       summary: resume
-      description: Resume producer node
+      description: Resume producer node. Takes no arguments and returns no values.
       operationId: resume
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              description: Resumes activity for producer
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/paused:
     post:
       summary: paused
-      description: Retreives paused status for producer node
+      description: Retrieves paused status for producer node. Takes no arguments and returns no values.
       operationId: paused
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                type: boolean
-                description: True if producer is paused, false otherwise
+                $ref: '#/components/schemas/OK_PAUSED'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/get_runtime_options:
     post:
       summary: get_runtime_options
-      description: Retreives run time options for producer node
+      description: Retrieves runtime options for producer node.
       operationId: get_runtime_options
-      parameters: []
+      responses:
+        "201":
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Runtime_Options'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
+  /producer/update_runtime_options:
+    post:
+      summary: update_runtime_options
+      description: Update runtime options for producer node. May post any of the runtime options in combination or alone.
+      operationId: update_runtime_options
       requestBody:
         content:
           application/json:
             schema:
-              type: object
-              properties: {}
+              $ref: '#/components/schemas/Runtime_Options'
       responses:
-        "200":
+        "201":
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
+  /producer/get_greylist:
+    post:
+      summary: get_greylist
+      description: Retrieves the greylist for producer node.
+      operationId: get_greylist
+      responses:
+        "201":
           description: OK
           content:
             application/json:
               schema:
                 type: object
-                description: Returns run time options set for the producer
                 properties:
-                  max_transaction_time:
-                    type: integer
-                    description: Max transaction time
-                  max_irreversible_block_age:
-                    type: integer
-                    description: Max irreversible block age
-                  produce_time_offset_us:
-                    type: integer
-                    description: Time offset
-                  last_block_time_offset_us:
-                    type: integer
-                    description: Last block time offset
-                  max_scheduled_transaction_time_per_block_ms:
-                    type: integer
-                    description: Max scheduled transaction time per block in ms
-                  subjective_cpu_leeway_us:
-                    type: integer
-                    description: Subjective CPU leeway
-                  incoming_defer_ratio:
-                    type: integer
-                    description: Incoming defer ration
-  /producer/update_runtime_options:
-    post:
-      summary: update_runtime_options
-      description: Update run time options for producer node
-      operationId: update_runtime_options
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              required:
-                - options
-              properties:
-                options:
-                  type: object
-                  description: Defines the run time options to set for the producer
-                  properties:
-                    max_transaction_time:
-                      type: integer
-                      description: Max transaction time
-                    max_irreversible_block_age:
-                      type: integer
-                      description: Max irreversible block age
-                    produce_time_offset_us:
-                      type: integer
-                      description: Time offset
-                    last_block_time_offset_us:
-                      type: integer
-                      description: Last block time offset
-                    max_scheduled_transaction_time_per_block_ms:
-                      type: integer
-                      description: Max scheduled transaction time per block in ms
-                    subjective_cpu_leeway_us:
-                      type: integer
-                      description: Subjective CPU leeway
-                    incoming_defer_ratio:
-                      type: integer
-                      description: Incoming defer ration
-
-      responses:
-        "200":
-          description: OK
+                  accounts:
+                    type: array
+                    description: Array of account names stored in the greylist
+                    items:
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+        "400":
+          description: client error
           content:
             application/json:
               schema:
-                description: Returns Nothing
-  /producer/get_greylist:
-    post:
-      summary: get_greylist
-      description: Retreives the greylist for producer node
-      operationId: get_greylist
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                type: array
-                description: List of account names stored in the greylist
-                items:
-                  $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                $ref: '#/components/schemas/Error'
 
   /producer/add_greylist_accounts:
     post:
       summary: add_greylist_accounts
-      description: Adds accounts to grey list for producer node
+      description: Adds accounts to greylist for producer node. At least one account is required.
       operationId: add_greylist_accounts
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - params
               properties:
-                params:
-                  type: object
-                  properties:
-                    accounts:
-                      type: array
-                      description: List of account names to add
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-
+                  accounts:
+                    type: array
+                    description: List of account names to add
+                    items:
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/remove_greylist_accounts:
     post:
       summary: remove_greylist_accounts
-      description: Removes accounts from greylist for producer node
+      description: Removes accounts from greylist for producer node. At least one account is required.
       operationId: remove_greylist_accounts
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - params
               properties:
-                params:
-                  type: object
-                  properties:
-                    accounts:
-                      type: array
-                      description: List of account names to remove
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-
+                accounts:
+                  type: array
+                  description: List of account names to remove
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                type: array
-                description: List of account names stored in the greylist
-                items:
-                  $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/get_whitelist_blacklist:
     post:
       summary: get_whitelist_blacklist
-      description: Retreives the white list and black list for producer node
+      description: Retrieves the whitelist and blacklist for producer node. A JSON object containing whitelist and blacklist information.
       operationId: get_whitelist_blacklist
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
-
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
                 type: object
-                description: Defines the actor whitelist and blacklist, the contract whitelist and blacklist, the action blacklist and key blacklist
                 properties:
                   actor_whitelist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
                   actor_blacklist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
                   contract_whitelist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
                   contract_blacklist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
                   action_blacklist:
                     type: array
                     items:
@@ -308,116 +247,118 @@ paths:
                       description: Array of two string values, the account name as the first and action name as the second
                       items:
                         allOf:
-                          - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                          - $ref: "https://eosio.github.io/schemata/v2.0/oas/CppSignature.yaml"
+                          - $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                          - $ref: "https://docs.eosnetwork.com/openapi/v2.0/CppSignature.yaml"
                   key_blacklist:
                     type: array
                     items:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/KeyType.yaml"
+                      - $ref: "https://docs.eosnetwork.com/openapi/v2.0/KeyType.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/set_whitelist_blacklist:
     post:
       summary: set_whitelist_blacklist
-      description: Sets the white list and black list for producer node
+      description: Defines the whitelist and blacklist for a producer node. Takes a JSON object containing whitelist and blacklist information.
       operationId: set_whitelist_blacklist
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - params
               properties:
-                params:
-                  type: object
-                  description: Defines the actor whitelist and blacklist, the contract whitelist and blacklist, the action blacklist and key blacklist
-                  properties:
-                    actor_whitelist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                    actor_blacklist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                    contract_whitelist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                    contract_blacklist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                    action_blacklist:
-                      type: array
-                      items:
-                        type: array
-                        description: Array of two string values, the account name as the first and action name as the second
-                        items:
-                          allOf:
-                            - $ref: "https://eosio.github.io/schemata/v2.0/oas/Name.yaml"
-                            - $ref: "https://eosio.github.io/schemata/v2.0/oas/CppSignature.yaml"
-                    key_blacklist:
-                      type: array
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/KeyType.yaml"
-
+                actor_whitelist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                actor_blacklist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                contract_whitelist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                contract_blacklist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                action_blacklist:
+                  type: array
+                  items:
+                    type: array
+                    description: Array of two string values, the account name as the first and action name as the second
+                    items:
+                      anyOf:
+                        - $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                        - $ref: "https://docs.eosnetwork.com/openapi/v2.0/CppSignature.yaml"
+                key_blacklist:
+                  type: array
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/KeyType.yaml"
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
   /producer/create_snapshot:
     post:
       summary: create_snapshot
-      description: Creates a snapshot for producer node
+      description: Creates a snapshot for producer node. Returns error when unable to create a snapshot.
       operationId: create_snapshot
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              required:
-                - next
-              properties:
-                next:
-                  type: object
-                  description: Defines the snapshot to be created
-                  properties:
-                    snapshot_name:
-                      type: string
-                      description: The name of the snapshot
-                    head_block_id:
-                      $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml"
-
-
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                type: object
+                properties:
+                  head_block_id:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                  head_block_num:
+                    type: integer
+                    description: Highest block number on the chain
+                    example: 5102
+                  head_block_time:
+                    type: string
+                    description: Highest block unix timestamp
+                    example: 2020-11-16T00:00:00.000
+                  version:
+                    type: integer
+                    description: version number
+                    example: 6
+                  snapshot_name:
+                    type: string
+                    description: The path and file name of the snapshot
+                    example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/get_integrity_hash:
     post:
       summary: get_integrity_hash
-      description: Retreives the integrity hash for producer node
+      description: Retrieves the integrity hash for producer node
       operationId: get_integrity_hash
-      parameters: []
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties: {}
-
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
@@ -425,69 +366,67 @@ paths:
                 type: object
                 description: Defines the integrity hash information details
                 properties:
-                  integrity_hash:
-                    $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml"
                   head_block_id:
-                    $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml"
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                  integrity_hash:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/schedule_protocol_feature_activations:
     post:
       summary: schedule_protocol_feature_activations
-      description: Schedule protocol feature activation for producer node
+      description: Schedule protocol feature activation for producer node. Note some features may require pre-activation. Will return error for duplicate requests or when feature required pre-activation.
       operationId: schedule_protocol_feature_activations
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - schedule
               properties:
-                schedule:
-                  type: object
-                  properties:
-                    protocol_features_to_activate:
-                      type: array
-                      description: List of protocol features to activate
-                      items:
-                        $ref: "https://eosio.github.io/schemata/v2.0/oas/Sha256.yaml"
-
+                protocol_features_to_activate:
+                  type: array
+                  description: List of protocol features to activate
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
               schema:
-                description: Returns Nothing
+                $ref: '#/components/schemas/OK'
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
 
   /producer/get_supported_protocol_features:
     post:
       summary: get_supported_protocol_features
-      description: Retreives supported protocol features for producer node
+      description: Retrieves supported protocol features for producer node. Pass filters in as part of the request body.
       operationId: get_supported_protocol_features
-      parameters: []
       requestBody:
         content:
           application/json:
             schema:
               type: object
-              required:
-                - params
               properties:
-                params:
-                  type: object
-                  description: Defines filters based on which to return the supported protocol features
-                  properties:
-                    exclude_disabled:
-                      type: boolean
-                      description: Exclude disabled protocol features
-                    exclude_unactivatable:
-                      type: boolean
-                      description: Exclude unactivatable protocol features
-
+                exclude_disabled:
+                  type: boolean
+                  description: Exclude disabled protocol features
+                exclude_unactivatable:
+                  type: boolean
+                  description: Exclude unactivatable protocol features
+                  example: false
       responses:
-        "200":
+        "201":
           description: OK
           content:
             application/json:
@@ -495,4 +434,196 @@ paths:
                 type: array
                 description: Variant type, an array of strings with the supported protocol features
                 items:
-                  type: string
+                  type: object
+                  properties:
+                    feature_digest:
+                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                    subjective_restrictions:
+                      type: object
+                      properties:
+                        enabled:
+                          type: boolean
+                          example: true
+                        preactivation_required:
+                          type: boolean
+                          example: true
+                        earliest_allowed_activation_time:
+                          type: string
+                          example: "1970-01-01T00:00:00.000"
+                        description_digest:
+                          $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                        dependencies:
+                          type: array
+                          items:
+                            $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                        protocol_feature_type:
+                          type: string
+                          example: "builtin"
+                        specification:
+                          type: array
+                          items:
+                            type: object
+                            properties:
+                              name:
+                                type: string
+                              value:
+                                type: string
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
+
+  /producer/get_account_ram_corrections:
+    post:
+      summary: get_account_ram_corrections
+      description: Retrieves accounts with ram corrections.
+      operationId: get_account_ram_corrections
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                lower_bound:
+                  type: integer
+                  description: lowest account key
+                upper_bound:
+                  type: integer
+                  description: highest account key
+                limit:
+                  type: integer
+                  description: number of rows to scan
+                  example: 10
+                reverse:
+                  type: boolean
+                  description: direction of search
+                  example: false
+      responses:
+        "201":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                  - rows
+                properties:
+                  rows:
+                    type: array
+                    items:
+                      type: string
+                  more:
+                    type: array
+                    items:
+                      - $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+
+components:
+  securitySchemes: {}
+  schemas:
+    Error:
+      type: object
+      properties:
+        code:
+          type: integer
+          description: http return code
+          example: 400
+        message:
+          type: string
+          description: summary of error
+          example: Invalid Request
+        error:
+          type: object
+          description: details on the error
+          properties:
+            code:
+              type: integer
+              description: internal error code
+              example: 3200006
+            name:
+              type: string
+              description: name of error
+              example: invalid_http_request
+            what:
+              type: string
+              description: prettier version of error name
+              example: invalid http request
+            details:
+              type: array
+              description: list of additional information for debugging
+              items:
+                type: object
+                properties:
+                  message:
+                    type: string
+                    description: debugging message
+                    example: Unable to parse valid input from POST body
+                  file:
+                    type: string
+                    description: file where error was thrown
+                    example: http_plugin.hpp
+                  line_number:
+                    type: integer
+                    description: line number in file where error was thrown
+                    example: 246
+                  method:
+                    type: string
+                    description: function executed when error occurred
+                    example: parse_params
+    OK:
+      type: object
+      properties:
+        result:
+          type: string
+          description: status
+          example: ok
+    OK_PAUSED:
+      type: object
+      properties:
+        json:
+          type: boolean
+          description: true/false indicating paused state
+          example: true
+    Runtime_Options:
+      type: object
+      properties:
+        max_transaction_time:
+          type: integer
+          description: Max transaction time
+          example: 100
+        max_irreversible_block_age:
+          type: integer
+          description: Max irreversible block age
+          example: -1
+        produce_time_offset_us:
+          type: integer
+          description: Time offset
+          example: -100000
+        last_block_time_offset_us:
+          type: integer
+          description: Last block time offset
+          example: -200000
+        max_scheduled_transaction_time_per_block_ms:
+          type: integer
+          description: Max scheduled transaction time per block in ms
+          example: 100
+        subjective_cpu_leeway_us:
+          type: integer
+          description: in micro seconds
+          example: 10
+        incoming_defer_ratio:
+          type: string
+          description: Incoming defer ratio, parsed to double
+          example: "1.00000000000000000"
+        greylist_limit:
+          type: integer
+          description: limit on number of Names supported by greylist
+          example: 1000

plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp

@@ -114,7 +114,10 @@ public:
 
    get_account_ram_corrections_result  get_account_ram_corrections( const get_account_ram_corrections_params& params ) const;
 
-    void log_failed_transaction(const transaction_id_type& trx_id, const chain::packed_transaction_ptr& packed_trx_ptr, const char* reason) const;
+   void log_failed_transaction(const transaction_id_type& trx_id, const chain::packed_transaction_ptr& packed_trx_ptr, const char* reason) const;
+
+   // thread-safe, called when a new block is received
+   void received_block();
 
  private:
    std::shared_ptr<class producer_plugin_impl> my;

plugins/producer_plugin/include/eosio/producer_plugin/subjective_billing.hpp

@@ -180,7 +180,8 @@ public:
       fc_dlog( log, "Subjective billed accounts ${n} removed ${r}", ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
    }
 
-   bool remove_expired( fc::logger& log, const fc::time_point& pending_block_time, const fc::time_point& now, const fc::time_point& deadline ) {
+   template <typename Yield>
+   bool remove_expired( fc::logger& log, const fc::time_point& pending_block_time, const fc::time_point& now, Yield&& yield ) {
       bool exhausted = false;
       auto& idx = _trx_cache_index.get<by_expiry>();
       if( !idx.empty() ) {
@@ -189,7 +190,7 @@ public:
          uint32_t num_expired = 0;
 
          while( !idx.empty() ) {
-            if( deadline <= fc::time_point::now() ) {
+            if( yield() ) {
                exhausted = true;
                break;
             }

plugins/producer_plugin/producer_plugin.cpp

@@ -41,6 +41,13 @@ using std::vector;
 using std::deque;
 using boost::signals2::scoped_connection;
 
+// in release/3.2+ move to fc logger.hpp
+#define fc_tlog( LOGGER, FORMAT, ... ) \
+  FC_MULTILINE_MACRO_BEGIN \
+   if( (LOGGER).is_enabled( fc::log_level::all ) ) \
+      (LOGGER).log( FC_LOG_MESSAGE( all, FORMAT, __VA_ARGS__ ) ); \
+  FC_MULTILINE_MACRO_END
+
 #undef FC_LOG_AND_DROP
 #define LOG_AND_DROP()  \
    catch ( const guard_exception& e ) { \
@@ -197,8 +204,9 @@ public:
    {
    }
 
-   void set_max_failures_per_account( uint32_t max_failures ) {
-       max_failures_per_account = max_failures;
+   void set_max_failures_per_account( uint32_t max_failures, uint32_t size ) {
+      max_failures_per_account = max_failures;
+      reset_window_size_in_num_blocks = size;
    }
 
    void add( const account_name& n, int64_t exception_code ) {
@@ -218,10 +226,24 @@ public:
       return false;
    }
 
-   void report() const {
-      if( _log.is_enabled( fc::log_level::debug ) ) {
+   void report_and_clear(uint32_t block_num) {
+      if (last_reset_block_num != block_num && (block_num % reset_window_size_in_num_blocks == 0) ) {
+         report(block_num);
+         failed_accounts.clear();
+         last_reset_block_num = block_num;
+      }
+   }
+
+   fc::time_point next_reset_timepoint(uint32_t current_block_num, fc::time_point current_block_time) const {
+      auto num_blocks_to_reset = reset_window_size_in_num_blocks - (current_block_num % reset_window_size_in_num_blocks);
+      return current_block_time + fc::milliseconds(num_blocks_to_reset * eosio::chain::config::block_interval_ms);
+   }
+
+private:
+   void report(uint32_t block_num) const {
+      if( _log.is_enabled(fc::log_level::debug)) {
          auto now = fc::time_point::now();
-         for( const auto& e : failed_accounts ) {
+         for ( const auto& e : failed_accounts ) {
             std::string reason;
             if( e.second.is_deadline() ) reason += "deadline";
             if( e.second.is_tx_cpu_usage() ) {
@@ -242,12 +264,6 @@ public:
          }
       }
    }
-
-   void clear() {
-      failed_accounts.clear();
-   }
-
-private:
    struct account_failure {
       enum class ex_fields : uint8_t {
          ex_deadline_exception = 1,
@@ -282,6 +298,8 @@ private:
 
    std::map<account_name, account_failure> failed_accounts;
    uint32_t max_failures_per_account = 3;
+   uint32_t last_reset_block_num = 0;
+   uint32_t reset_window_size_in_num_blocks = 1;
    const eosio::subjective_billing& subjective_billing;
 };
 
@@ -322,6 +340,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       std::optional<named_thread_pool>                          _thread_pool;
 
       std::atomic<int32_t>                                      _max_transaction_time_ms; // modified by app thread, read by net_plugin thread pool
+      std::atomic<bool>                                         _received_block{false}; // modified by net_plugin thread pool and app thread
       fc::microseconds                                          _max_irreversible_block_age_us;
       int32_t                                                   _produce_time_offset_us = 0;
       int32_t                                                   _last_block_time_offset_us = 0;
@@ -339,9 +358,6 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
       chain_plugin* chain_plug = nullptr;
 
-      incoming::channels::block::channel_type::handle         _incoming_block_subscription;
-      incoming::channels::transaction::channel_type::handle   _incoming_transaction_subscription;
-
       compat::channels::transaction_ack::channel_type&        _transaction_ack_channel;
 
       incoming::methods::block_sync::method_type::handle        _incoming_block_sync_provider;
@@ -431,7 +447,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          _subjective_billing.abort_block();
       }
 
-      bool on_incoming_block(const signed_block_ptr& block, const std::optional<block_id_type>& block_id) {
+      bool on_incoming_block(const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
          auto& chain = chain_plug->chain();
          if ( _pending_block_mode == pending_block_mode::producing ) {
             fc_wlog( _log, "dropped incoming block #${num} id: ${id}",
@@ -439,6 +455,11 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
             return false;
          }
 
+         // start a new speculative block, speculative start_block may have been interrupted
+         auto ensure = fc::make_scoped_exit([this](){
+            schedule_production_loop();
+         });
+
          const auto& id = block_id ? *block_id : block->calculate_id();
          auto blk_num = block->block_num();
 
@@ -452,16 +473,14 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          if( existing ) { return false; }
 
          // start processing of block
-         auto bsf = chain.create_block_state_future( id, block );
+         std::future<block_state_ptr> bsf;
+         if( !bsp ) {
+            bsf = chain.create_block_state_future( id, block );
+         }
 
          // abort the pending block
          abort_block();
 
-         // exceptions throw out, make sure we restart our loop
-         auto ensure = fc::make_scoped_exit([this](){
-            schedule_production_loop();
-         });
-
          // push the new block
          auto handle_error = [&](const auto& e)
          {
@@ -471,7 +490,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          };
 
          try {
-            chain.push_block( bsf, [this]( const branch_type& forked_branch ) {
+            const block_state_ptr& bspr = bsp ? bsp : bsf.get();
+            chain.push_block( bspr, [this]( const branch_type& forked_branch ) {
                _unapplied_transactions.add_forked( forked_branch );
             }, [this]( const transaction_id_type& id ) {
                return _unapplied_transactions.get_trx( id );
@@ -673,8 +693,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
             auto first_auth = trx->packed_trx()->get_transaction().first_authorizer();
             if( _account_fails.failure_limit( first_auth ) ) {
                send_response( std::static_pointer_cast<fc::exception>( std::make_shared<tx_cpu_usage_exceeded>(
-                     FC_LOG_MESSAGE( error, "transaction ${id} exceeded failure limit for account ${a}",
-                                     ("id", trx->id())("a", first_auth) ) ) ) );
+                     FC_LOG_MESSAGE( error, "transaction ${id} exceeded failure limit for account ${a} until ${next_reset_time}",
+                                     ("id", trx->id())("a", first_auth)
+                                     ("next_reset_time", _account_fails.next_reset_timepoint(chain.head_block_num(),chain.head_block_time())) ) ) ) );
                return true;
             }
 
@@ -694,7 +715,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
             auto prev_billed_cpu_time_us = trx->billed_cpu_time_us;
             auto trace = chain.push_transaction( trx, block_deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );
-            fc_dlog( _trx_failed_trace_log, "Subjective bill for ${a}: ${b} elapsed ${t}us", ("a",first_auth)("b",sub_bill)("t",trace->elapsed));
+            fc_tlog( _log, "Subjective bill for ${a}: ${b} elapsed ${t}us", ("a",first_auth)("b",sub_bill)("t",trace->elapsed));
             if( trace->except ) {
                if( exception_is_exhausted( *trace->except ) ) {
                   _unapplied_transactions.add_incoming( trx, persist_until_expired, return_failure_traces, next );
@@ -786,6 +807,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          exhausted
       };
 
+      inline bool should_interrupt_start_block( const fc::time_point& deadline ) const;
       start_block_result start_block();
 
       fc::time_point calculate_pending_block_time() const;
@@ -876,7 +898,9 @@ void producer_plugin::set_program_options(
          ("subjective-cpu-leeway-us", boost::program_options::value<int32_t>()->default_value( config::default_subjective_cpu_leeway_us ),
           "Time in microseconds allowed for a transaction that starts with insufficient CPU quota to complete and cover its CPU usage.")
          ("subjective-account-max-failures", boost::program_options::value<uint32_t>()->default_value(3),
-          "Sets the maximum amount of failures that are allowed for a given account per block.")
+          "Sets the maximum amount of failures that are allowed for a given account per window size.")
+         ("subjective-account-max-failures-window-size", boost::program_options::value<uint32_t>()->default_value(1),
+          "Sets the window size in number of blocks for subjective-account-max-failures.")
          ("subjective-account-decay-time-minutes", bpo::value<uint32_t>()->default_value( config::account_cpu_usage_average_window_ms / 1000 / 60 ),
           "Sets the time to return full subjective cpu for accounts")
          ("incoming-defer-ratio", bpo::value<double>()->default_value(1.0),
@@ -1026,7 +1050,13 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
 
    my->_keosd_provider_timeout_us = fc::milliseconds(options.at("keosd-provider-timeout").as<int32_t>());
 
-   my->_account_fails.set_max_failures_per_account( options.at("subjective-account-max-failures").as<uint32_t>() );
+   auto subjective_account_max_failures_window_size = options.at("subjective-account-max-failures-window-size").as<uint32_t>();
+   EOS_ASSERT( subjective_account_max_failures_window_size > 0, plugin_config_exception,
+               "subjective-account-max-failures-window-size ${s} must be greater than 0", ("s", subjective_account_max_failures_window_size) );
+
+   my->_account_fails.set_max_failures_per_account( options.at("subjective-account-max-failures").as<uint32_t>(),
+                                                    subjective_account_max_failures_window_size );
+
 
    my->_produce_time_offset_us = options.at("produce-time-offset-us").as<int32_t>();
    EOS_ASSERT( my->_produce_time_offset_us <= 0 && my->_produce_time_offset_us >= -config::block_interval_us, plugin_config_exception,
@@ -1128,23 +1158,9 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
-   my->_incoming_block_subscription = app().get_channel<incoming::channels::block>().subscribe(
-         [this](const signed_block_ptr& block) {
-      try {
-         my->on_incoming_block(block, {});
-      } LOG_AND_DROP();
-   });
-
-   my->_incoming_transaction_subscription = app().get_channel<incoming::channels::transaction>().subscribe(
-         [this](const packed_transaction_ptr& trx) {
-      try {
-         my->on_incoming_transaction_async(trx, false, false, false, [](const auto&){});
-      } LOG_AND_DROP();
-   });
-
    my->_incoming_block_sync_provider = app().get_method<incoming::methods::block_sync>().register_provider(
-         [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id) {
-      return my->on_incoming_block(block, block_id);
+         [this](const signed_block_ptr& block, const std::optional<block_id_type>& block_id, const block_state_ptr& bsp) {
+      return my->on_incoming_block(block, block_id, bsp);
    });
 
    my->_incoming_transaction_async_provider = app().get_method<incoming::methods::transaction_async>().register_provider(
@@ -1242,6 +1258,7 @@ void producer_plugin::plugin_shutdown() {
    }
 
    app().post( 0, [me = my](){} ); // keep my pointer alive until queue is drained
+   fc_ilog(_log, "exit shutdown");
 }
 
 void producer_plugin::handle_sighup() {
@@ -1679,6 +1696,14 @@ fc::time_point producer_plugin_impl::calculate_block_deadline( const fc::time_po
    }
 }
 
+bool producer_plugin_impl::should_interrupt_start_block( const fc::time_point& deadline ) const {
+   if( _pending_block_mode == pending_block_mode::producing ) {
+      return deadline <= fc::time_point::now();
+   }
+   // if we can produce then honor deadline so production starts on time
+   return (!_producers.empty() && deadline <= fc::time_point::now()) || _received_block;
+}
+
 producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    chain::controller& chain = chain_plug->chain();
 
@@ -1687,7 +1712,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
    const auto& hbs = chain.head_block_state();
 
-   if( chain.get_terminate_at_block() > 0 && chain.get_terminate_at_block() < chain.head_block_num() ) {
+   if( chain.get_terminate_at_block() > 0 && chain.get_terminate_at_block() <= chain.head_block_num() ) {
       ilog("Reached configured maximum block ${num}; terminating", ("num", chain.get_terminate_at_block()));
       app().quit();
       return start_block_result::failed;
@@ -1854,16 +1879,19 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
       }
 
       try {
+         _account_fails.report_and_clear(hbs->block_num);
+
          if( !remove_expired_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
          if( !remove_expired_blacklisted_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
-         if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(), preprocess_deadline ) )
+         if( !_subjective_billing.remove_expired( _log, chain.pending_block_time(), fc::time_point::now(),
+                                                  [&](){ return should_interrupt_start_block( preprocess_deadline ); } ) ) {
             return start_block_result::exhausted;
+         }
 
          // limit execution of pending incoming to once per block
          size_t pending_incoming_process_limit = _unapplied_transactions.incoming_size();
-         _account_fails.clear();
 
          if( !process_unapplied_trxs( preprocess_deadline ) )
             return start_block_result::exhausted;
@@ -1882,7 +1910,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
          if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit()
             return start_block_result::failed;
-         if (preprocess_deadline <= fc::time_point::now() || block_is_exhausted()) {
+         if ( should_interrupt_start_block( preprocess_deadline ) || block_is_exhausted() ) {
             return start_block_result::exhausted;
          } else {
             if( !process_incoming_trxs( preprocess_deadline, pending_incoming_process_limit ) )
@@ -1913,7 +1941,7 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
    size_t num_expired_persistent = 0;
    size_t num_expired_other = 0;
    size_t orig_count = _unapplied_transactions.size();
-   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, deadline,
+   bool exhausted = !_unapplied_transactions.clear_expired( pending_block_time, [&](){ return should_interrupt_start_block( deadline ); },
                   [chain_plug = chain_plug, &num_expired_persistent, &num_expired_other, pbm = _pending_block_mode,
                    &chain, has_producers = !_producers.empty()]( const packed_transaction_ptr& packed_trx_ptr, trx_enum_type trx_type ) {
             if( trx_type == trx_enum_type::persisted ) {
@@ -1955,7 +1983,7 @@ bool producer_plugin_impl::remove_expired_trxs( const fc::time_point& deadline )
             }
    });
 
-   if( exhausted ) {
+   if( exhausted && _pending_block_mode == pending_block_mode::producing ) {
       fc_wlog( _log, "Unable to process all expired transactions in unapplied queue before deadline, "
                      "Persistent expired ${persistent_expired}, Other expired ${other_expired}",
                ("persistent_expired", num_expired_persistent)("other_expired", num_expired_other) );
@@ -1981,7 +2009,7 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point
       int orig_count = _blacklisted_transactions.size();
 
       while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= lib_time) {
-         if (deadline <= fc::time_point::now()) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2009,7 +2037,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
       auto end_itr = (_pending_block_mode == pending_block_mode::producing) ?
                      _unapplied_transactions.unapplied_end()   : _unapplied_transactions.persisted_end();
       while( itr != end_itr ) {
-         if( deadline <= fc::time_point::now() ) {
+         if( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2024,8 +2052,9 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
                ++num_failed;
                if( itr->next ) {
                   itr->next( std::make_shared<tx_cpu_usage_exceeded>(
-                        FC_LOG_MESSAGE( error, "transaction ${id} exceeded failure limit for account ${a}",
-                                        ("id", trx->id())("a", first_auth) ) ) );
+                        FC_LOG_MESSAGE( error, "transaction ${id} exceeded failure limit for account ${a} until ${next_reset_time}",
+                                        ("id", trx->id())("a", first_auth)
+                                        ("next_reset_time", _account_fails.next_reset_timepoint(chain.head_block_num(),chain.head_block_time())) ) ) );
                }
                itr = _unapplied_transactions.erase( itr );
                continue;
@@ -2048,7 +2077,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
                || trx->read_only;
 
             auto trace = chain.push_transaction( trx, deadline, max_trx_time, prev_billed_cpu_time_us, false, sub_bill );
-            fc_dlog( _trx_failed_trace_log, "Subjective unapplied bill for ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
+            fc_tlog( _log, "Subjective unapplied bill for ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
             if( trace->except ) {
                if( exception_is_exhausted( *trace->except ) ) {
                   if( block_is_exhausted() ) {
@@ -2057,7 +2086,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
                      break;
                   }
                } else {
-                  fc_dlog( _trx_failed_trace_log, "Subjective unapplied bill for failed ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
+                  fc_tlog( _log, "Subjective unapplied bill for failed ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
                   auto failure_code = trace->except->code();
                   if( failure_code != tx_duplicate::code_value ) {
                      // this failed our configured maximum transaction time, we don't want to replay it
@@ -2076,7 +2105,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
                   continue;
                }
             } else {
-               fc_dlog( _trx_successful_trace_log, "Subjective unapplied bill for success ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
+               fc_tlog( _log, "Subjective unapplied bill for success ${a}: ${b} prev ${t}us", ("a",first_auth)("b",prev_billed_cpu_time_us)("t",trace->elapsed));
                // if db_read_mode SPECULATIVE then trx is in the pending block and not immediately reverted
                if (!disable_subjective_billing)
                   _subjective_billing.subjective_bill( trx->id(), trx->packed_trx()->expiration(), first_auth, trace->elapsed,
@@ -2210,7 +2239,7 @@ bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline
       auto itr = _unapplied_transactions.incoming_begin();
       auto end = _unapplied_transactions.incoming_end();
       while( pending_incoming_process_limit && itr != end ) {
-         if (deadline <= fc::time_point::now()) {
+         if ( should_interrupt_start_block( deadline ) ) {
             exhausted = true;
             break;
          }
@@ -2250,6 +2279,7 @@ bool producer_plugin_impl::block_is_exhausted() const {
 // -> Idle
 // --> Start block B (block time y.000) at time x.500
 void producer_plugin_impl::schedule_production_loop() {
+   _received_block = false;
    _timer.cancel();
 
    auto result = start_block();
@@ -2437,9 +2467,6 @@ void producer_plugin_impl::produce_block() {
 
    block_state_ptr new_bs = chain.head_block_state();
 
-   _account_fails.report();
-   _account_fails.clear();
-
    ilog("Produced block ${id}... #${n} @ ${t} signed by ${p} [trxs: ${count}, lib: ${lib}, confirmed: ${confs}]",
         ("p",new_bs->header.producer)("id",new_bs->id.str().substr(8,16))
         ("n",new_bs->block_num)("t",new_bs->header.timestamp)
@@ -2447,6 +2474,10 @@ void producer_plugin_impl::produce_block() {
 
 }
 
+void producer_plugin::received_block() {
+   my->_received_block = true;
+}
+
 void producer_plugin::log_failed_transaction(const transaction_id_type& trx_id, const packed_transaction_ptr& packed_trx_ptr, const char* reason) const {
    fc_dlog(_trx_failed_trace_log, "[TRX_TRACE] Speculative execution is REJECTING tx: ${txid} : ${why}",
             ("txid", trx_id)("why", reason));

plugins/producer_plugin/test/test_subjective_billing.cpp

@@ -50,7 +50,7 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) {
       BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) );
 
       // expires transactions but leaves them in the decay at full value
-      sub_bill.remove_expired( log, now + fc::microseconds(1), now, fc::time_point::maximum() );
+      sub_bill.remove_expired( log, now + fc::microseconds(1), now, [](){ return false; } );
 
       BOOST_CHECK_EQUAL( 13+11, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 9, sub_bill.get_subjective_bill(b, now) );
@@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) {
       BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) );
 
-      sub_bill.remove_expired( log, now, now, fc::time_point::maximum() );
+      sub_bill.remove_expired( log, now, now, [](){ return false; } );
       BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) );
 
@@ -148,7 +148,7 @@ BOOST_AUTO_TEST_CASE( subjective_bill_test ) {
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(a, endtime) );
       BOOST_CHECK_EQUAL( 0, sub_bill.get_subjective_bill(b, endtime) );
 
-      sub_bill.remove_expired( log, now + fc::microseconds(1), now, fc::time_point::maximum() );
+      sub_bill.remove_expired( log, now + fc::microseconds(1), now, [](){ return false; } );
       BOOST_CHECK_EQUAL( 1024 + 1024, sub_bill.get_subjective_bill(a, now) );
       BOOST_CHECK_EQUAL( 1024, sub_bill.get_subjective_bill(b, now) );
 

plugins/trace_api_plugin/store_provider.cpp

@@ -307,8 +307,11 @@ namespace eosio::trace_api {
                _maintenance_condition.wait(lock);
             }
 
+            if (_maintenance_shutdown) {
+               break;
+            }
+
             uint32_t best_known_lib = _best_known_lib;
-            bool shutdown = _maintenance_shutdown;
             lock.unlock();
 
             log(std::string("Waking up to handle lib: ") + std::to_string(best_known_lib));
@@ -319,10 +322,6 @@ namespace eosio::trace_api {
                   last_lib = best_known_lib;
                } FC_LOG_AND_DROP();
             }
-
-            if (shutdown) {
-               break;
-            }
          }
       });
    }

plugins/trace_api_plugin/trace_api_plugin.cpp

@@ -455,6 +455,7 @@ void trace_api_plugin::plugin_startup() {
 void trace_api_plugin::plugin_shutdown() {
    my->plugin_shutdown();
    rpc->plugin_shutdown();
+   fc_ilog( _log, "exit shutdown");
 }
 
 void trace_api_plugin::handle_sighup() {

programs/keosd/main.cpp

@@ -71,6 +71,17 @@ bfs::path determine_home_directory()
    return home;
 }
 
+enum return_codes {
+   OTHER_FAIL        = -2,
+   INITIALIZE_FAIL   = -1,
+   SUCCESS           = 0,
+   BAD_ALLOC         = 1,
+   DATABASE_DIRTY    = 2,
+   FIXED_REVERSIBLE  = SUCCESS,
+   EXTRACTED_GENESIS = SUCCESS,
+   NODE_MANAGEMENT_SUCCESS = 5
+};
+
 int main(int argc, char** argv)
 {
    try {
@@ -90,6 +101,7 @@ int main(int argc, char** argv)
              opts.count("print-default-config")) {
             return 0;
          }
+         return INITIALIZE_FAIL;
       }
       initialize_logging();
       auto& http = app().get_plugin<http_plugin>();

scripts/pinned_build.sh

@@ -132,7 +132,7 @@ pushdir ${LEAP_DIR}
 
 # build Leap
 echo "Building Leap ${SCRIPT_DIR}"
-try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=${LLVM_DIR}/lib/cmake -DCMAKE_PREFIX_PATH=${BOOST_DIR}/bin ${SCRIPT_DIR}/..
+try cmake -DCMAKE_TOOLCHAIN_FILE=${SCRIPT_DIR}/pinned_toolchain.cmake -DCMAKE_INSTALL_PREFIX=${LEAP_PINNED_INSTALL_PREFIX:-/usr/local} -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=${LLVM_DIR}/lib/cmake -DCMAKE_PREFIX_PATH=${BOOST_DIR}/bin ${SCRIPT_DIR}/..
 
 try make -j${JOBS}
 try cpack

tests/CMakeLists.txt

@@ -120,11 +120,11 @@ add_test(NAME restart-scenarios-test-hard_replay COMMAND tests/restart-scenarios
 set_property(TEST restart-scenarios-test-hard_replay PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME restart-scenarios-test-none COMMAND tests/restart-scenarios-test.py -c none --kill-sig term -p4 -v --clean-run --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST restart-scenarios-test-none PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME terminate-scenarios-test-resync COMMAND tests/terminate-scenarios-test.py -p 1 -c resync --kill-count 1 --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME terminate-scenarios-test-resync COMMAND tests/terminate-scenarios-test.py -c resync --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST terminate-scenarios-test-resync PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME terminate-scenarios-test-replay COMMAND tests/terminate-scenarios-test.py -p 1 -c replay --kill-count 1 --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME terminate-scenarios-test-replay COMMAND tests/terminate-scenarios-test.py -c replay --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST terminate-scenarios-test-replay PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME terminate-scenarios-test-hard_replay COMMAND tests/terminate-scenarios-test.py -p 1 -c hardReplay --kill-count 1 --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME terminate-scenarios-test-hard_replay COMMAND tests/terminate-scenarios-test.py -c hardReplay --terminate-at-block 10 --kill-sig term --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST terminate-scenarios-test-hard_replay PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST validate_dirty_db_test PROPERTY LABELS nonparallelizable_tests)

tests/Cluster.py

@@ -1430,14 +1430,14 @@ class Cluster(object):
         time.sleep(1) # Give processes time to stand down
         return True
 
-    def relaunchEosInstances(self, cachePopen=False, nodeArgs=""):
+    def relaunchEosInstances(self, cachePopen=False, nodeArgs="", waitForTerm=False):
 
         chainArg=self.__chainSyncStrategy.arg + " " + nodeArgs
 
         newChain= False if self.__chainSyncStrategy.name in [Utils.SyncHardReplayTag, Utils.SyncNoneTag] else True
         for i in range(0, len(self.nodes)):
             node=self.nodes[i]
-            if node.killed and not node.relaunch(chainArg, newChain=newChain, cachePopen=cachePopen):
+            if node.killed and not node.relaunch(chainArg, newChain=newChain, cachePopen=cachePopen, waitForTerm=waitForTerm):
                 return False
 
         return True

tests/Node.py

@@ -1210,7 +1210,7 @@ class Node(object):
 
     # pylint: disable=too-many-locals
     # If nodeosPath is equal to None, it will use the existing nodeos path
-    def relaunch(self, chainArg=None, newChain=False, skipGenesis=True, timeout=Utils.systemWaitTimeout, addSwapFlags=None, cachePopen=False, nodeosPath=None, waitForTerm=True):
+    def relaunch(self, chainArg=None, newChain=False, skipGenesis=True, timeout=Utils.systemWaitTimeout, addSwapFlags=None, cachePopen=False, nodeosPath=None, waitForTerm=False):
 
         assert(self.pid is None)
         assert(self.killed)
@@ -1265,15 +1265,16 @@ class Node(object):
             except subprocess.TimeoutExpired:
                 return False
             with open(popen.errfile.name, 'r') as f:
-                if "Reached configured maximum block 10; terminating" in f.read():
+                if "successfully exiting" in f.read():
                     return True
                 else:
                     return False
 
-        if "terminate-at-block" not in cmd or not waitForTerm:
-            isAlive=Utils.waitForBool(isNodeAlive, timeout, sleepTime=1)
-        else:
+        if waitForTerm:
             isAlive=Utils.waitForBoolWithArg(didNodeExitGracefully, self.popenProc, timeout, sleepTime=1)
+        else:
+            isAlive=Utils.waitForBool(isNodeAlive, timeout, sleepTime=1)
+
         if isAlive:
             Utils.Print("Node relaunch was successful.")
         else:

tests/nodeos_forked_chain_test.py

@@ -375,6 +375,8 @@ try:
     if preKillBlockProducer == "defproducerj" or preKillBlockProducer == "defproducerk":
         # wait for defproduceri so there is plenty of time to send kill before defproducerk
         nonProdNode.waitForProducer("defproduceri")
+        preKillBlockNum=nonProdNode.getBlockNum()
+        preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum)
     Print("preKillBlockProducer = {}".format(preKillBlockProducer))
     # kill at last block before defproducerl, since the block it is killed on will get propagated
     killAtProducer="defproducerk"

tests/nodeos_read_terminate_at_block_test.py

@@ -25,8 +25,6 @@ relaunchTimeout = 10
 numOfProducers = 1
 totalNodes = 4
 
-termAtFutureBlockNum = 30
-
 # Parse command line arguments
 args = TestHelper.parse_args({
     "-v",
@@ -47,7 +45,7 @@ keepLogs = args.keep_logs
 # Wrapper function to execute test
 # This wrapper function will resurrect the node to be tested, and shut
 # it down by the end of the test
-def executeTest(cluster, producerNode, testNodeId, testNodeArgs, resultMsgs):
+def executeTest(cluster, testNodeId, testNodeArgs, resultMsgs):
     testNode = None
     testResult = False
     resultDesc = "!!!BUG IS CONFIRMED ON TEST CASE #{} ({})".format(
@@ -63,17 +61,12 @@ def executeTest(cluster, producerNode, testNodeId, testNodeArgs, resultMsgs):
             )
         )
 
-        # Get current information from the producer.
-        producerNode.waitForHeadToAdvance()
-
-        # Launch the node with the terminate-at-block option specified
-        # in testNodeArgs. The option for each node has already
-        # been set via specificExtraNodeosArg in the cluster launch.
-        cluster.launchUnstarted(cachePopen=True)
         testNode = cluster.getNode(testNodeId)
+        assert not testNode.verifyAlive() # resets pid so reluanch works
+        testNode.relaunch(addSwapFlags={"--terminate-at-block": "9999999"}, cachePopen=True)
 
         # Wait for node to start up.
-        time.sleep(10)
+        time.sleep(3)
 
         # Check the node stops at the correct block.
         checkStatus(testNode, testNodeArgs)
@@ -107,8 +100,7 @@ def checkStatus(testNode, testNodeArgs):
     """Test --terminate-at-block stops at the correct block."""
     Print(" ".join([
         "The test node has begun receiving from the producing node and",
-        "is expected to stop at or little bigger than the block number",
-        "specified here: ",
+        "is expected to stop at the block number specified here: ",
         testNodeArgs
     ]))
 
@@ -122,6 +114,12 @@ def checkStatus(testNode, testNodeArgs):
     else:
         checkHeadOrSpeculative(head, lib)
 
+    # Check for the terminate at block message.
+    match = re.search(r"--terminate-at-block (\d+)", testNodeArgs)
+    termAtBlock = int(match.group(1))
+
+    assert head == termAtBlock, f"head {head} termAtBlock {termAtBlock}"
+
 
 def checkReplay(testNode, testNodeArgs):
     """Test --terminate-at-block with --replay-blockchain."""
@@ -135,20 +133,18 @@ def checkReplay(testNode, testNodeArgs):
         testNodeArgs
     ]))
 
-    testNode.relaunch(chainArg="--replay-blockchain", cachePopen=True)
+    assert not testNode.verifyAlive()
+    testNode.relaunch(chainArg="--replay-blockchain", addSwapFlags={"--terminate-at-block": "9999999"}, cachePopen=True)
 
     # Wait for node to finish up.
-    time.sleep(10)
+    time.sleep(3)
 
     # Check for the terminate at block message.
-    match = re.search(r"--terminate-at-block=(\d+)", testNodeArgs)
+    match = re.search(r"--terminate-at-block (\d+)", testNodeArgs)
     termAtBlock = int(match.group(1))
 
-    termMsg = "Reached configured maximum block {}; terminating".format(
-        termAtBlock
-    )
-
-    assert checkLog(testNode.popenProc.errfile.name, termMsg)
+    head, lib = getBlockNumInfo(testNode)
+    assert head == termAtBlock, f"head {head} termAtBlock {termAtBlock}"
 
 
 def getBlockNumInfo(testNode):
@@ -159,11 +155,12 @@ def getBlockNumInfo(testNode):
         info = testNode.getInfo()
 
         if not info:
-            break
+            continue
 
         try:
             head = info["head_block_num"]
             lib = info["last_irreversible_block_num"]
+            break
 
         except KeyError:
             pass
@@ -172,26 +169,6 @@ def getBlockNumInfo(testNode):
     return head, lib
 
 
-def checkLog(log, message, sleepDuration=1, maxAttempt=10):
-    attemptCnt = 0
-    found = False
-
-    while not found and attemptCnt < maxAttempt:
-        try:
-            with open(log, "r") as f:
-                found = message in f.read()
-
-        except OSError:
-            pass
-
-        if not found:
-            time.sleep(sleepDuration)
-
-        attemptCnt = attemptCnt + 1
-
-    return found
-
-
 def checkIrreversible(head, lib):
     assert head == lib, (
         "Head ({}) should be equal to lib ({})".format(head, lib)
@@ -215,9 +192,9 @@ testSuccessful = False
 try:
     specificNodeosArgs = {
         0 : "--enable-stale-production",
-        1 : "--read-mode irreversible --terminate-at-block=250",
-        2 : "--read-mode speculative --terminate-at-block=550",
-        3 : "--read-mode head --terminate-at-block=850",
+        1 : "--read-mode irreversible --terminate-at-block 75",
+        2 : "--read-mode speculative --terminate-at-block 100",
+        3 : "--read-mode head --terminate-at-block 125",
     }
     traceNodeosArgs = " --plugin eosio::trace_api_plugin --trace-no-abis "
 
@@ -229,7 +206,6 @@ try:
         prodCount=numOfProducers,
         totalProducers=numOfProducers,
         totalNodes=totalNodes,
-        unstartedNodes=totalNodes - numOfProducers,
         pnodes=1,
         useBiosBootFile=False,
         topo="mesh",
@@ -240,6 +216,11 @@ try:
     producingNodeId = 0
     producingNode = cluster.getNode(producingNodeId)
 
+    # wait for all to terminate, needs to be larger than largest terminate-at-block
+    producingNode.waitForBlock( 150, timeout=150 )
+    cluster.biosNode.kill(signal.SIGTERM)
+    producingNode.kill(signal.SIGTERM)
+
     # Start executing test cases here
     Utils.Print("Script Begin .............................")
 
@@ -250,7 +231,6 @@ try:
 
         success = executeTest(
             cluster,
-            producingNode,
             nodeId,
             nodeArgs,
             testResultMsgs

tests/terminate-scenarios-test.py

@@ -20,16 +20,15 @@ import random
 Print=Utils.Print
 errorExit=Utils.errorExit
 
-args=TestHelper.parse_args({"-p","-d","-s","-c","--kill-sig","--kill-count","--keep-logs"
+args=TestHelper.parse_args({"-d","-s","-c","--kill-sig","--keep-logs"
                             ,"--dump-error-details","-v","--leave-running","--clean-run"
                             ,"--terminate-at-block"})
-pnodes=args.p
+pnodes=1
 topo=args.s
 delay=args.d
 chainSyncStrategyStr=args.c
 debug=args.v
 total_nodes = pnodes
-killCount=args.kill_count if args.kill_count > 0 else 1
 killSignal=args.kill_sig
 killEosInstances= not args.leave_running
 dumpErrorDetails=args.dump_error_details
@@ -70,19 +69,20 @@ try:
     if not cluster.waitOnClusterBlockNumSync(3):
         errorExit("Cluster never stabilized")
 
-    Print("Kill %d cluster node instances." % (killCount))
-    if cluster.killSomeEosInstances(killCount, killSignal) is False:
+    Print("Kill cluster node instance.")
+    if cluster.killSomeEosInstances(1, killSignal) is False:
         errorExit("Failed to kill Eos instances")
+    assert not cluster.getNode(0).verifyAlive()
     Print("nodeos instances killed.")
 
-    Print ("Relaunch dead cluster nodes instances.")
+    Print ("Relaunch dead cluster node instance.")
     nodeArg = "--terminate-at-block %d" % terminate if terminate > 0 else ""
     if nodeArg != "":
         if chainSyncStrategyStr == "hardReplay":
             nodeArg += " --truncate-at-block %d" % terminate
-    if cluster.relaunchEosInstances(cachePopen=True, nodeArgs=nodeArg) is False:
-        errorExit("Failed to relaunch Eos instances")
-    Print("nodeos instances relaunched.")
+    if cluster.relaunchEosInstances(cachePopen=True, nodeArgs=nodeArg, waitForTerm=(terminate > 0)) is False:
+        errorExit("Failed to relaunch Eos instance")
+    Print("nodeos instance relaunched.")
 
     testSuccessful=True
 finally:

tutorials/bios-boot-tutorial/bios-boot-tutorial.py

@@ -332,9 +332,9 @@ def stepSetSystemContract():
     retry(args.cleos + 'push action eosio activate \'["ad9e3d8f650687709fd68f4b90b41f7d825a365b02c23a636cef88ac2ac00c43"]\' -p eosio@active')
     # DISALLOW_EMPTY_PRODUCER_SCHEDULE
     retry(args.cleos + 'push action eosio activate \'["68dcaa34c0517d19666e6b33add67351d8c5f69e999ca1e37931bc410a297428"]\' -p eosio@active')
-     # FIX_LINKAUTH_RESTRICTION
+    # FIX_LINKAUTH_RESTRICTION
     retry(args.cleos + 'push action eosio activate \'["e0fb64b1085cc5538970158d05a009c24e276fb94e1a0bf6a528b48fbc4ff526"]\' -p eosio@active')
-     # REPLACE_DEFERRED
+    # REPLACE_DEFERRED
     retry(args.cleos + 'push action eosio activate \'["ef43112c6543b88db2283a2e077278c315ae2c84719a8b25f25cc88565fbea99"]\' -p eosio@active')
     # NO_DUPLICATE_DEFERRED_ID
     retry(args.cleos + 'push action eosio activate \'["4a90c00d55454dc5b059055ca213579c6ea856967712a56017487886a4d4cc0f"]\' -p eosio@active')
@@ -346,6 +346,12 @@ def stepSetSystemContract():
     retry(args.cleos + 'push action eosio activate \'["4fca8bd82bbd181e714e283f83e1b45d95ca5af40fb89ad3977b653c448f78c2"]\' -p eosio@active')
     # WTMSIG_BLOCK_SIGNATURES
     retry(args.cleos + 'push action eosio activate \'["299dcb6af692324b899b39f16d5a530a33062804e41f09dc97e9f156b4476707"]\' -p eosio@active')
+    # GET_CODE_HASH
+    retry(args.cleos + 'push action eosio activate \'["bcd2a26394b36614fd4894241d3c451ab0f6fd110958c3423073621a70826e99"]\' -p eosio@active')
+    # GET_BLOCK_NUM
+    retry(args.cleos + 'push action eosio activate \'["35c2186cc36f7bb4aeaf4487b36e57039ccf45a9136aa856a5d569ecca55ef2b"]\' -p eosio@active')
+    # CRYPTO_PRIMITIVES
+    retry(args.cleos + 'push action eosio activate \'["6bcb40a24e49c26d0a60513b6aeb8551d264e4717f306b81a37a5afb3b47cedc"]\' -p eosio@active')
     sleep(1)
 
     # install eosio.system latest version

unittests/block_tests.cpp

@@ -44,9 +44,9 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test)
 
    // Push block with invalid transaction to other chain
    tester validator;
-   auto bs = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
+   auto bsf = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
    validator.control->abort_block();
-   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
+   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
    [] (const fc::exception &e)->bool {
       return e.code() == account_name_exists_exception::code_value ;
    }) ;
@@ -82,9 +82,9 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_mroot_test)
 
    // Push block with invalid transaction to other chain
    tester validator;
-   auto bs = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
+   auto bsf = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b );
    validator.control->abort_block();
-   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
+   BOOST_REQUIRE_EXCEPTION(validator.control->push_block( bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception ,
                            [] (const fc::exception &e)->bool {
                               return e.code() == block_validate_exception::code_value &&
                                      e.to_detail_string().find("invalid block transaction merkle root") != std::string::npos;

unittests/forked_tests.cpp

@@ -267,9 +267,9 @@ BOOST_AUTO_TEST_CASE( forking ) try {
    signed_block bad_block = std::move(*b);
    bad_block.action_mroot = bad_block.previous;
    auto bad_id = bad_block.calculate_id();
-   auto bad_block_bs = c.control->create_block_state_future( bad_id, std::make_shared<signed_block>(std::move(bad_block)) );
+   auto bad_block_bsf = c.control->create_block_state_future( bad_id, std::make_shared<signed_block>(std::move(bad_block)) );
    c.control->abort_block();
-   BOOST_REQUIRE_EXCEPTION(c.control->push_block( bad_block_bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception,
+   BOOST_REQUIRE_EXCEPTION(c.control->push_block( bad_block_bsf.get(), forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception,
       [] (const fc::exception &ex)->bool {
          return ex.to_detail_string().find("block signed by unexpected key") != std::string::npos;
       });

unittests/unapplied_transaction_queue_tests.cpp

@@ -305,7 +305,7 @@ BOOST_AUTO_TEST_CASE( unapplied_transaction_queue_test ) try {
    q.add_aborted( { trx20, trx22 } );
    q.add_persisted( trx21 );
    q.add_persisted( trx23 );
-   q.clear_expired( fc::time_point::now(), fc::time_point::now() + fc::seconds( 300 ), [](auto, auto){} );
+   q.clear_expired( fc::time_point::now(), [](){ return false; }, [](auto, auto){} );
    BOOST_CHECK( q.size() == 2 );
    BOOST_REQUIRE( next( q ) == trx23 );
    BOOST_REQUIRE( next( q ) == trx22 );