Merge branch 'fix_missing_node_py_changes' of https://github.com/AntelopeIO/leap into fix_missing_node_py_changes

fix noisy `ignoring packed attribute` warning on `IR::ControlStructureImm`

.github/actions/parallel-ctest-containers/main.mjs

@@ -28,7 +28,7 @@ try {
    let subprocesses = [];
    tests.forEach(t => {
       subprocesses.push(new Promise(resolve => {
-         child_process.spawn("docker", ["run", "--name", t, "--init", "baseimage", "bash", "-c", `cd build; ctest --output-on-failure -R '^${t}$'`], {stdio:"inherit"}).on('close', code => resolve(code));
+         child_process.spawn("docker", ["run", "--security-opt", "seccomp=unconfined", "-e", "GITHUB_ACTIONS=True", "--name", t, "--init", "baseimage", "bash", "-c", `cd build; ctest --output-on-failure -R '^${t}$'`], {stdio:"inherit"}).on('close', code => resolve(code));
       }));
    });
 

.github/workflows/build.yaml

@@ -120,7 +120,9 @@ jobs:
       matrix:
         platform: [ubuntu20, ubuntu22]
     runs-on: ["self-hosted", "enf-x86-hightier"]
-    container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+    container: 
+      image: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+      options: --security-opt seccomp=unconfined
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
@@ -157,7 +159,7 @@ jobs:
         uses: ./.github/actions/parallel-ctest-containers
         with:
           container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
-          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd"]'
+          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
           log-tarball-prefix: ${{matrix.platform}}
           tests-label: nonparallelizable_tests
       - name: Upload logs from failed tests  
@@ -186,7 +188,7 @@ jobs:
         uses: ./.github/actions/parallel-ctest-containers
         with:
           container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
-          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd"]'
+          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
           log-tarball-prefix: ${{matrix.platform}}
           tests-label: long_running_tests
       - name: Upload logs from failed tests

.gitignore

@@ -82,7 +82,7 @@ deps/*
 dependencies/*
 
 etc/eosio/node_*
-var/lib/node_*
+TestLogs*
 .vscode
 .idea/
 *.iws

.gitmodules

@@ -22,6 +22,9 @@
 [submodule "libraries/libfc/secp256k1/secp256k1"]
 	path = libraries/libfc/secp256k1/secp256k1
 	url = https://github.com/bitcoin-core/secp256k1
+[submodule "libraries/prometheus/prometheus-cpp"]
+	path = libraries/prometheus/prometheus-cpp
+	url = https://github.com/jupp0r/prometheus-cpp.git
 [submodule "libraries/libfc/libraries/bn256"]
 	path = libraries/libfc/libraries/bn256
 	url = https://github.com/AntelopeIO/bn256

CMakeLists.txt

@@ -14,7 +14,7 @@ set( CMAKE_CXX_EXTENSIONS ON )
 set( CXX_STANDARD_REQUIRED ON)
 
 set(VERSION_MAJOR 4)
-set(VERSION_MINOR 0)
+set(VERSION_MINOR 1)
 set(VERSION_PATCH 0)
 set(VERSION_SUFFIX dev)
 
@@ -200,9 +200,6 @@ add_subdirectory( benchmark )
 
 option(DISABLE_WASM_SPEC_TESTS "disable building of wasm spec unit tests" OFF)
 
-install(FILES testnet.template DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio/launcher COMPONENT dev EXCLUDE_FROM_ALL)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testnet.template ${CMAKE_CURRENT_BINARY_DIR}/etc/eosio/launcher/testnet.template COPYONLY)
-
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/eosio.version.in ${CMAKE_CURRENT_BINARY_DIR}/eosio.version.hpp)
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/eosio.version.hpp DESTINATION ${CMAKE_INSTALL_FULL_INCLUDEDIR} COMPONENT dev EXCLUDE_FROM_ALL)
 
@@ -235,10 +232,58 @@ configure_file(libraries/wasm-jit/LICENSE                                  licen
 configure_file(libraries/libfc/secp256k1/secp256k1/COPYING                 licenses/leap/LICENSE.secp256k1 COPYONLY)
 configure_file(libraries/libfc/include/fc/crypto/webauthn_json/license.txt licenses/leap/LICENSE.rapidjson COPYONLY)
 configure_file(libraries/eos-vm/LICENSE                                    licenses/leap/LICENSE.eos-vm    COPYONLY)
+configure_file(libraries/prometheus/prometheus-cpp/LICENSE                 licenses/leap/LICENSE.prom      COPYONLY)
 configure_file(programs/cleos/LICENSE.CLI11                                licenses/leap/LICENSE.CLI11     COPYONLY)
 
 install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/licenses/leap" DESTINATION "${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/" COMPONENT base)
 
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/libraries/testing/contracts DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/libraries/testing COMPONENT dev EXCLUDE_FROM_ALL
+        PATTERN "CMakeFiles" EXCLUDE
+        PATTERN "*.cmake" EXCLUDE
+        PATTERN "Makefile" EXCLUDE)
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/unittests/contracts DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/unittests COMPONENT dev EXCLUDE_FROM_ALL
+        PATTERN "CMakeFiles" EXCLUDE
+        PATTERN "*.cmake" EXCLUDE
+        PATTERN "Makefile" EXCLUDE)
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests/TestHarness DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/tests COMPONENT dev EXCLUDE_FROM_ALL
+        FILES_MATCHING
+        PATTERN "*.py"
+        PATTERN "*.json"
+        PATTERN "__pycache__" EXCLUDE
+        PATTERN "CMakeFiles" EXCLUDE)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tests/launcher.py DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/tests COMPONENT dev EXCLUDE_FROM_ALL)
+
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
+   # Cmake versions < 3.21 did not support installing symbolic links to a directory via install(FILES ...)
+   # Cmake 3.21 fixed this (https://gitlab.kitware.com/cmake/cmake/-/commit/d71a7cc19d6e03f89581efdaa8d63db216184d40)
+   # If/when the lowest cmake version supported becomes >= 3.21, the else block as well as the postinit and prerm scripts
+   # would be a good option to remove in favor of using the facilities in cmake / cpack directly.
+
+   add_custom_target(link_target ALL
+      COMMAND ${CMAKE_COMMAND} -E make_directory lib/python3/dist-packages
+      COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness lib/python3/dist-packages/TestHarness
+      COMMAND ${CMAKE_COMMAND} -E make_directory share/leap_testing
+      COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin share/leap_testing/bin)
+
+   install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib/python3/dist-packages/TestHarness DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages COMPONENT dev EXCLUDE_FROM_ALL)
+   install(FILES ${CMAKE_CURRENT_BINARY_DIR}/share/leap_testing/bin DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing COMPONENT dev EXCLUDE_FROM_ALL)
+else()
+   # The following install(CODE ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
+   # However, it can flag as an error if using `make package` instead of `make dev-install` for installation.
+   # This is due to those directories and symbolic links being created in the postinit script during the package install instead.
+
+   # Note If/when doing `make package`, it is not a true error if you see:
+   # Error creating directory "/<leap_install_dir>/lib/python3/dist-packages".
+   # CMake Error: failed to create symbolic link '/<leap_install_dir>/lib/python3/dist-packages/TestHarness': no such file or directory
+   # CMake Error: failed to create symbolic link '/<leap_install_dir>/share/leap_testing/bin': no such file or directory
+
+   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages)" COMPONENT dev EXCLUDE_FROM_ALL)
+   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)" COMPONENT dev EXCLUDE_FROM_ALL)
+   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)" COMPONENT dev EXCLUDE_FROM_ALL)
+
+   # The `make package` installation of symlinks happens via the `postinst` script installed in cmake.package via the line below
+endif()
+
 configure_file(${CMAKE_SOURCE_DIR}/libraries/cli11/bash-completion/completions/leap-util
                ${CMAKE_BINARY_DIR}/programs/leap-util/bash-completion/completions/leap-util COPYONLY)
 configure_file(${CMAKE_SOURCE_DIR}/libraries/cli11/bash-completion/completions/cleos

README.md

@@ -55,6 +55,7 @@ Requirements to build:
 - git
 - GMP
 - Python 3
+- python3-numpy
 - zlib
 
 ### Step 1 - Clone
@@ -71,7 +72,7 @@ git clone --recursive https://github.com/AntelopeIO/leap.git
 git clone --recursive git@github.com:AntelopeIO/leap.git
 ```
 
-> ℹ️ **HTTPS vs. SSH Clone** ℹ️
+> ℹ️ **HTTPS vs. SSH Clone** ℹ️  
 Both an HTTPS or SSH git clone will yield the same result - a folder named `leap` containing our source code. It doesn't matter which type you use.
 
 Navigate into that folder:
@@ -95,13 +96,13 @@ git submodule update --init --recursive
 ### Step 3 - Build
 Select build instructions below for a [pinned build](#pinned-build) (preferred) or an [unpinned build](#unpinned-build).
 
-> ℹ️ **Pinned vs. Unpinned Build** ℹ️
+> ℹ️ **Pinned vs. Unpinned Build** ℹ️  
 We have two types of builds for Leap: "pinned" and "unpinned." The only difference is that pinned builds use specific versions for some dependencies hand-picked by the Leap engineers - they are "pinned" to those versions. In contrast, unpinned builds use the default dependency versions available on the build system at the time. We recommend performing a "pinned" build to ensure the compiler and boost versions remain the same between builds of different Leap versions. Leap requires these versions to remain the same, otherwise its state might need to be recovered from a portable snapshot or the chain needs to be replayed.
 
-> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️
+> ⚠️ **A Warning On Parallel Compilation Jobs (`-j` flag)** ⚠️  
 When building C/C++ software, often the build is performed in parallel via a command such as `make -j "$(nproc)"` which uses all available CPU threads. However, be aware that some compilation units (`*.cpp` files) in Leap will consume nearly 4GB of memory. Failures due to memory exhaustion will typically, but not always, manifest as compiler crashes. Using all available CPU threads may also prevent you from doing other things on your computer during compilation. For these reasons, consider reducing this value.
 
-> 🐋 **Docker and `sudo`** 🐋
+> 🐋 **Docker and `sudo`** 🐋  
 If you are in an Ubuntu docker container, omit `sudo` from all commands because you run as `root` by default. Most other docker containers also exclude `sudo`, especially Debian-family containers. If your shell prompt is a hash tag (`#`), omit `sudo`.
 
 #### Pinned Build
@@ -110,12 +111,14 @@ Make sure you are in the root of the `leap` repo, then run the `install_depts.sh
 sudo scripts/install_deps.sh
 ```
 
-Next, run the pinned build script. You have to give it three arguments, in the following order:
-  - A temporary folder, for all dependencies that need to be built from source.
-  - A build folder, where the binaries you need to install will be built to.
-  - The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
+Next, run the pinned build script. You have to give it three arguments in the following order:
+1. A temporary folder, for all dependencies that need to be built from source.
+1. A build folder, where the binaries you need to install will be built to.
+1. The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
 
-The following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads (Note: you don't need `sudo` for this command):
+> 🔒 You do not need to run this script with `sudo` or as root.
+
+For example, the following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads:
 ```bash
 scripts/pinned_build.sh deps build "$(nproc)"
 ```

docs/01_nodeos/02_usage/03_development-environment/01_local-multi-node-testnet.md

@@ -143,7 +143,7 @@ We now have an account that is available to have a contract assigned to it, enab
 In the fourth terminal window, start a second `nodeos` instance. Notice that this command line is substantially longer than the one we used above to create the first producer. This is necessary to avoid collisions with the first `nodeos` instance. Fortunately, you can just cut and paste this command line and adjust the keys:
 
 ```sh
-nodeos --producer-name inita --plugin eosio::chain_api_plugin --plugin eosio::net_api_plugin --http-server-address 127.0.0.1:8889 --p2p-listen-endpoint 127.0.0.1:9877 --p2p-peer-address 127.0.0.1:9876 --config-dir node2 --data-dir node2 --private-key [\"EOS6hMjoWRF2L8x9YpeqtUEcsDKAyxSuM1APicxgRU1E3oyV5sDEg\",\"5JgbL2ZnoEAhTudReWH1RnMuQS6DBeLZt4ucV6t8aymVEuYg7sr\"]
+nodeos --producer-name inita --plugin eosio::chain_api_plugin --plugin eosio::net_api_plugin --http-server-address 127.0.0.1:8889 --p2p-listen-endpoint 127.0.0.1:9877 --p2p-peer-address 127.0.0.1:9876 --config-dir node2 --data-dir node2 --signature-provider EOS6hMjoWRF2L8x9YpeqtUEcsDKAyxSuM1APicxgRU1E3oyV5sDEg=KEY:5JgbL2ZnoEAhTudReWH1RnMuQS6DBeLZt4ucV6t8aymVEuYg7sr
 ```
 
 The output from this new node will show a little activity but will stop reporting until the last step in this tutorial, when the `inita` account is registered as a producer account and activated. Here is some example output from a newly started node. Your output might look a little different, depending on how much time you took entering each of these commands. Furthermore, this example is only the last few lines of output:

docs/01_nodeos/03_plugins/http_client_plugin/index.md

@@ -1,39 +0,0 @@
-## Description
-
-The `http_client_plugin`  is an internal utility plugin, providing the `producer_plugin` the ability to use securely an external `keosd` instance as its block signer. It can only be used when the `producer_plugin` is configured to produce blocks.
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::http_client_plugin
-https-client-root-cert = "path/to/my/certificate.pem"
-https-client-validate-peers = 1
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::http_client_plugin  \
-           --https-client-root-cert "path/to/my/certificate.pem"  \
-           --https-client-validate-peers 1
-```
-
-## Options
-
-These can be specified from both the `nodeos` command-line or the `config.ini` file:
-
-```console
-Config Options for eosio::http_client_plugin:
-  --https-client-root-cert arg          PEM encoded trusted root certificate 
-                                        (or path to file containing one) used 
-                                        to validate any TLS connections made.  
-                                        (may specify multiple times)
-                                        
-  --https-client-validate-peers arg (=1)
-                                        true: validate that the peer 
-                                        certificates are valid and trusted, 
-                                        false: ignore cert errors
-```
-
-## Dependencies
-
-* [`producer_plugin`](../producer_plugin/index.md)

docs/01_nodeos/03_plugins/http_plugin/index.md

@@ -29,16 +29,6 @@ Config Options for eosio::http_plugin:
                                         The local IP and port to listen for
                                         incoming http connections; set blank to
                                         disable.
-  --https-server-address arg            The local IP and port to listen for
-                                        incoming https connections; leave blank
-                                        to disable.
-  --https-certificate-chain-file arg    Filename with the certificate chain to
-                                        present on https connections. PEM
-                                        format. Required for https.
-  --https-private-key-file arg          Filename with https private key in PEM
-                                        format. Required for https
-  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use:
-                                        secp384r1 or prime256v1
   --access-control-allow-origin arg     Specify the Access-Control-Allow-Origin
                                         to be returned on each request
   --access-control-allow-headers arg    Specify the Access-Control-Allow-Header

docs/01_nodeos/03_plugins/index.md

@@ -11,7 +11,6 @@ For information on specific plugins, just select from the list below:
 * [`chain_api_plugin`](chain_api_plugin/index.md)
 * [`chain_plugin`](chain_plugin/index.md)
 * [`db_size_api_plugin`](db_size_api_plugin/index.md)
-* [`http_client_plugin`](http_client_plugin/index.md)
 * [`http_plugin`](http_plugin/index.md)
 * [`net_api_plugin`](net_api_plugin/index.md)
 * [`net_plugin`](net_plugin/index.md)

docs/01_nodeos/03_plugins/producer_plugin/index.md

@@ -39,10 +39,6 @@ Config Options for eosio::producer_plugin:
   -p [ --producer-name ] arg            ID of producer controlled by this node
                                         (e.g. inita; may specify multiple
                                         times)
-  --private-key arg                     (DEPRECATED - Use signature-provider
-                                        instead) Tuple of [public key, WIF
-                                        private key] (may specify multiple
-                                        times)
   --signature-provider arg (=EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV=KEY:5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3)
                                         Key=Value pairs in the form
                                         <public-key>=<provider-spec>

docs/01_nodeos/06_logging/01_logging-levels.md

@@ -81,7 +81,7 @@ Sample `logging.json`:
   - Answers the question: is this useful information for a user that is monitoring the log output. Care should be taken not to create excessive log output; similar to `info` level logging.
   - Enabling `debug` level logging should provide greater insight into behavior without overwhelming the output with log entries.
   - `debug` level should not be used for *trace* level logging; to that end, use `all` (see below).
-  - Like `info`, no `debug` level logging should be produced for every transaction. There are specific transaction level loggers dedicated to transaction level logging: `transaction`, `transaction_trace_failure`, `transaction_trace_success`, `transaction_failure_tracing`, `transaction_success_tracing`.
+  - Like `info`, no `debug` level logging should be produced for every transaction. There are specific transaction level loggers dedicated to transaction level logging: `transaction`, `transaction_trace_failure`, `transaction_trace_success`, `transaction_failure_tracing`, `transaction_success_tracing`, `transient_trx_success_tracing`, `transient_trx_failure_tracing`.
 * `all` (trace) - For logging that would be overwhelming if `debug` level logging were used.
   - Can be used for trace level logging. Only used in a few places and not completely supported.
   - *Note*: In the future a different logging library may provide better trace level logging support. The current logging framework is not performant enough to enable excess trace level output.

docs/03_keosd/10_usage.md

@@ -44,16 +44,6 @@ Config Options for eosio::http_plugin:
   --http-server-address arg             The local IP and port to listen for
                                         incoming http connections; leave blank
                                         to disable.
-  --https-server-address arg            The local IP and port to listen for
-                                        incoming https connections; leave blank
-                                        to disable.
-  --https-certificate-chain-file arg    Filename with the certificate chain to
-                                        present on https connections. PEM
-                                        format. Required for https.
-  --https-private-key-file arg          Filename with https private key in PEM
-                                        format. Required for https
-  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use:
-                                        secp384r1 or prime256v1
   --access-control-allow-origin arg     Specify the Access-Control-Allow-Origin
                                         to be returned on each request.
   --access-control-allow-headers arg    Specify the Access-Control-Allow-Header

docs/10_utilities/eosio-blocklog.md

@@ -1,38 +0,0 @@
----
-content_title: eosio-blocklog
-link_text: eosio-blocklog
----
-
-`eosio-blocklog` is a command-line interface (CLI) utility that allows node operators to perform low-level tasks on the block logs created by a `nodeos` instance. `eosio-blocklog` can perform one of the following operations:
-
-* Convert a range of blocks to JSON format, as single objects or array.
-* Generate `blocks.index` from `blocks.log` in blocks directory.
-* Trim `blocks.log` and `blocks.index` between a range of blocks.
-* Perform consistency test between `blocks.log` and `blocks.index`.
-* Output the results of the operation to a file or `stdout` (default).
-
-## Usage
-```sh
-eosio-blocklog <options> ...
-```
-
-## Options
-
-Option (=default) | Description
--|-
-`--blocks-dir arg (="blocks")` | The location of the blocks directory (absolute path or relative to the current directory)
-`-o [ --output-file ] arg` | The file to write the generated output to (absolute or relative path). If not specified then output is to `stdout`
-`-f [ --first ] arg (=0)` | The first block number to log or the first block to keep if `trim-blocklog` specified
-`-l [ --last ] arg (=4294967295)` | the last block number to log or the last block to keep if `trim-blocklog` specified
-`--no-pretty-print` | Do not pretty print the output. Useful if piping to `jq` to improve performance
-`--as-json-array` | Print out JSON blocks wrapped in JSON array (otherwise the output is free-standing JSON objects)
-`--make-index` | Create `blocks.index` from `blocks.log`. Must give `blocks-dir` location. Give `output-file` relative to current directory or absolute path (default is `<blocks-dir>/blocks.index`)
-`--trim-blocklog` | Trim `blocks.log` and `blocks.index`. Must give `blocks-dir` and `first` and/or `last` options.
-`--smoke-test` | Quick test that `blocks.log` and `blocks.index` are well formed and agree with each other
-`-h [ --help ]` | Print this help message and exit
-
-## Remarks
-
-When `eosio-blocklog` is launched, the utility attempts to perform the specified operation, then yields the following possible outcomes:
-* If successful, the selected operation is performed and the utility terminates with a zero error code (no error).
-* If unsuccessful, the utility outputs an error to `stderr` and terminates with a non-zero error code (indicating an error).

docs/10_utilities/index.md

@@ -5,5 +5,4 @@ link_text: Antelope Utilities
 
 This section contains documentation for additional utilities that complement or extend `nodeos` and potentially other Antelope software:
 
-* [eosio-blocklog](eosio-blocklog.md) - Low-level utility for node operators to interact with block log files.
 * [trace_api_util](trace_api_util.md) - Low-level utility for performing tasks associated with the [Trace API](../01_nodeos/03_plugins/trace_api_plugin/index.md).

libraries/CMakeLists.txt

@@ -12,6 +12,7 @@ add_subdirectory( wasm-jit )
 add_subdirectory( chainbase )
 set(APPBASE_ENABLE_AUTO_VERSION OFF CACHE BOOL "enable automatic discovery of version via 'git describe'")
 add_subdirectory( appbase )
+add_subdirectory( custom_appbase )
 add_subdirectory( chain )
 add_subdirectory( testing )
 add_subdirectory( version )
@@ -27,3 +28,4 @@ set(ENABLE_PROFILE OFF CACHE BOOL "Enable for profile builds")
 if(eos-vm IN_LIST EOSIO_WASM_RUNTIMES OR eos-vm-jit IN_LIST EOSIO_WASM_RUNTIMES)
 add_subdirectory( eos-vm )
 endif()
+add_subdirectory( prometheus )

libraries/chain/abi_serializer.cpp

@@ -70,9 +70,9 @@ namespace eosio { namespace chain {
       );
    }
 
-   abi_serializer::abi_serializer( const abi_def& abi, const yield_function_t& yield ) {
+   abi_serializer::abi_serializer( abi_def abi, const yield_function_t& yield ) {
       configure_built_in_types();
-      set_abi(abi, yield);
+      set_abi(std::move(abi), yield);
    }
 
    abi_serializer::abi_serializer( const abi_def& abi, const fc::microseconds& max_serialization_time) {
@@ -128,11 +128,19 @@ namespace eosio { namespace chain {
       built_in_types.emplace("extended_asset",            pack_unpack<extended_asset>());
    }
 
-   void abi_serializer::set_abi(const abi_def& abi, const yield_function_t& yield) {
+   void abi_serializer::set_abi(abi_def abi, const yield_function_t& yield) {
       impl::abi_traverse_context ctx(yield);
 
       EOS_ASSERT(starts_with(abi.version, "eosio::abi/1."), unsupported_abi_version_exception, "ABI has an unsupported version");
 
+      size_t types_size = abi.types.size();
+      size_t structs_size = abi.structs.size();
+      size_t actions_size = abi.actions.size();
+      size_t tables_size = abi.tables.size();
+      size_t error_messages_size = abi.error_messages.size();
+      size_t variants_size = abi.variants.value.size();
+      size_t action_results_size = abi.action_results.value.size();
+
       typedefs.clear();
       structs.clear();
       actions.clear();
@@ -141,41 +149,41 @@ namespace eosio { namespace chain {
       variants.clear();
       action_results.clear();
 
-      for( const auto& st : abi.structs )
-         structs[st.name] = st;
+      for( auto& st : abi.structs )
+         structs[st.name] = std::move(st);
 
-      for( const auto& td : abi.types ) {
+      for( auto& td : abi.types ) {
          EOS_ASSERT(!_is_type(td.new_type_name, ctx), duplicate_abi_type_def_exception,
                     "type already exists", ("new_type_name",impl::limit_size(td.new_type_name)));
-         typedefs[td.new_type_name] = td.type;
+         typedefs[std::move(td.new_type_name)] = std::move(td.type);
       }
 
-      for( const auto& a : abi.actions )
-         actions[a.name] = a.type;
+      for( auto& a : abi.actions )
+         actions[std::move(a.name)] = std::move(a.type);
 
-      for( const auto& t : abi.tables )
-         tables[t.name] = t.type;
+      for( auto& t : abi.tables )
+         tables[std::move(t.name)] = std::move(t.type);
 
-      for( const auto& e : abi.error_messages )
-         error_messages[e.error_code] = e.error_msg;
+      for( auto& e : abi.error_messages )
+         error_messages[std::move(e.error_code)] = std::move(e.error_msg);
 
-      for( const auto& v : abi.variants.value )
-         variants[v.name] = v;
+      for( auto& v : abi.variants.value )
+         variants[v.name] = std::move(v);
 
-      for( const auto& r : abi.action_results.value )
-         action_results[r.name] = r.result_type;
+      for( auto& r : abi.action_results.value )
+         action_results[std::move(r.name)] = std::move(r.result_type);
 
       /**
        *  The ABI vector may contain duplicates which would make it
        *  an invalid ABI
        */
-      EOS_ASSERT( typedefs.size() == abi.types.size(), duplicate_abi_type_def_exception, "duplicate type definition detected" );
-      EOS_ASSERT( structs.size() == abi.structs.size(), duplicate_abi_struct_def_exception, "duplicate struct definition detected" );
-      EOS_ASSERT( actions.size() == abi.actions.size(), duplicate_abi_action_def_exception, "duplicate action definition detected" );
-      EOS_ASSERT( tables.size() == abi.tables.size(), duplicate_abi_table_def_exception, "duplicate table definition detected" );
-      EOS_ASSERT( error_messages.size() == abi.error_messages.size(), duplicate_abi_err_msg_def_exception, "duplicate error message definition detected" );
-      EOS_ASSERT( variants.size() == abi.variants.value.size(), duplicate_abi_variant_def_exception, "duplicate variant definition detected" );
-      EOS_ASSERT( action_results.size() == abi.action_results.value.size(), duplicate_abi_action_results_def_exception, "duplicate action results definition detected" );
+      EOS_ASSERT( typedefs.size() == types_size, duplicate_abi_type_def_exception, "duplicate type definition detected" );
+      EOS_ASSERT( structs.size() == structs_size, duplicate_abi_struct_def_exception, "duplicate struct definition detected" );
+      EOS_ASSERT( actions.size() == actions_size, duplicate_abi_action_def_exception, "duplicate action definition detected" );
+      EOS_ASSERT( tables.size() == tables_size, duplicate_abi_table_def_exception, "duplicate table definition detected" );
+      EOS_ASSERT( error_messages.size() == error_messages_size, duplicate_abi_err_msg_def_exception, "duplicate error message definition detected" );
+      EOS_ASSERT( variants.size() == variants_size, duplicate_abi_variant_def_exception, "duplicate variant definition detected" );
+      EOS_ASSERT( action_results.size() == action_results_size, duplicate_abi_action_results_def_exception, "duplicate action results definition detected" );
 
       validate(ctx);
    }

libraries/chain/apply_context.cpp

@@ -182,7 +182,7 @@ void apply_context::exec_one()
       print_debug(receiver, trace);
    }
 
-   if (auto dm_logger = control.get_deep_mind_logger())
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient()))
    {
       dm_logger->on_end_action();
    }
@@ -289,7 +289,7 @@ void apply_context::require_recipient( account_name recipient ) {
          schedule_action( action_ordinal, recipient, false )
       );
 
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_require_recipient();
       }
    }
@@ -354,7 +354,7 @@ void apply_context::execute_inline( action&& a ) {
                   "inline action too big for nonprivileged account ${account}", ("account", a.account));
    }
    // No need to check authorization if replaying irreversible blocks or contract is privileged
-   if( !control.skip_auth_check() && !privileged ) {
+   if( !control.skip_auth_check() && !privileged && !trx_context.is_read_only() ) {
       try {
          control.get_authorization_manager()
                 .check_authorization( {a},
@@ -363,7 +363,7 @@ void apply_context::execute_inline( action&& a ) {
                                       control.pending_block_time() - trx_context.published,
                                       std::bind(&transaction_context::checktime, &this->trx_context),
                                       false,
-                                      trx_context.is_dry_run(),
+                                      trx_context.is_dry_run(), // check_but_dont_fail
                                       inherited_authorizations
                                     );
 
@@ -394,7 +394,7 @@ void apply_context::execute_inline( action&& a ) {
       schedule_action( std::move(a), inline_receiver, false )
    );
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_send_inline();
    }
 }
@@ -419,13 +419,14 @@ void apply_context::execute_context_free_inline( action&& a ) {
       schedule_action( std::move(a), inline_receiver, true )
    );
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_send_context_free_inline();
    }
 }
 
 
 void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, account_name payer, transaction&& trx, bool replace_existing ) {
+   EOS_ASSERT( !trx_context.is_read_only(), transaction_exception, "cannot schedule a deferred transaction from within a readonly transaction" );
    EOS_ASSERT( trx.context_free_actions.size() == 0, cfa_inside_generated_tx, "context free actions are not currently allowed in generated transactions" );
 
    bool enforce_actor_whitelist_blacklist = trx_context.enforce_whiteblacklist && control.is_speculative_block()
@@ -557,7 +558,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
                   subjective_block_production_exception,
                   "Replacing a deferred transaction is temporarily disabled." );
 
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", ptr->id)), "deferred_trx", "cancel", "deferred_trx_cancel");
       }
 
@@ -575,7 +576,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
          trx_id_for_new_obj = ptr->trx_id;
       }
 
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_cancel_deferred(deep_mind_handler::operation_qualifier::modify, *ptr);
       }
 
@@ -592,7 +593,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
 
          trx_size = gtx.set( trx );
 
-         if (auto dm_logger = control.get_deep_mind_logger()) {
+         if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
             dm_logger->on_send_deferred(deep_mind_handler::operation_qualifier::modify, gtx);
             dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", gtx.id)), "deferred_trx", "update", "deferred_trx_add");
          }
@@ -609,7 +610,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
 
          trx_size = gtx.set( trx );
 
-         if (auto dm_logger = control.get_deep_mind_logger()) {
+         if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
             dm_logger->on_send_deferred(deep_mind_handler::operation_qualifier::none, gtx);
             dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", gtx.id)), "deferred_trx", "add", "deferred_trx_add");
          }
@@ -626,10 +627,11 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
 }
 
 bool apply_context::cancel_deferred_transaction( const uint128_t& sender_id, account_name sender ) {
+   EOS_ASSERT( !trx_context.is_read_only(), transaction_exception, "cannot cancel a deferred transaction from within a readonly transaction" );
    auto& generated_transaction_idx = db.get_mutable_index<generated_transaction_multi_index>();
    const auto* gto = db.find<generated_transaction_object,by_sender_id>(boost::make_tuple(sender, sender_id));
    if ( gto ) {
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_cancel_deferred(deep_mind_handler::operation_qualifier::none, *gto);
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", gto->id)), "deferred_trx", "cancel", "deferred_trx_cancel");
       }
@@ -670,7 +672,7 @@ const table_id_object& apply_context::find_or_create_table( name code, name scop
       return *existing_tid;
    }
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       std::string event_id = RAM_EVENT_ID("${code}:${scope}:${table}",
          ("code", code)
          ("scope", scope)
@@ -687,14 +689,14 @@ const table_id_object& apply_context::find_or_create_table( name code, name scop
       t_id.table = table;
       t_id.payer = payer;
 
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_create_table(t_id);
       }
    });
 }
 
 void apply_context::remove_table( const table_id_object& tid ) {
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       std::string event_id = RAM_EVENT_ID("${code}:${scope}:${table}",
          ("code", tid.code)
          ("scope", tid.scope)
@@ -705,7 +707,7 @@ void apply_context::remove_table( const table_id_object& tid ) {
 
    update_db_usage(tid.payer, - config::billable_size_v<table_id_object>);
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_remove_table(tid);
    }
 
@@ -778,11 +780,13 @@ int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t b
 }
 
 int apply_context::db_store_i64( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ) {
+   EOS_ASSERT( !trx_context.is_read_only(), table_operation_not_permitted, "cannot store a db record when executing a readonly transaction" );
    return db_store_i64( receiver, scope, table, payer, id, buffer, buffer_size);
 }
 
 int apply_context::db_store_i64( name code, name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ) {
 //   require_write_lock( scope );
+   EOS_ASSERT( !trx_context.is_read_only(), table_operation_not_permitted, "cannot store a db record when executing a readonly transaction" );
    const auto& tab = find_or_create_table( code, scope, table, payer );
    auto tableid = tab.id;
 
@@ -801,7 +805,7 @@ int apply_context::db_store_i64( name code, name scope, name table, const accoun
 
    int64_t billable_size = (int64_t)(buffer_size + config::billable_size_v<key_value_object>);
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       std::string event_id = RAM_EVENT_ID("${table_code}:${scope}:${table_name}:${primkey}",
          ("table_code", tab.code)
          ("scope", tab.scope)
@@ -813,7 +817,7 @@ int apply_context::db_store_i64( name code, name scope, name table, const accoun
 
    update_db_usage( payer, billable_size);
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_db_store_i64(tab, obj);
    }
 
@@ -822,6 +826,7 @@ int apply_context::db_store_i64( name code, name scope, name table, const accoun
 }
 
 void apply_context::db_update_i64( int iterator, account_name payer, const char* buffer, size_t buffer_size ) {
+   EOS_ASSERT( !trx_context.is_read_only(), table_operation_not_permitted, "cannot update a db record when executing a readonly transaction" );
    const key_value_object& obj = keyval_cache.get( iterator );
 
    const auto& table_obj = keyval_cache.get_table( obj.t_id );
@@ -836,7 +841,7 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char*
    if( payer == account_name() ) payer = obj.payer;
 
    std::string event_id;
-   if (control.get_deep_mind_logger() != nullptr) {
+   if (control.get_deep_mind_logger(trx_context.is_transient()) != nullptr) {
       event_id = RAM_EVENT_ID("${table_code}:${scope}:${table_name}:${primkey}",
          ("table_code", table_obj.code)
          ("scope", table_obj.scope)
@@ -847,27 +852,27 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char*
 
    if( account_name(obj.payer) != payer ) {
       // refund the existing payer
-      if (auto dm_logger = control.get_deep_mind_logger())
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient()))
       {
          dm_logger->on_ram_trace(std::string(event_id), "table_row", "remove", "primary_index_update_remove_old_payer");
       }
       update_db_usage( obj.payer,  -(old_size) );
       // charge the new payer
-      if (auto dm_logger = control.get_deep_mind_logger())
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient()))
       {
          dm_logger->on_ram_trace(std::move(event_id), "table_row", "add", "primary_index_update_add_new_payer");
       }
       update_db_usage( payer,  (new_size));
    } else if(old_size != new_size) {
       // charge/refund the existing payer the difference
-      if (auto dm_logger = control.get_deep_mind_logger())
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient()))
       {
          dm_logger->on_ram_trace(std::move(event_id) , "table_row", "update", "primary_index_update");
       }
       update_db_usage( obj.payer, new_size - old_size);
    }
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_db_update_i64(table_obj, obj, payer, buffer, buffer_size);
    }
 
@@ -878,6 +883,7 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char*
 }
 
 void apply_context::db_remove_i64( int iterator ) {
+   EOS_ASSERT( !trx_context.is_read_only(), table_operation_not_permitted, "cannot remove a db record when executing a readonly transaction" );
    const key_value_object& obj = keyval_cache.get( iterator );
 
    const auto& table_obj = keyval_cache.get_table( obj.t_id );
@@ -885,7 +891,7 @@ void apply_context::db_remove_i64( int iterator ) {
 
 //   require_write_lock( table_obj.scope );
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       std::string event_id = RAM_EVENT_ID("${table_code}:${scope}:${table_name}:${primkey}",
          ("table_code", table_obj.code)
          ("scope", table_obj.scope)
@@ -897,7 +903,7 @@ void apply_context::db_remove_i64( int iterator ) {
 
    update_db_usage( obj.payer,  -(obj.value.size() + config::billable_size_v<key_value_object>) );
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_db_remove_i64(table_obj, obj);
    }
 
@@ -1029,17 +1035,27 @@ int apply_context::db_end_i64( name code, name scope, name table ) {
 
 uint64_t apply_context::next_global_sequence() {
    const auto& p = control.get_dynamic_global_properties();
-   db.modify( p, [&]( auto& dgp ) {
-      ++dgp.global_action_sequence;
-   });
-   return p.global_action_sequence;
+   if ( trx_context.is_read_only() ) {
+      // To avoid confusion of duplicated global sequence number, hard code to be 0.
+      return 0;
+   } else {
+      db.modify( p, [&]( auto& dgp ) {
+         ++dgp.global_action_sequence;
+      });
+      return p.global_action_sequence;
+   }
 }
 
 uint64_t apply_context::next_recv_sequence( const account_metadata_object& receiver_account ) {
-   db.modify( receiver_account, [&]( auto& ra ) {
-      ++ra.recv_sequence;
-   });
-   return receiver_account.recv_sequence;
+   if ( trx_context.is_read_only() ) {
+      // To avoid confusion of duplicated receive sequence number, hard code to be 0.
+      return 0;
+   } else {
+      db.modify( receiver_account, [&]( auto& ra ) {
+         ++ra.recv_sequence;
+      });
+      return receiver_account.recv_sequence;
+   }
 }
 uint64_t apply_context::next_auth_sequence( account_name actor ) {
    const auto& amo = db.get<account_metadata_object,by_name>( actor );

libraries/chain/authorization_manager.cpp

@@ -134,6 +134,7 @@ namespace eosio { namespace chain {
                                                                       permission_name name,
                                                                       permission_id_type parent,
                                                                       const authority& auth,
+                                                                      bool is_trx_transient,
                                                                       time_point initial_creation_time
                                                                     )
    {
@@ -158,7 +159,7 @@ namespace eosio { namespace chain {
          p.last_updated = creation_time;
          p.auth         = auth;
 
-         if (auto dm_logger = _control.get_deep_mind_logger()) {
+         if (auto dm_logger = _control.get_deep_mind_logger(is_trx_transient)) {
             dm_logger->on_create_permission(p);
          }
       });
@@ -169,6 +170,7 @@ namespace eosio { namespace chain {
                                                                       permission_name name,
                                                                       permission_id_type parent,
                                                                       authority&& auth,
+                                                                      bool is_trx_transient,
                                                                       time_point initial_creation_time
                                                                     )
    {
@@ -193,20 +195,20 @@ namespace eosio { namespace chain {
          p.last_updated = creation_time;
          p.auth         = std::move(auth);
 
-         if (auto dm_logger = _control.get_deep_mind_logger()) {
+         if (auto dm_logger = _control.get_deep_mind_logger(is_trx_transient)) {
             dm_logger->on_create_permission(p);
          }
       });
       return perm;
    }
 
-   void authorization_manager::modify_permission( const permission_object& permission, const authority& auth ) {
+   void authorization_manager::modify_permission( const permission_object& permission, const authority& auth, bool is_trx_transient ) {
       for(const key_weight& k: auth.keys)
          EOS_ASSERT(k.key.which() < _db.get<protocol_state_object>().num_supported_key_types, unactivated_key_type,
            "Unactivated key type used when modifying permission");
 
       _db.modify( permission, [&](permission_object& po) {
-         auto dm_logger = _control.get_deep_mind_logger();
+         auto dm_logger = _control.get_deep_mind_logger(is_trx_transient);
 
          std::optional<permission_object> old_permission;
          if (dm_logger) {
@@ -216,13 +218,13 @@ namespace eosio { namespace chain {
          po.auth = auth;
          po.last_updated = _control.pending_block_time();
 
-         if (auto dm_logger = _control.get_deep_mind_logger()) {
+         if (auto dm_logger = _control.get_deep_mind_logger(is_trx_transient)) {
             dm_logger->on_modify_permission(*old_permission, po);
          }
       });
    }
 
-   void authorization_manager::remove_permission( const permission_object& permission ) {
+   void authorization_manager::remove_permission( const permission_object& permission, bool is_trx_transient ) {
       const auto& index = _db.template get_index<permission_index, by_parent>();
       auto range = index.equal_range(permission.id);
       EOS_ASSERT( range.first == range.second, action_validate_exception,
@@ -230,7 +232,7 @@ namespace eosio { namespace chain {
 
       _db.get_mutable_index<permission_usage_index>().remove_object( permission.usage_id._id );
 
-      if (auto dm_logger = _control.get_deep_mind_logger()) {
+      if (auto dm_logger = _control.get_deep_mind_logger(is_trx_transient)) {
          dm_logger->on_remove_permission(permission);
       }
 

libraries/chain/controller.cpp

@@ -34,6 +34,7 @@
 #include <fc/variant_object.hpp>
 
 #include <new>
+#include <shared_mutex>
 
 namespace eosio { namespace chain {
 
@@ -224,7 +225,6 @@ struct controller_impl {
    std::optional<pending_state>    pending;
    block_state_ptr                 head;
    fork_database                   fork_db;
-   wasm_interface                  wasmif;
    resource_limits_manager         resource_limits;
    authorization_manager           authorization;
    protocol_feature_manager        protocol_features;
@@ -238,17 +238,25 @@ struct controller_impl {
    uint32_t                        snapshot_head_block = 0;
    struct chain; // chain is a namespace so use an embedded type for the named_thread_pool tag
    named_thread_pool<chain>        thread_pool;
-   platform_timer                  timer;
    deep_mind_handler*              deep_mind_logger = nullptr;
    bool                            okay_to_print_integrity_hash_on_stop = false;
+
+   std::thread::id                 main_thread_id;
+   thread_local static platform_timer timer; // a copy for main thread and each read-only thread
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
-   vm::wasm_allocator               wasm_alloc;
+   thread_local static vm::wasm_allocator wasm_alloc; // a copy for main thread and each read-only thread
 #endif
+   // Ideally wasmif should be thread_local which must be a static.
+   // Unittests can create multiple controller objects (testers) at the same time,
+   // which overwrites the same static wasmif, is used for eosvmoc too.
+   wasm_interface  wasmif;  // used by main thread and all threads for EOSVMOC
+   thread_local static std::unique_ptr<wasm_interface> wasmif_thread_local; // a copy for each read-only thread, used by eos-vm and eos-vm-jit
 
    typedef pair<scope_name,action_name>                   handler_key;
    map< account_name, map<handler_key, apply_handler> >   apply_handlers;
    unordered_map< builtin_protocol_feature_t, std::function<void(controller_impl&)>, enum_hash<builtin_protocol_feature_t> > protocol_feature_activation_handlers;
 
+
    void pop_block() {
       auto prev = fork_db.get_block( head->header.previous );
 
@@ -293,16 +301,17 @@ struct controller_impl {
     db( cfg.state_dir,
         cfg.read_only ? database::read_only : database::read_write,
         cfg.state_size, false, cfg.db_map_mode ),
-    blog( cfg.blocks_dir, cfg.prune_config ),
+    blog( cfg.blocks_dir, cfg.blog ),
     fork_db( cfg.blocks_dir / config::reversible_blocks_dir_name ),
-    wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config, !cfg.profile_accounts.empty() ),
-    resource_limits( db, [&s]() { return s.get_deep_mind_logger(); }),
+    resource_limits( db, [&s](bool is_trx_transient) { return s.get_deep_mind_logger(is_trx_transient); }),
     authorization( s, db ),
-    protocol_features( std::move(pfs), [&s]() { return s.get_deep_mind_logger(); } ),
+    protocol_features( std::move(pfs), [&s](bool is_trx_transient) { return s.get_deep_mind_logger(is_trx_transient); } ),
     conf( cfg ),
     chain_id( chain_id ),
     read_mode( cfg.read_mode ),
-    thread_pool()
+    thread_pool(),
+    main_thread_id( std::this_thread::get_id() ),
+    wasmif( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty() )
    {
       fork_db.open( [this]( block_timestamp_type timestamp,
                             const flat_set<digest_type>& cur_features,
@@ -328,7 +337,7 @@ struct controller_impl {
       set_activation_handler<builtin_protocol_feature_t::crypto_primitives>();
 
       self.irreversible_block.connect([this](const block_state_ptr& bsp) {
-         wasmif.current_lib(bsp->block_num);
+         get_wasm_interface().current_lib(bsp->block_num);
       });
 
 
@@ -384,7 +393,10 @@ struct controller_impl {
    }
 
    void dmlog_applied_transaction(const transaction_trace_ptr& t) {
-      if (auto dm_logger = get_deep_mind_logger()) {
+      // dmlog_applied_transaction is called by push_scheduled_transaction
+      // where transient transactions are not possible, and by push_transaction
+      // only when the transaction is not transient
+      if (auto dm_logger = get_deep_mind_logger(false)) {
          dm_logger->on_applied_transaction(self.head_block_num() + 1, t);
       }
    }
@@ -392,20 +404,21 @@ struct controller_impl {
    void log_irreversible() {
       EOS_ASSERT( fork_db.root(), fork_database_exception, "fork database not properly initialized" );
 
-      const auto& log_head = blog.head();
+      const block_id_type log_head_id = blog.head_id();
+      const bool valid_log_head = !log_head_id.empty();
 
-      auto lib_num = log_head ? log_head->block_num() : (blog.first_block_num() - 1);
+      const auto lib_num = valid_log_head ? block_header::num_from_id(log_head_id) : (blog.first_block_num() - 1);
 
       auto root_id = fork_db.root()->id;
 
-      if( log_head ) {
-         EOS_ASSERT( root_id == blog.head_id(), fork_database_exception, "fork database root does not match block log head" );
+      if( valid_log_head ) {
+         EOS_ASSERT( root_id == log_head_id, fork_database_exception, "fork database root does not match block log head" );
       } else {
          EOS_ASSERT( fork_db.root()->block_num == lib_num, fork_database_exception,
                      "empty block log expects the first appended block to build off a block that is not the fork database root. root block number: ${block_num}, lib: ${lib_num}", ("block_num", fork_db.root()->block_num) ("lib_num", lib_num) );
       }
 
-      auto fork_head = (read_mode == db_read_mode::IRREVERSIBLE) ? fork_db.pending_head() : fork_db.head();
+      const auto fork_head = fork_db_head();
 
       if( fork_head->dpos_irreversible_blocknum <= lib_num )
          return;
@@ -483,12 +496,12 @@ struct controller_impl {
    }
 
    void replay(std::function<bool()> check_shutdown) {
-      if( !blog.head() && !fork_db.root() ) {
+      auto blog_head = blog.head();
+      if( !blog_head && !fork_db.root() ) {
          fork_db.reset( *head );
          return;
       }
 
-      auto blog_head = blog.head();
       replaying = true;
       auto start_block_num = head->block_num + 1;
       auto start = fc::time_point::now();
@@ -570,8 +583,8 @@ struct controller_impl {
       ilog( "Starting initialization from snapshot, this may take a significant amount of time" );
       try {
          snapshot->validate();
-         if( blog.head() ) {
-            read_from_snapshot( snapshot, blog.first_block_num(), blog.head()->block_num() );
+         if( auto blog_head = blog.head() ) {
+            read_from_snapshot( snapshot, blog.first_block_num(), blog_head->block_num() );
          } else {
             read_from_snapshot( snapshot, 0, std::numeric_limits<uint32_t>::max() );
             const uint32_t lib_num = head->block_num;
@@ -587,10 +600,6 @@ struct controller_impl {
          elog( "Failed initialization from snapshot - db storage not configured to have enough storage for the provided snapshot, please increase and retry snapshot" );
          shutdown();
       }
-
-      if (conf.prune_config && conf.prune_config->prune_blocks == 0) {
-         blog.remove();
-      }
    }
 
    void startup(std::function<void()> shutdown, std::function<bool()> check_shutdown, const genesis_state& genesis) {
@@ -624,10 +633,6 @@ struct controller_impl {
          blog.reset( genesis, head->block );
       }
       init(check_shutdown);
-
-      if (conf.prune_config && conf.prune_config->prune_blocks == 0) {
-         blog.remove();
-      }
    }
 
    void startup(std::function<void()> shutdown, std::function<bool()> check_shutdown) {
@@ -637,15 +642,15 @@ struct controller_impl {
       this->shutdown = shutdown;
       uint32_t lib_num = fork_db.root()->block_num;
       auto first_block_num = blog.first_block_num();
-      if( blog.head() ) {
-         EOS_ASSERT( first_block_num <= lib_num && lib_num <= blog.head()->block_num(),
+      if( auto blog_head = blog.head() ) {
+         EOS_ASSERT( first_block_num <= lib_num && lib_num <= blog_head->block_num(),
                      block_log_exception,
                      "block log (ranging from ${block_log_first_num} to ${block_log_last_num}) does not contain the last irreversible block (${fork_db_lib})",
                      ("block_log_first_num", first_block_num)
-                     ("block_log_last_num", blog.head()->block_num())
+                     ("block_log_last_num", blog_head->block_num())
                      ("fork_db_lib", lib_num)
          );
-         lib_num = blog.head()->block_num();
+         lib_num = blog_head->block_num();
       } else {
          if( first_block_num != (lib_num + 1) ) {
             blog.reset( chain_id, lib_num + 1 );
@@ -658,10 +663,6 @@ struct controller_impl {
       head = fork_db.head();
 
       init(check_shutdown);
-
-      if (conf.prune_config && conf.prune_config->prune_blocks == 0) {
-         blog.remove();
-      }
    }
 
 
@@ -712,7 +713,8 @@ struct controller_impl {
 
       protocol_features.init( db );
 
-      if (auto dm_logger = get_deep_mind_logger()) {
+      // At startup, no transaction specific logging is possible
+      if (auto dm_logger = get_deep_mind_logger(false)) {
          dm_logger->on_startup(db, head->block_num);
       }
 
@@ -1028,22 +1030,23 @@ struct controller_impl {
       });
 
       const auto& owner_permission  = authorization.create_permission(name, config::owner_name, 0,
-                                                                      owner, initial_timestamp );
+                                                                      owner, false, initial_timestamp );
       const auto& active_permission = authorization.create_permission(name, config::active_name, owner_permission.id,
-                                                                      active, initial_timestamp );
+                                                                      active, false, initial_timestamp );
 
-      resource_limits.initialize_account(name);
+      resource_limits.initialize_account(name, false);
 
       int64_t ram_delta = config::overhead_per_account_ram_bytes;
       ram_delta += 2*config::billable_size_v<permission_object>;
       ram_delta += owner_permission.auth.get_billable_size();
       ram_delta += active_permission.auth.get_billable_size();
 
-      if (auto dm_logger = get_deep_mind_logger()) {
+      // This is only called at startup, no transaction specific logging is possible
+      if (auto dm_logger = get_deep_mind_logger(false)) {
          dm_logger->on_ram_trace(RAM_EVENT_ID("${name}", ("name", name)), "account", "add", "newaccount");
       }
 
-      resource_limits.add_pending_ram_usage(name, ram_delta);
+      resource_limits.add_pending_ram_usage(name, ram_delta, false); // false for doing dm logging
       resource_limits.verify_account_ram_usage(name);
    }
 
@@ -1096,17 +1099,24 @@ struct controller_impl {
                                                                              config::majority_producers_permission_name,
                                                                              active_permission.id,
                                                                              active_producers_authority,
+                                                                             false,
                                                                              genesis.initial_timestamp );
                                             authorization.create_permission( config::producers_account_name,
                                                                              config::minority_producers_permission_name,
                                                                              majority_permission.id,
                                                                              active_producers_authority,
+                                                                             false,
                                                                              genesis.initial_timestamp );
 
    }
 
    // The returned scoped_exit should not exceed the lifetime of the pending which existed when make_block_restore_point was called.
-   fc::scoped_exit<std::function<void()>> make_block_restore_point() {
+   fc::scoped_exit<std::function<void()>> make_block_restore_point( bool is_read_only = false ) {
+      if ( is_read_only ) {
+         std::function<void()> callback = []() { };
+         return fc::make_scoped_exit( std::move(callback) );
+      }
+
       auto& bb = std::get<building_block>(pending->_block_stage);
       auto orig_trx_receipts_size           = bb._pending_trx_receipts.size();
       auto orig_trx_metas_size              = bb._pending_trx_metas.size();
@@ -1153,14 +1163,14 @@ struct controller_impl {
          etrx.set_reference_block( self.head_block_id() );
       }
 
-      if (auto dm_logger = get_deep_mind_logger()) {
-         dm_logger->on_onerror(etrx);
-      }
-
       transaction_checktime_timer trx_timer(timer);
       const packed_transaction trx( std::move( etrx ) );
       transaction_context trx_context( self, trx, std::move(trx_timer), start );
 
+      if (auto dm_logger = get_deep_mind_logger(trx_context.is_transient())) {
+         dm_logger->on_onerror(etrx);
+      }
+
       trx_context.block_deadline = block_deadline;
       trx_context.max_transaction_time_subjective = max_transaction_time;
       trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time;
@@ -1209,12 +1219,13 @@ struct controller_impl {
    }
 
    int64_t remove_scheduled_transaction( const generated_transaction_object& gto ) {
-      if (auto dm_logger = get_deep_mind_logger()) {
+      // deferred transactions cannot be transient.
+      if (auto dm_logger = get_deep_mind_logger(false)) {
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", gto.id)), "deferred_trx", "remove", "deferred_trx_removed");
       }
 
       int64_t ram_delta = -(config::billable_size_v<generated_transaction_object> + gto.packed_trx.size());
-      resource_limits.add_pending_ram_usage( gto.payer, ram_delta );
+      resource_limits.add_pending_ram_usage( gto.payer, ram_delta, false ); // false for doing dm logging
       // No need to verify_account_ram_usage since we are only reducing memory
 
       db.remove( gto );
@@ -1319,7 +1330,7 @@ struct controller_impl {
 
       uint32_t cpu_time_to_bill_us = billed_cpu_time_us;
 
-      transaction_checktime_timer trx_timer(timer);
+      transaction_checktime_timer trx_timer( timer );
       transaction_context trx_context( self, *trx->packed_trx(), std::move(trx_timer) );
       trx_context.leeway =  fc::microseconds(0); // avoid stealing cpu resource
       trx_context.block_deadline = block_deadline;
@@ -1337,7 +1348,8 @@ struct controller_impl {
          trace->except_ptr = std::current_exception();
          trace->elapsed = fc::time_point::now() - start;
 
-         if (auto dm_logger = get_deep_mind_logger()) {
+         // deferred transactions cannot be transient
+         if (auto dm_logger = get_deep_mind_logger(false)) {
             dm_logger->on_fail_deferred();
          }
       };
@@ -1518,7 +1530,7 @@ struct controller_impl {
       transaction_trace_ptr trace;
       try {
          auto start = fc::time_point::now();
-         const bool check_auth = !self.skip_auth_check() && !trx->implicit();
+         const bool check_auth = !self.skip_auth_check() && !trx->implicit() && !trx->is_read_only();
          const fc::microseconds sig_cpu_usage = trx->signature_cpu_usage();
 
          if( !explicit_billed_cpu_time ) {
@@ -1577,14 +1589,14 @@ struct controller_impl {
             trx_context.exec();
             trx_context.finalize(); // Automatically rounds up network and CPU usage in trace and bills payers if successful
 
-            auto restore = make_block_restore_point();
+            auto restore = make_block_restore_point( trx->is_read_only() );
 
-            if (!trx->implicit()) {
+            trx->billed_cpu_time_us = trx_context.billed_cpu_time_us;
+            if (!trx->implicit() && !trx->is_read_only()) {
                transaction_receipt::status_enum s = (trx_context.delay == fc::seconds(0))
                                                     ? transaction_receipt::executed
                                                     : transaction_receipt::delayed;
                trace->receipt = push_receipt(*trx->packed_trx(), s, trx_context.billed_cpu_time_us, trace->net_usage);
-               trx->billed_cpu_time_us = trx_context.billed_cpu_time_us;
                std::get<building_block>(pending->_block_stage)._pending_trx_metas.emplace_back(trx);
             } else {
                transaction_receipt_header r;
@@ -1594,22 +1606,22 @@ struct controller_impl {
                trace->receipt = r;
             }
 
-            fc::move_append( std::get<building_block>(pending->_block_stage)._action_receipt_digests,
-                             std::move(trx_context.executed_action_receipt_digests) );
+            if ( !trx->is_read_only() ) {
+               fc::move_append( std::get<building_block>(pending->_block_stage)._action_receipt_digests,
+                                std::move(trx_context.executed_action_receipt_digests) );
+                if ( !trx->is_dry_run() ) {
+                   // call the accept signal but only once for this transaction
+                   if (!trx->accepted) {
+                       trx->accepted = true;
+                       emit(self.accepted_transaction, trx);
+                   }
 
-            // call the accept signal but only once for this transaction
-            if (!trx->is_dry_run()) {
-                if (!trx->accepted) {
-                    trx->accepted = true;
-                    emit(self.accepted_transaction, trx);
+                   dmlog_applied_transaction(trace);
+                   emit(self.applied_transaction, std::tie(trace, trx->packed_trx()));
                 }
-
-                dmlog_applied_transaction(trace);
-                emit(self.applied_transaction, std::tie(trace, trx->packed_trx()));
             }
 
-
-            if ( trx->is_dry_run() ) {
+            if ( trx->is_transient() ) {
                // remove trx from pending block by not canceling 'restore'
                trx_context.undo(); // this will happen automatically in destructor, but make it more explicit
             } else if ( pending->_block_status == controller::block_status::ephemeral ) {
@@ -1624,7 +1636,7 @@ struct controller_impl {
                trx_context.squash();
             }
 
-            if( !trx->is_dry_run() ) {
+            if( !trx->is_transient() ) {
                pending->_block_report.total_net_usage += trace->net_usage;
                pending->_block_report.total_cpu_usage_us += trace->receipt->cpu_usage_us;
                pending->_block_report.total_elapsed_time += trace->elapsed;
@@ -1647,7 +1659,7 @@ struct controller_impl {
            handle_exception(wrapper);
          }
 
-         if (!trx->is_dry_run()) {
+         if (!trx->is_transient()) {
             emit(self.accepted_transaction, trx);
             dmlog_applied_transaction(trace);
             emit(self.applied_transaction, std::tie(trace, trx->packed_trx()));
@@ -1673,7 +1685,8 @@ struct controller_impl {
 
       emit( self.block_start, head->block_num + 1 );
 
-      if (auto dm_logger = get_deep_mind_logger()) {
+      // at block level, no transaction specific logging is possible
+      if (auto dm_logger = get_deep_mind_logger(false)) {
          // The head block represents the block just before this one that is about to start, so add 1 to get this block num
          dm_logger->on_start_block(head->block_num + 1);
       }
@@ -1925,7 +1938,8 @@ struct controller_impl {
             EOS_ASSERT( bsp == head, fork_database_exception, "committed block did not become the new head in fork database");
          }
 
-         if (auto dm_logger = get_deep_mind_logger()) {
+         // at block level, no transaction specific logging is possible
+         if (auto dm_logger = get_deep_mind_logger(false)) {
             dm_logger->on_accepted_block(bsp);
          }
 
@@ -2320,7 +2334,8 @@ struct controller_impl {
          ilog("switching forks from ${current_head_id} (block number ${current_head_num}) to ${new_head_id} (block number ${new_head_num})",
               ("current_head_id", head->id)("current_head_num", head->block_num)("new_head_id", new_head->id)("new_head_num", new_head->block_num) );
 
-         if (auto dm_logger = get_deep_mind_logger()) {
+         // not possible to log transaction specific infor when switching forks
+         if (auto dm_logger = get_deep_mind_logger(false)) {
             dm_logger->on_switch_forks(head->id, new_head->id);
          }
 
@@ -2639,23 +2654,65 @@ struct controller_impl {
          trx.set_reference_block( self.head_block_id() );
       }
 
-      if (auto dm_logger = get_deep_mind_logger()) {
+      // onblock transaction cannot be transient
+      if (auto dm_logger = get_deep_mind_logger(false)) {
          dm_logger->on_onblock(trx);
       }
 
       return trx;
    }
 
-   inline deep_mind_handler* get_deep_mind_logger() const {
-      return deep_mind_logger;
+   inline deep_mind_handler* get_deep_mind_logger(bool is_trx_transient) const {
+      // do not perform deep mind logging for read-only and dry-run transactions
+      return is_trx_transient ? nullptr : deep_mind_logger;
    }
 
    uint32_t earliest_available_block_num() const {
       return (blog.first_block_num() != 0) ? blog.first_block_num() : fork_db.root()->block_num;
    }
 
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+   bool is_eos_vm_oc_enabled() const {
+      return ( conf.eosvmoc_tierup || conf.wasm_runtime == wasm_interface::vm_type::eos_vm_oc );
+   }
+#endif
+
+   // only called from non-main threads (read-only trx execution threads)
+   // when producer_plugin starts them
+   void init_thread_local_data() {
+      EOS_ASSERT( !is_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+      if ( is_eos_vm_oc_enabled() )
+         // EOSVMOC needs further initialization of its thread local data
+         wasmif.init_thread_local_data();
+      else
+#endif
+         // Non-EOSVMOC needs a wasmif per thread
+         wasmif_thread_local = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+   }
+
+   bool is_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+
+   wasm_interface& get_wasm_interface() {
+      if ( is_main_thread()
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+          || is_eos_vm_oc_enabled()
+#endif
+         )
+         return wasmif;
+      else
+         return *wasmif_thread_local;
+   }
+
+   block_state_ptr fork_db_head() const;
 }; /// controller_impl
 
+thread_local platform_timer controller_impl::timer;
+#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
+thread_local eosio::vm::wasm_allocator controller_impl::wasm_alloc;
+#endif
+thread_local std::unique_ptr<wasm_interface> controller_impl::wasmif_thread_local;
+
 const resource_limits_manager&   controller::get_resource_limits_manager()const
 {
    return my->resource_limits;
@@ -2726,7 +2783,7 @@ chainbase::database& controller::mutable_db()const { return my->db; }
 
 const fork_database& controller::fork_db()const { return my->fork_db; }
 
-void controller::preactivate_feature( const digest_type& feature_digest ) {
+void controller::preactivate_feature( const digest_type& feature_digest, bool is_trx_transient ) {
    const auto& pfs = my->protocol_features.get_protocol_feature_set();
    auto cur_time = pending_block_time();
 
@@ -2823,7 +2880,7 @@ void controller::preactivate_feature( const digest_type& feature_digest ) {
                ("digest", feature_digest)
    );
 
-   if (auto dm_logger = get_deep_mind_logger()) {
+   if (auto dm_logger = get_deep_mind_logger(is_trx_transient)) {
       const auto feature = pfs.get_protocol_feature(feature_digest);
 
       dm_logger->on_preactivate_feature(feature);
@@ -3008,36 +3065,23 @@ block_state_ptr controller::head_block_state()const {
    return my->head;
 }
 
+block_state_ptr controller_impl::fork_db_head() const {
+   if( read_mode == db_read_mode::IRREVERSIBLE ) {
+      // When in IRREVERSIBLE mode fork_db blocks are marked valid when they become irreversible so that
+      // fork_db.head() returns irreversible block
+      // Use pending_head since this method should return the chain head and not last irreversible.
+      return fork_db.pending_head();
+   } else {
+      return fork_db.head();
+   }
+}
+
 uint32_t controller::fork_db_head_block_num()const {
-   return my->fork_db.head()->block_num;
+   return my->fork_db_head()->block_num;
 }
 
 block_id_type controller::fork_db_head_block_id()const {
-   return my->fork_db.head()->id;
-}
-
-time_point controller::fork_db_head_block_time()const {
-   return my->fork_db.head()->header.timestamp;
-}
-
-account_name  controller::fork_db_head_block_producer()const {
-   return my->fork_db.head()->header.producer;
-}
-
-uint32_t controller::fork_db_pending_head_block_num()const {
-   return my->fork_db.pending_head()->block_num;
-}
-
-block_id_type controller::fork_db_pending_head_block_id()const {
-   return my->fork_db.pending_head()->id;
-}
-
-time_point controller::fork_db_pending_head_block_time()const {
-   return my->fork_db.pending_head()->header.timestamp;
-}
-
-account_name  controller::fork_db_pending_head_block_producer()const {
-   return my->fork_db.pending_head()->header.producer;
+   return my->fork_db_head()->id;
 }
 
 time_point controller::pending_block_time()const {
@@ -3101,7 +3145,7 @@ const global_property_object& controller::get_global_properties()const {
   return my->db.get<global_property_object>();
 }
 
-signed_block_ptr controller::fetch_block_by_id( block_id_type id )const {
+signed_block_ptr controller::fetch_block_by_id( const block_id_type& id )const {
    auto state = my->fork_db.get_block(id);
    if( state && state->block ) return state->block;
    auto bptr = my->blog.read_block_by_num( block_header::num_from_id(id) );
@@ -3109,6 +3153,14 @@ signed_block_ptr controller::fetch_block_by_id( block_id_type id )const {
    return signed_block_ptr();
 }
 
+std::optional<signed_block_header> controller::fetch_block_header_by_id( const block_id_type& id )const {
+   auto state = my->fork_db.get_block(id);
+   if( state && state->block ) return state->header;
+   auto result = my->blog.read_block_header_by_num( block_header::num_from_id(id) );
+   if( result && result->calculate_id() == id ) return result;
+   return {};
+}
+
 signed_block_ptr controller::fetch_block_by_number( uint32_t block_num )const  { try {
    auto blk_state = fetch_block_state_by_number( block_num );
    if( blk_state ) {
@@ -3118,25 +3170,25 @@ signed_block_ptr controller::fetch_block_by_number( uint32_t block_num )const  {
    return my->blog.read_block_by_num(block_num);
 } FC_CAPTURE_AND_RETHROW( (block_num) ) }
 
+std::optional<signed_block_header> controller::fetch_block_header_by_number( uint32_t block_num )const  { try {
+   auto blk_state = fetch_block_state_by_number( block_num );
+   if( blk_state ) {
+      return blk_state->header;
+   }
+
+   return my->blog.read_block_header_by_num(block_num);
+} FC_CAPTURE_AND_RETHROW( (block_num) ) }
+
 block_state_ptr controller::fetch_block_state_by_id( block_id_type id )const {
    auto state = my->fork_db.get_block(id);
    return state;
 }
 
 block_state_ptr controller::fetch_block_state_by_number( uint32_t block_num )const  { try {
-   if( my->read_mode == db_read_mode::IRREVERSIBLE ) {
-      return my->fork_db.search_on_branch( my->fork_db.pending_head()->id, block_num );
-   } else {
-      return my->fork_db.search_on_branch( my->fork_db.head()->id, block_num );
-   }
+   return my->fork_db.search_on_branch( fork_db_head_block_id(), block_num );
 } FC_CAPTURE_AND_RETHROW( (block_num) ) }
 
 block_id_type controller::get_block_id_for_num( uint32_t block_num )const { try {
-   const auto& tapos_block_summary = db().get<block_summary_object>((uint16_t)block_num);
-
-   if( block_header::num_from_id(tapos_block_summary.block_id) == block_num )
-      return tapos_block_summary.block_id;
-
    const auto& blog_head = my->blog.head();
 
    bool find_in_blog = (blog_head && block_num <= blog_head->block_num());
@@ -3335,7 +3387,7 @@ const apply_handler* controller::find_apply_handler( account_name receiver, acco
    return nullptr;
 }
 wasm_interface& controller::get_wasm_interface() {
-   return my->wasmif;
+   return my->get_wasm_interface();
 }
 
 const account_object& controller::get_account( account_name name )const
@@ -3486,7 +3538,9 @@ void controller::add_to_ram_correction( account_name account, uint64_t ram_bytes
       } );
    }
 
-   if (auto dm_logger = get_deep_mind_logger()) {
+   // on_add_ram_correction is only called for deferred transaction
+   // (in apply_context::schedule_deferred_transaction)
+   if (auto dm_logger = get_deep_mind_logger(false)) {
       dm_logger->on_add_ram_correction(*ptr, ram_bytes);
    }
 }
@@ -3495,8 +3549,8 @@ bool controller::all_subjective_mitigations_disabled()const {
    return my->conf.disable_all_subjective_mitigations;
 }
 
-deep_mind_handler* controller::get_deep_mind_logger()const {
-   return my->get_deep_mind_logger();
+deep_mind_handler* controller::get_deep_mind_logger(bool is_trx_transient)const {
+   return my->get_deep_mind_logger(is_trx_transient);
 }
 
 void controller::enable_deep_mind(deep_mind_handler* logger) {
@@ -3513,6 +3567,12 @@ vm::wasm_allocator& controller::get_wasm_allocator() {
 }
 #endif
 
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+bool controller::is_eos_vm_oc_enabled() const {
+   return my->is_eos_vm_oc_enabled();
+}
+#endif
+
 std::optional<uint64_t> controller::convert_exception_to_error_code( const fc::exception& e ) {
    const chain_exception* e_ptr = dynamic_cast<const chain_exception*>( &e );
 
@@ -3572,7 +3632,11 @@ std::optional<chain_id_type> controller::extract_chain_id_from_db( const path& s
       if (gpo==nullptr) return {};
 
       return gpo->chain_id;
-   } catch (std::system_error &) {} //  do not propagate db_error_code::not_found" for absent db, so it will be created 
+   } catch( const std::system_error& e ) {
+      // do not propagate db_error_code::not_found for absent db, so it will be created
+      if( e.code().value() != chainbase::db_error_code::not_found )
+         throw;
+   }
 
    return {};
 }
@@ -3606,10 +3670,22 @@ void controller::replace_account_keys( name account, name permission, const publ
       p.auth = authority(key);
    });
    int64_t new_size = (int64_t)(chain::config::billable_size_v<permission_object> + perm->auth.get_billable_size());
-   rlm.add_pending_ram_usage(account, new_size - old_size);
+   rlm.add_pending_ram_usage(account, new_size - old_size, false); // false for doing dm logging
    rlm.verify_account_ram_usage(account);
 }
 
+void controller::set_db_read_only_mode() {
+   mutable_db().set_read_only_mode();
+}
+
+void controller::unset_db_read_only_mode() {
+   mutable_db().unset_read_only_mode();
+}
+
+void controller::init_thread_local_data() {
+   my->init_thread_local_data();
+}
+
 /// Protocol feature activation handlers:
 
 template<>
@@ -3639,11 +3715,12 @@ void controller_impl::on_activation<builtin_protocol_feature_t::replace_deferred
                ("name", itr->name)("adjust", itr->ram_correction)("current", current_ram_usage) );
       }
 
-      if (auto dm_logger = get_deep_mind_logger()) {
+      // This method is only called for deferred transaction
+      if (auto dm_logger = get_deep_mind_logger(false)) {
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", itr->id._id)), "deferred_trx", "correction", "deferred_trx_ram_correction");
       }
 
-      resource_limits.add_pending_ram_usage( itr->name, ram_delta );
+      resource_limits.add_pending_ram_usage( itr->name, ram_delta, false ); // false for doing dm logging
       db.remove( *itr );
    }
 }

libraries/chain/eosio_contract.cpp

@@ -65,6 +65,7 @@ void validate_authority_precondition( const apply_context& context, const author
  *  This method is called assuming precondition_system_newaccount succeeds a
  */
 void apply_eosio_newaccount(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "newaccount not allowed in read-only transaction" );
    auto create = context.get_action().data_as<newaccount>();
    try {
    context.require_authorization(create.creator);
@@ -107,18 +108,18 @@ void apply_eosio_newaccount(apply_context& context) {
    }
 
    const auto& owner_permission  = authorization.create_permission( create.name, config::owner_name, 0,
-                                                                    std::move(create.owner) );
+                                                                    std::move(create.owner), context.trx_context.is_transient() );
    const auto& active_permission = authorization.create_permission( create.name, config::active_name, owner_permission.id,
-                                                                    std::move(create.active) );
+                                                                    std::move(create.active), context.trx_context.is_transient() );
 
-   context.control.get_mutable_resource_limits_manager().initialize_account(create.name);
+   context.control.get_mutable_resource_limits_manager().initialize_account(create.name, context.trx_context.is_transient());
 
    int64_t ram_delta = config::overhead_per_account_ram_bytes;
    ram_delta += 2*config::billable_size_v<permission_object>;
    ram_delta += owner_permission.auth.get_billable_size();
    ram_delta += active_permission.auth.get_billable_size();
 
-   if (auto dm_logger = context.control.get_deep_mind_logger()) {
+   if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
       dm_logger->on_ram_trace(RAM_EVENT_ID("${name}", ("name", create.name)), "account", "add", "newaccount");
    }
 
@@ -127,6 +128,7 @@ void apply_eosio_newaccount(apply_context& context) {
 } FC_CAPTURE_AND_RETHROW( (create) ) }
 
 void apply_eosio_setcode(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "setcode not allowed in read-only transaction" );
    auto& db = context.db;
    auto  act = context.get_action().data_as<setcode>();
    context.require_authorization(act.account);
@@ -194,7 +196,7 @@ void apply_eosio_setcode(apply_context& context) {
    });
 
    if (new_size != old_size) {
-      if (auto dm_logger = context.control.get_deep_mind_logger()) {
+      if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
          const char* operation = "update";
          if (old_size <= 0) {
             operation = "add";
@@ -210,6 +212,7 @@ void apply_eosio_setcode(apply_context& context) {
 }
 
 void apply_eosio_setabi(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "setabi ot allowed in read-only transaction" );
    auto& db  = context.db;
    auto  act = context.get_action().data_as<setabi>();
 
@@ -232,7 +235,7 @@ void apply_eosio_setabi(apply_context& context) {
    });
 
    if (new_size != old_size) {
-      if (auto dm_logger = context.control.get_deep_mind_logger()) {
+      if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
          const char* operation = "update";
          if (old_size <= 0) {
             operation = "add";
@@ -248,6 +251,7 @@ void apply_eosio_setabi(apply_context& context) {
 }
 
 void apply_eosio_updateauth(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "updateauth not allowed in read-only transaction" );
 
    auto update = context.get_action().data_as<updateauth>();
    context.require_authorization(update.account); // only here to mark the single authority on this action as used
@@ -297,21 +301,21 @@ void apply_eosio_updateauth(apply_context& context) {
 
       int64_t old_size = (int64_t)(config::billable_size_v<permission_object> + permission->auth.get_billable_size());
 
-      authorization.modify_permission( *permission, update.auth );
+      authorization.modify_permission( *permission, update.auth, context.trx_context.is_transient() );
 
       int64_t new_size = (int64_t)(config::billable_size_v<permission_object> + permission->auth.get_billable_size());
 
-      if (auto dm_logger = context.control.get_deep_mind_logger()) {
+      if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", permission->id)), "auth", "update", "updateauth_update");
       }
 
       context.add_ram_usage( permission->owner, new_size - old_size );
    } else {
-      const auto& p = authorization.create_permission( update.account, update.permission, parent_id, update.auth );
+      const auto& p = authorization.create_permission( update.account, update.permission, parent_id, update.auth, context.trx_context.is_transient() );
 
       int64_t new_size = (int64_t)(config::billable_size_v<permission_object> + p.auth.get_billable_size());
 
-      if (auto dm_logger = context.control.get_deep_mind_logger()) {
+      if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", p.id)), "auth", "add", "updateauth_create");
       }
 
@@ -322,6 +326,8 @@ void apply_eosio_updateauth(apply_context& context) {
 void apply_eosio_deleteauth(apply_context& context) {
 //   context.require_write_lock( config::eosio_auth_scope );
 
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "deleteauth not allowed in read-only transaction" );
+
    auto remove = context.get_action().data_as<deleteauth>();
    context.require_authorization(remove.account); // only here to mark the single authority on this action as used
 
@@ -344,11 +350,11 @@ void apply_eosio_deleteauth(apply_context& context) {
    const auto& permission = authorization.get_permission({remove.account, remove.permission});
    int64_t old_size = config::billable_size_v<permission_object> + permission.auth.get_billable_size();
 
-   if (auto dm_logger = context.control.get_deep_mind_logger()) {
+   if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
       dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", permission.id)), "auth", "remove", "deleteauth");
    }
 
-   authorization.remove_permission( permission );
+   authorization.remove_permission( permission, context.trx_context.is_transient() );
 
    context.add_ram_usage( remove.account, -old_size );
 
@@ -357,6 +363,8 @@ void apply_eosio_deleteauth(apply_context& context) {
 void apply_eosio_linkauth(apply_context& context) {
 //   context.require_write_lock( config::eosio_auth_scope );
 
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "linkauth not allowed in read-only transaction" );
+
    auto requirement = context.get_action().data_as<linkauth>();
    try {
       EOS_ASSERT(!requirement.requirement.empty(), action_validate_exception, "Required permission cannot be empty");
@@ -401,7 +409,7 @@ void apply_eosio_linkauth(apply_context& context) {
             link.required_permission = requirement.requirement;
          });
 
-         if (auto dm_logger = context.control.get_deep_mind_logger()) {
+         if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
             dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", l.id)), "auth_link", "add", "linkauth");
          }
 
@@ -417,6 +425,8 @@ void apply_eosio_linkauth(apply_context& context) {
 void apply_eosio_unlinkauth(apply_context& context) {
 //   context.require_write_lock( config::eosio_auth_scope );
 
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "unlinkauth not allowed in read-only transaction" );
+
    auto& db = context.db;
    auto unlink = context.get_action().data_as<unlinkauth>();
 
@@ -426,7 +436,7 @@ void apply_eosio_unlinkauth(apply_context& context) {
    auto link = db.find<permission_link_object, by_action_name>(link_key);
    EOS_ASSERT(link != nullptr, action_validate_exception, "Attempting to unlink authority, but no link found");
 
-   if (auto dm_logger = context.control.get_deep_mind_logger()) {
+   if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
       dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", link->id)), "auth_link", "remove", "unlinkauth");
    }
 
@@ -439,6 +449,7 @@ void apply_eosio_unlinkauth(apply_context& context) {
 }
 
 void apply_eosio_canceldelay(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "canceldelay not allowed in read-only transaction" );
    auto cancel = context.get_action().data_as<canceldelay>();
    context.require_authorization(cancel.canceling_auth.actor); // only here to mark the single authority on this action as used
 

libraries/chain/include/eosio/chain/abi_serializer.hpp

@@ -35,10 +35,10 @@ struct abi_serializer {
    using yield_function_t = fc::optional_delegate<void(size_t)>;
 
    abi_serializer(){ configure_built_in_types(); }
-   abi_serializer( const abi_def& abi, const yield_function_t& yield );
+   abi_serializer( abi_def abi, const yield_function_t& yield );
    [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
    abi_serializer( const abi_def& abi, const fc::microseconds& max_serialization_time );
-   void set_abi( const abi_def& abi, const yield_function_t& yield );
+   void set_abi( abi_def abi, const yield_function_t& yield );
    [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
    void set_abi(const abi_def& abi, const fc::microseconds& max_serialization_time);
 

libraries/chain/include/eosio/chain/apply_context.hpp

@@ -1,6 +1,7 @@
 #pragma once
 #include <eosio/chain/controller.hpp>
 #include <eosio/chain/transaction.hpp>
+#include <eosio/chain/transaction_context.hpp>
 #include <eosio/chain/contract_table_objects.hpp>
 #include <eosio/chain/deep_mind.hpp>
 #include <fc/utility.hpp>
@@ -13,7 +14,6 @@ namespace chainbase { class database; }
 namespace eosio { namespace chain {
 
 class controller;
-class transaction_context;
 
 class apply_context {
    private:
@@ -177,6 +177,7 @@ class apply_context {
             int store( uint64_t scope, uint64_t table, const account_name& payer,
                        uint64_t id, secondary_key_proxy_const_type value )
             {
+               EOS_ASSERT( !context.trx_context.is_read_only(), table_operation_not_permitted, "cannot store a db record when executing a readonly transaction" );
                EOS_ASSERT( payer != account_name(), invalid_table_payer, "must specify a valid account to pay for new record" );
 
 //               context.require_write_lock( scope );
@@ -193,7 +194,7 @@ class apply_context {
                context.db.modify( tab, [&]( auto& t ) {
                  ++t.count;
 
-                  if (auto dm_logger = context.control.get_deep_mind_logger()) {
+                  if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
                      std::string event_id = RAM_EVENT_ID("${code}:${scope}:${table}:${index_name}",
                         ("code", t.code)
                         ("scope", t.scope)
@@ -211,12 +212,13 @@ class apply_context {
             }
 
             void remove( int iterator ) {
+               EOS_ASSERT( !context.trx_context.is_read_only(), table_operation_not_permitted, "cannot remove a db record when executing a readonly transaction" );
                const auto& obj = itr_cache.get( iterator );
 
                const auto& table_obj = itr_cache.get_table( obj.t_id );
                EOS_ASSERT( table_obj.code == context.receiver, table_access_violation, "db access violation" );
 
-               if (auto dm_logger = context.control.get_deep_mind_logger()) {
+               if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
                   std::string event_id = RAM_EVENT_ID("${code}:${scope}:${table}:${index_name}",
                      ("code", table_obj.code)
                      ("scope", table_obj.scope)
@@ -243,6 +245,7 @@ class apply_context {
             }
 
             void update( int iterator, account_name payer, secondary_key_proxy_const_type secondary ) {
+               EOS_ASSERT( !context.trx_context.is_read_only(), table_operation_not_permitted, "cannot update a db record when executing a readonly transaction" );
                const auto& obj = itr_cache.get( iterator );
 
                const auto& table_obj = itr_cache.get_table( obj.t_id );
@@ -255,7 +258,7 @@ class apply_context {
                int64_t billing_size =  config::billable_size_v<ObjectType>;
 
                std::string event_id;
-               if (context.control.get_deep_mind_logger() != nullptr) {
+               if (context.control.get_deep_mind_logger(context.trx_context.is_transient()) != nullptr) {
                   event_id = RAM_EVENT_ID("${code}:${scope}:${table}:${index_name}",
                      ("code", table_obj.code)
                      ("scope", table_obj.scope)
@@ -265,12 +268,12 @@ class apply_context {
                }
 
                if( obj.payer != payer ) {
-                  if (auto dm_logger = context.control.get_deep_mind_logger())
+                  if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient()))
                   {
                      dm_logger->on_ram_trace(std::string(event_id), "secondary_index", "remove", "secondary_index_remove");
                   }
                   context.update_db_usage( obj.payer, -(billing_size) );
-                  if (auto dm_logger = context.control.get_deep_mind_logger())
+                  if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient()))
                   {
                      dm_logger->on_ram_trace(std::move(event_id), "secondary_index", "add", "secondary_index_update_add_new_payer");
                   }

libraries/chain/include/eosio/chain/authorization_manager.hpp

@@ -31,6 +31,7 @@ namespace eosio { namespace chain {
                                                      permission_name name,
                                                      permission_id_type parent,
                                                      const authority& auth,
+                                                     bool is_trx_transient,
                                                      time_point initial_creation_time = time_point()
                                                    );
 
@@ -38,12 +39,13 @@ namespace eosio { namespace chain {
                                                      permission_name name,
                                                      permission_id_type parent,
                                                      authority&& auth,
+                                                     bool is_trx_transient,
                                                      time_point initial_creation_time = time_point()
                                                    );
 
-         void modify_permission( const permission_object& permission, const authority& auth );
+         void modify_permission( const permission_object& permission, const authority& auth, bool is_trx_transient );
 
-         void remove_permission( const permission_object& permission );
+         void remove_permission( const permission_object& permission, bool is_trx_transient );
 
          void update_permission_usage( const permission_object& permission );
 

libraries/chain/include/eosio/chain/block_log.hpp

@@ -2,10 +2,11 @@
 #include <fc/filesystem.hpp>
 #include <eosio/chain/block.hpp>
 #include <eosio/chain/genesis_state.hpp>
+#include <eosio/chain/block_log_config.hpp>
 
 namespace eosio { namespace chain {
 
-   namespace detail { class block_log_impl; }
+   namespace detail { struct block_log_impl; }
 
    /* The block log is an external append only log of the blocks with a header. Blocks should only
     * be written to the log after they irreverisble as the log is append only. The log is a doubly
@@ -34,18 +35,15 @@ namespace eosio { namespace chain {
     * An optional "pruned" mode can be activated which stores a 4 byte trailer on the log file indicating
     * how many blocks at the end of the log are valid. Any earlier blocks in the log are assumed destroyed
     * and unreadable due to reclamation for purposes of saving space.
+    *
+    * Object thread-safe. Not safe to have multiple block_log objects to same data_dir.
     */
 
-   struct block_log_prune_config {
-      uint32_t                prune_blocks;                  //number of blocks to prune to when doing a prune
-      size_t                  prune_threshold = 4*1024*1024; //(approximately) how many bytes need to be added before a prune is performed
-      std::optional<size_t>   vacuum_on_close;               //when set, a vacuum is performed on dtor if log contains less than this many live bytes
-   };
 
    class block_log {
       public:
-         block_log(const fc::path& data_dir, std::optional<block_log_prune_config> prune_config);
-         block_log(block_log&& other);
+         explicit block_log(const fc::path& data_dir, const block_log_config& config = block_log_config{});
+         block_log(block_log&& other) noexcept;
          ~block_log();
 
          void append(const signed_block_ptr& b, const block_id_type& id);
@@ -54,12 +52,11 @@ namespace eosio { namespace chain {
          void flush();
          void reset( const genesis_state& gs, const signed_block_ptr& genesis_block );
          void reset( const chain_id_type& chain_id, uint32_t first_block_num );
-         void remove(); // remove blocks.log and blocks.index
 
-         signed_block_ptr read_block(uint64_t file_pos)const;
-         void             read_block_header(block_header& bh, uint64_t file_pos)const;
          signed_block_ptr read_block_by_num(uint32_t block_num)const;
+         std::optional<signed_block_header> read_block_header_by_num(uint32_t block_num)const;
          block_id_type    read_block_id_by_num(uint32_t block_num)const;
+
          signed_block_ptr read_block_by_id(const block_id_type& id)const {
             return read_block_by_num(block_header::num_from_id(id));
          }
@@ -67,10 +64,11 @@ namespace eosio { namespace chain {
          /**
           * Return offset of block in file, or block_log::npos if it does not exist.
           */
-         uint64_t get_block_pos(uint32_t block_num) const;
-         signed_block_ptr        read_head()const;
-         const signed_block_ptr& head()const;
-         const block_id_type&    head_id()const;
+
+         signed_block_ptr read_head()const; //use blocklog
+         signed_block_ptr head()const;
+         block_id_type head_id()const;
+
          uint32_t                first_block_num() const;
 
          static const uint64_t npos = std::numeric_limits<uint64_t>::max();
@@ -78,6 +76,10 @@ namespace eosio { namespace chain {
          static const uint32_t min_supported_version;
          static const uint32_t max_supported_version;
 
+         /**
+          * All static methods expected to be called on quiescent block log
+          */
+
          static fc::path repair_log( const fc::path& data_dir, uint32_t truncate_at_block = 0, const char* reversible_block_dir_name="" );
 
          static std::optional<genesis_state> extract_genesis_state( const fc::path& data_dir );
@@ -94,37 +96,24 @@ namespace eosio { namespace chain {
 
          static bool is_pruned_log(const fc::path& data_dir);
 
-         static bool extract_block_range(const fc::path& block_dir, const fc::path&output_dir, block_num_type& start, block_num_type& end, bool rename_input=false);
+         static void extract_block_range(const fc::path& block_dir, const fc::path&output_dir, block_num_type start, block_num_type end);
 
+         static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block);
+         static int  trim_blocklog_end(const fc::path& block_dir, uint32_t n);
+
+         // used for unit test to generate older version blocklog
+         static void set_initial_version(uint32_t);
+         uint32_t    version() const;
+         uint64_t get_block_pos(uint32_t block_num) const;
+
+         /**
+          * @param n Only test 1 block out of every n blocks. If n is 0, the interval is adjusted so that at most 8 blocks are tested.
+          */
+         static void smoke_test(const fc::path& block_dir, uint32_t n);
+
+         static void split_blocklog(const fc::path& block_dir, const fc::path& dest_dir, uint32_t stride);
+         static void merge_blocklogs(const fc::path& block_dir, const fc::path& dest_dir);
    private:
-         void open(const fc::path& data_dir);
-         void construct_index();
-
          std::unique_ptr<detail::block_log_impl> my;
    };
-
-//to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed
-//   block_timestamp_type timestamp;                  //bytes 0:3
-//   account_name         producer;                   //bytes 4:11
-//   uint16_t             confirmed;                  //bytes 12:13
-//   block_id_type        previous;                   //bytes 14:45, low 4 bytes is big endian block number of previous block
-
-   struct trim_data {            //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test()
-      trim_data(fc::path block_dir);
-      ~trim_data();
-      uint64_t block_index(uint32_t n) const;
-      uint64_t block_pos(uint32_t n);
-      fc::path block_file_name, index_file_name;        //full pathname for blocks.log and blocks.index
-      uint32_t version = 0;                              //blocklog version
-      uint32_t first_block = 0;                          //first block in blocks.log
-      uint32_t last_block = 0;                          //last block in blocks.log
-      FILE* blk_in = nullptr;                            //C style files for reading blocks.log and blocks.index
-      FILE* ind_in = nullptr;                            //C style files for reading blocks.log and blocks.index
-      //we use low level file IO because it is distinctly faster than C++ filebuf or iostream
-      uint64_t first_block_pos = 0;                      //file position in blocks.log for the first block in the log
-      genesis_state gs;
-      chain_id_type chain_id;
-
-      static constexpr int blknum_offset{14};            //offset from start of block to 4 byte block number, valid for the only allowed versions
-   };
 } }

libraries/chain/include/eosio/chain/block_log_config.hpp

@@ -0,0 +1,31 @@
+#pragma once
+#include <boost/filesystem/path.hpp>
+#include <variant>
+
+namespace eosio { namespace chain {
+
+   namespace bfs = boost::filesystem;
+
+   struct basic_blocklog_config {};
+
+   struct empty_blocklog_config {};
+
+   struct partitioned_blocklog_config {
+      bfs::path retained_dir;
+      bfs::path archive_dir;
+      uint32_t  stride                  = UINT32_MAX;
+      uint32_t  max_retained_files      = UINT32_MAX;
+   };
+
+   struct prune_blocklog_config {
+      uint32_t prune_blocks; // number of blocks to prune to when doing a prune
+      size_t   prune_threshold =
+            4 * 1024 * 1024; //(approximately) how many bytes need to be added before a prune is performed
+      std::optional<size_t>
+            vacuum_on_close; // when set, a vacuum is performed on dtor if log contains less than this many live bytes
+   };
+
+   using block_log_config =
+         std::variant<basic_blocklog_config, empty_blocklog_config, partitioned_blocklog_config, prune_blocklog_config>;
+
+}} // namespace eosio::chain

libraries/chain/include/eosio/chain/chain_id_type.hpp

@@ -37,6 +37,10 @@ namespace chain {
 
       void reflector_init()const;
 
+      static chain_id_type empty_chain_id() {
+         return {};
+      }
+
       private:
          chain_id_type() = default;
 

libraries/chain/include/eosio/chain/controller.hpp

@@ -68,7 +68,7 @@ namespace eosio { namespace chain {
             flat_set< pair<account_name, action_name> > action_blacklist;
             flat_set<public_key_type> key_blacklist;
             path                     blocks_dir             =  chain::config::default_blocks_dir_name;
-            std::optional<block_log_prune_config>  prune_config;
+            block_log_config         blog;
             path                     state_dir              =  chain::config::default_state_dir_name;
             uint64_t                 state_size             =  chain::config::default_state_size;
             uint64_t                 state_guard_size       =  chain::config::default_state_guard_size;
@@ -119,7 +119,7 @@ namespace eosio { namespace chain {
          void startup( std::function<void()> shutdown, std::function<bool()> check_shutdown, const genesis_state& genesis);
          void startup( std::function<void()> shutdown, std::function<bool()> check_shutdown);
 
-         void preactivate_feature( const digest_type& feature_digest );
+         void preactivate_feature( const digest_type& feature_digest, bool is_trx_transient );
 
          vector<digest_type> get_preactivated_protocol_features()const;
 
@@ -221,13 +221,6 @@ namespace eosio { namespace chain {
 
          uint32_t             fork_db_head_block_num()const;
          block_id_type        fork_db_head_block_id()const;
-         time_point           fork_db_head_block_time()const;
-         account_name         fork_db_head_block_producer()const;
-
-         uint32_t             fork_db_pending_head_block_num()const;
-         block_id_type        fork_db_pending_head_block_id()const;
-         time_point           fork_db_pending_head_block_time()const;
-         account_name         fork_db_pending_head_block_producer()const;
 
          time_point                     pending_block_time()const;
          account_name                   pending_block_producer()const;
@@ -243,13 +236,19 @@ namespace eosio { namespace chain {
          block_id_type last_irreversible_block_id() const;
          time_point last_irreversible_block_time() const;
 
+         // thread-safe
          signed_block_ptr fetch_block_by_number( uint32_t block_num )const;
-         signed_block_ptr fetch_block_by_id( block_id_type id )const;
-
+         // thread-safe
+         signed_block_ptr fetch_block_by_id( const block_id_type& id )const;
+         // thread-safe
+         std::optional<signed_block_header> fetch_block_header_by_number( uint32_t block_num )const;
+         // thread-safe
+         std::optional<signed_block_header> fetch_block_header_by_id( const block_id_type& id )const;
+         // return block_state from forkdb, thread-safe
          block_state_ptr fetch_block_state_by_number( uint32_t block_num )const;
          // return block_state from forkdb, thread-safe
          block_state_ptr fetch_block_state_by_id( block_id_type id )const;
-
+         // thread-safe
          block_id_type get_block_id_for_num( uint32_t block_num )const;
 
          sha256 calculate_integrity_hash();
@@ -311,12 +310,13 @@ namespace eosio { namespace chain {
          void add_to_ram_correction( account_name account, uint64_t ram_bytes );
          bool all_subjective_mitigations_disabled()const;
 
-         deep_mind_handler* get_deep_mind_logger() const;
+         deep_mind_handler* get_deep_mind_logger(bool is_trx_transient) const;
          void enable_deep_mind( deep_mind_handler* logger );
          uint32_t earliest_available_block_num() const;
 
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
          vm::wasm_allocator&  get_wasm_allocator();
+         bool is_eos_vm_oc_enabled() const;
 #endif
 
          static std::optional<uint64_t> convert_exception_to_error_code( const fc::exception& e );
@@ -348,9 +348,8 @@ namespace eosio { namespace chain {
             if( n.good() ) {
                try {
                   const auto& a = get_account( n );
-                  abi_def abi;
-                  if( abi_serializer::to_abi( a.abi, abi ))
-                     return abi_serializer( abi, yield );
+                  if( abi_def abi; abi_serializer::to_abi( a.abi, abi ))
+                     return abi_serializer( std::move(abi), yield );
                } FC_CAPTURE_AND_LOG((n))
             }
             return std::optional<abi_serializer>();
@@ -371,6 +370,10 @@ namespace eosio { namespace chain {
       void replace_producer_keys( const public_key_type& key );
       void replace_account_keys( name account, name permission, const public_key_type& key );
 
+      void set_db_read_only_mode();
+      void unset_db_read_only_mode();
+      void init_thread_local_data();
+
       private:
          friend class apply_context;
          friend class transaction_context;

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -573,6 +573,12 @@ namespace eosio { namespace chain {
                                     3170011, "The signer returned no valid block signatures" )
       FC_DECLARE_DERIVED_EXCEPTION( unsupported_multiple_block_signatures,  producer_exception,
                                     3170012, "The signer returned multiple signatures but that is not supported" )
+      FC_DECLARE_DERIVED_EXCEPTION( duplicate_snapshot_request,  producer_exception,
+                                    3170013, "Snapshot has been already scheduled with specified parameters" )
+      FC_DECLARE_DERIVED_EXCEPTION( snapshot_request_not_found,  producer_exception,
+                                    3170014, "Snapshot request not found" )
+      FC_DECLARE_DERIVED_EXCEPTION( invalid_snapshot_request,  producer_exception,
+                                    3170015, "Invalid snapshot request" )
 
    FC_DECLARE_DERIVED_EXCEPTION( reversible_blocks_exception,           chain_exception,
                                  3180000, "Reversible Blocks exception" )

libraries/chain/include/eosio/chain/log_catalog.hpp

@@ -0,0 +1,280 @@
+#pragma once
+#include <boost/container/flat_map.hpp>
+#include <boost/filesystem/path.hpp>
+#include <boost/iostreams/device/mapped_file.hpp>
+#include <fc/io/cfile.hpp>
+#include <fc/io/datastream.hpp>
+#include <regex>
+
+namespace eosio {
+namespace chain {
+
+namespace bfs = boost::filesystem;
+
+template <typename Lambda>
+void for_each_file_in_dir_matches(const bfs::path& dir, std::string pattern, Lambda&& lambda) {
+   const std::regex        my_filter(pattern);
+   std::smatch             what;
+   bfs::directory_iterator end_itr; // Default ctor yields past-the-end
+   for (bfs::directory_iterator p(dir); p != end_itr; ++p) {
+      // Skip if not a file
+      if (!bfs::is_regular_file(p->status()))
+         continue;
+      // skip if it does not match the pattern
+      if (!std::regex_match(p->path().filename().string(), what, my_filter))
+         continue;
+      lambda(p->path());
+   }
+}
+
+struct null_verifier {
+   template <typename LogData>
+   void verify(const LogData&, const bfs::path&) {}
+};
+
+template <typename LogData, typename LogIndex, typename LogVerifier = null_verifier>
+struct log_catalog {
+   using block_num_t = uint32_t;
+
+   struct mapped_type {
+      block_num_t last_block_num;
+      bfs::path   filename_base;
+   };
+   using collection_t              = boost::container::flat_map<block_num_t, mapped_type>;
+   using size_type                 = typename collection_t::size_type;
+   static constexpr size_type npos = std::numeric_limits<size_type>::max();
+
+   bfs::path    retained_dir;
+   bfs::path    archive_dir;
+   size_type    max_retained_files = std::numeric_limits<size_type>::max();
+   collection_t collection;
+   size_type    active_index = npos;
+   LogData      log_data;
+   LogIndex     log_index;
+   LogVerifier  verifier;
+
+   bool empty() const { return collection.empty(); }
+
+   block_num_t first_block_num() const {
+      if (empty())
+         return std::numeric_limits<block_num_t>::max();
+      return collection.begin()->first;
+   }
+
+   block_num_t last_block_num() const {
+      if (empty())
+         return std::numeric_limits<block_num_t>::min();
+      return collection.rbegin()->second.last_block_num;
+   }
+
+   static bfs::path make_absolute_dir(const bfs::path& base_dir, bfs::path new_dir) {
+      if (new_dir.is_relative())
+         new_dir = base_dir / new_dir;
+
+      if (!bfs::is_directory(new_dir))
+         bfs::create_directories(new_dir);
+
+      return new_dir;
+   }
+
+   void open(const bfs::path& log_dir, const bfs::path& retained_path, const bfs::path& archive_path, const char* name,
+             const char* suffix_pattern = R"(-\d+-\d+\.log)") {
+
+      retained_dir = make_absolute_dir(log_dir, retained_path.empty() ? log_dir : retained_path);
+      if (!archive_path.empty()) {
+         archive_dir = make_absolute_dir(log_dir, archive_path);
+      }
+
+      for_each_file_in_dir_matches(retained_dir, std::string(name) + suffix_pattern, [this](bfs::path path) {
+         auto log_path               = path;
+         auto index_path             = path.replace_extension("index");
+         auto path_without_extension = log_path.parent_path() / log_path.stem().string();
+
+         LogData log(log_path);
+
+         verifier.verify(log, log_path);
+
+         // check if index file matches the log file
+         if (!index_matches_data(index_path, log))
+            log.construct_index(index_path);
+
+         auto existing_itr = collection.find(log.first_block_num());
+         if (existing_itr != collection.end()) {
+            if (log.last_block_num() <= existing_itr->second.last_block_num) {
+               wlog("${log_path} contains the overlapping range with ${existing_path}.log, dropping ${log_path} "
+                    "from catalog",
+                    ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string()));
+               return;
+            } else {
+               wlog(
+                   "${log_path} contains the overlapping range with ${existing_path}.log, droping ${existing_path}.log "
+                   "from catelog",
+                   ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string()));
+            }
+         }
+
+         collection.insert_or_assign(log.first_block_num(), mapped_type{log.last_block_num(), path_without_extension});
+      });
+   }
+
+   bool index_matches_data(const bfs::path& index_path, LogData& log) const {
+      if (!bfs::exists(index_path))
+         return false;
+
+      auto num_blocks_in_index = bfs::file_size(index_path) / sizeof(uint64_t);
+      if (num_blocks_in_index != log.num_blocks())
+         return false;
+
+      // make sure the last 8 bytes of index and log matches
+      fc::cfile index_file;
+      index_file.set_file_path(index_path);
+      index_file.open("r");
+      index_file.seek_end(-sizeof(uint64_t));
+      uint64_t pos;
+      index_file.read(reinterpret_cast<char*>(&pos), sizeof(pos));
+      return pos == log.last_block_position();
+   }
+
+   std::optional<uint64_t> get_block_position(uint32_t block_num) {
+      try {
+         if (active_index != npos) {
+            auto active_item = collection.nth(active_index);
+            if (active_item->first <= block_num && block_num <= active_item->second.last_block_num) {
+               return log_index.nth_block_position(block_num - log_data.first_block_num());
+            }
+         }
+         if (block_num < first_block_num())
+            return {};
+
+         auto it = --collection.upper_bound(block_num);
+
+         if (block_num <= it->second.last_block_num) {
+            auto name = it->second.filename_base;
+            log_data.open(name.replace_extension("log"));
+            log_index.open(name.replace_extension("index"));
+            active_index = collection.index_of(it);
+            return log_index.nth_block_position(block_num - log_data.first_block_num());
+         }
+         return {};
+      } catch (...) {
+         active_index = npos;
+         return {};
+      }
+   }
+
+   fc::datastream<fc::cfile>* ro_stream_for_block(uint32_t block_num) {
+      auto pos = get_block_position(block_num);
+      if (pos) {
+         return &log_data.ro_stream_at(*pos);
+      }
+      return nullptr;
+   }
+
+   template <typename ...Rest>
+   auto ro_stream_for_block(uint32_t block_num, Rest&& ...rest) -> std::optional<decltype( std::declval<LogData>().ro_stream_at(0, std::forward<Rest&&>(rest)...))> {
+      auto pos = get_block_position(block_num);
+      if (pos) {
+         return log_data.ro_stream_at(*pos, std::forward<Rest&&>(rest)...);
+      }
+      return {};
+   }
+
+   std::optional<block_id_type> id_for_block(uint32_t block_num) {
+      auto pos = get_block_position(block_num);
+      if (pos) {
+         return log_data.block_id_at(*pos);
+      }
+      return {};
+   }
+
+   static void rename_if_not_exists(bfs::path old_name, bfs::path new_name) {
+      if (!bfs::exists(new_name)) {
+         bfs::rename(old_name, new_name);
+      } else {
+         bfs::remove(old_name);
+         wlog("${new_name} already exists, just removing ${old_name}",
+              ("old_name", old_name.string())("new_name", new_name.string()));
+      }
+   }
+
+   static void rename_bundle(bfs::path orig_path, bfs::path new_path) {
+      rename_if_not_exists(orig_path.replace_extension(".log"), new_path.replace_extension(".log"));
+      rename_if_not_exists(orig_path.replace_extension(".index"), new_path.replace_extension(".index"));
+   }
+
+   /// Add a new entry into the catalog.
+   ///
+   /// Notice that \c start_block_num must be monotonically increasing between the invocations of this function
+   /// so that the new entry would be inserted at the end of the flat_map; otherwise, \c active_index would be
+   /// invalidated and the mapping between the log data their block range would be wrong. This function is only used
+   /// during the splitting of block log. Using this function for other purpose should make sure if the monotonically
+   /// increasing block num guarantee can be met.
+   void add(uint32_t start_block_num, uint32_t end_block_num, const bfs::path& dir, const char* name) {
+
+      const int bufsize = 64;
+      char      buf[bufsize];
+      snprintf(buf, bufsize, "%s-%u-%u", name, start_block_num, end_block_num);
+      bfs::path new_path = retained_dir / buf;
+      rename_bundle(dir / name, new_path);
+      size_type items_to_erase = 0;
+      collection.emplace(start_block_num, mapped_type{end_block_num, new_path});
+      if (collection.size() >= max_retained_files) {
+         items_to_erase =
+            max_retained_files > 0 ? collection.size() - max_retained_files : collection.size();
+
+         for (auto it = collection.begin(); it < collection.begin() + items_to_erase; ++it) {
+            auto orig_name = it->second.filename_base;
+            if (archive_dir.empty()) {
+               // delete the old files when no backup dir is specified
+               bfs::remove(orig_name.replace_extension("log"));
+               bfs::remove(orig_name.replace_extension("index"));
+            } else {
+               // move the the archive dir
+               rename_bundle(orig_name, archive_dir / orig_name.filename());
+            }
+         }
+         collection.erase(collection.begin(), collection.begin() + items_to_erase);
+         active_index = active_index == npos || active_index < items_to_erase
+                        ? npos
+                        : active_index - items_to_erase;
+      }
+   }
+
+   /// Truncate the catalog so that the log/index bundle containing the block with \c block_num
+   /// would be rename to \c new_name; the log/index bundles with blocks strictly higher
+   /// than \c block_num would be deleted, and all the renamed/removed entries would be erased
+   /// from the catalog.
+   ///
+   /// \return if nonzero, it's the starting block number for the log/index bundle being renamed.
+   uint32_t truncate(uint32_t block_num, bfs::path new_name) {
+      if (collection.empty())
+         return 0;
+
+      auto remove_files = [](typename collection_t::const_reference v) {
+         auto name = v.second.filename_base;
+         bfs::remove(name.replace_extension("log"));
+         bfs::remove(name.replace_extension("index"));
+      };
+
+      active_index = npos;
+      auto it = collection.upper_bound(block_num);
+
+      if (it == collection.begin() || block_num > (it - 1)->second.last_block_num) {
+         std::for_each(it, collection.end(), remove_files);
+         collection.erase(it, collection.end());
+         return 0;
+      } else {
+         auto truncate_it = --it;
+         auto name        = truncate_it->second.filename_base;
+         bfs::rename(name.replace_extension("log"), new_name.replace_extension("log"));
+         bfs::rename(name.replace_extension("index"), new_name.replace_extension("index"));
+         std::for_each(truncate_it + 1, collection.end(), remove_files);
+         auto result = truncate_it->first;
+         collection.erase(truncate_it, collection.end());
+         return result;
+      }
+   }
+};
+
+} // namespace chain
+} // namespace eosio

libraries/chain/include/eosio/chain/log_data_base.hpp

@@ -0,0 +1,49 @@
+#pragma once
+#include <fc/io/cfile.hpp>
+
+namespace eosio {
+namespace chain {
+
+template <typename T>
+T read_data_at(fc::datastream<fc::cfile>& file, std::size_t offset) {
+   file.seek(offset);
+   T value;
+   fc::raw::unpack(file, value);
+   return value;
+}
+
+template <typename Derived>
+class log_data_base {
+ protected:
+   fc::datastream<fc::cfile> file;
+
+   Derived* self() { return static_cast<Derived*>(this); }
+   const Derived* self() const { return static_cast<const Derived*>(this); }
+
+ public:
+
+   log_data_base() = default;
+
+   void    close() { file.close(); }
+   bool    is_open() const { return file.is_open(); }
+
+   uint64_t    size() const { return self()->size(); }
+
+   uint32_t    last_block_num() { return self()->block_num_at(last_block_position()); }
+   uint64_t    last_block_position() {
+      uint32_t offset = sizeof(uint64_t);
+      if (self()->is_currently_pruned())
+         offset += sizeof(uint32_t);
+      return read_data_at<uint64_t>(file, size() - offset);
+   }
+
+   uint32_t num_blocks() {
+      if (self()->first_block_position() == size())
+         return 0;
+      else if (self()->is_currently_pruned())
+         return read_data_at<uint32_t>(file, size() - sizeof(uint32_t));
+      return last_block_num() - self()->first_block_num() + 1;
+   }
+};
+} // namespace chain
+} // namespace eosio

libraries/chain/include/eosio/chain/log_index.hpp

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <boost/filesystem/path.hpp>
+#include <fc/io/cfile.hpp>
+
+namespace eosio {
+namespace chain {
+/// copy up to n bytes from the current position of src to dest
+void copy_file_content(fc::cfile& src, fc::cfile& dest, uint64_t n = UINT64_MAX);
+
+template <typename Exception>
+class log_index {
+   fc::cfile file_;
+   std::size_t num_blocks_ = 0;
+ public:
+   log_index() = default;
+   log_index(const boost::filesystem::path& path) {
+      open(path);
+   }
+
+   void open(const boost::filesystem::path& path) {
+      if (file_.is_open())
+         file_.close();
+      file_.set_file_path(path);
+      file_.open("rb");
+      file_.seek_end(0);
+      num_blocks_ = file_.tellp()/ sizeof(uint64_t);
+      EOS_ASSERT(file_.tellp() % sizeof(uint64_t) == 0, Exception,
+                 "The size of ${file} is not a multiple of sizeof(uint64_t)", ("file", path.generic_string()));
+   }
+
+   bool is_open() const { return file_.is_open(); }
+
+   uint64_t back() { return nth_block_position(num_blocks()-1); }
+   unsigned num_blocks() const { return num_blocks_; }
+   uint64_t nth_block_position(uint32_t n) {
+      file_.seek(n*sizeof(uint64_t));
+      uint64_t r;
+      file_.read((char*)&r, sizeof(r));
+      return r;
+   }
+
+   void copy_to(fc::cfile& dest, uint64_t nbytes) {
+      file_.seek(0);
+      copy_file_content(file_, dest, nbytes);
+   }
+
+};
+
+} // namespace chain
+} // namespace eosio

libraries/chain/include/eosio/chain/producer_schedule.hpp

@@ -335,4 +335,3 @@ FC_REFLECT_DERIVED( eosio::chain::producer_schedule_change_extension, (eosio::ch
 FC_REFLECT( eosio::chain::shared_block_signing_authority_v0, (threshold)(keys))
 FC_REFLECT( eosio::chain::shared_producer_authority, (producer_name)(authority) )
 FC_REFLECT( eosio::chain::shared_producer_authority_schedule, (version)(producers) )
-

libraries/chain/include/eosio/chain/protocol_feature_manager.hpp

@@ -264,7 +264,7 @@ protected:
 class protocol_feature_manager {
 public:
 
-   protocol_feature_manager( protocol_feature_set&& pfs, std::function<deep_mind_handler*()> get_deep_mind_logger );
+   protocol_feature_manager( protocol_feature_set&& pfs, std::function<deep_mind_handler*(bool is_trx_transient)> get_deep_mind_logger );
 
    class const_iterator {
    public:
@@ -393,7 +393,7 @@ protected:
    bool                                   _initialized = false;
 
 private:
-   std::function<deep_mind_handler*()>           _get_deep_mind_logger;
+   std::function<deep_mind_handler*(bool is_trx_transient)> _get_deep_mind_logger;
 };
 
 std::optional<builtin_protocol_feature> read_builtin_protocol_feature( const fc::path& p  );

libraries/chain/include/eosio/chain/resource_limits.hpp

@@ -64,7 +64,7 @@ namespace eosio { namespace chain {
    class resource_limits_manager {
       public:
 
-         explicit resource_limits_manager(chainbase::database& db, std::function<deep_mind_handler*()> get_deep_mind_logger)
+         explicit resource_limits_manager(chainbase::database& db, std::function<deep_mind_handler*(bool is_trx_transient)> get_deep_mind_logger)
          :_db(db),_get_deep_mind_logger(get_deep_mind_logger)
          {
          }
@@ -74,17 +74,17 @@ namespace eosio { namespace chain {
          void add_to_snapshot( const snapshot_writer_ptr& snapshot ) const;
          void read_from_snapshot( const snapshot_reader_ptr& snapshot );
 
-         void initialize_account( const account_name& account );
+         void initialize_account( const account_name& account, bool is_trx_transient );
          void set_block_parameters( const elastic_limit_parameters& cpu_limit_parameters, const elastic_limit_parameters& net_limit_parameters );
 
          void update_account_usage( const flat_set<account_name>& accounts, uint32_t ordinal );
-         void add_transaction_usage( const flat_set<account_name>& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t ordinal );
+         void add_transaction_usage( const flat_set<account_name>& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t ordinal, bool is_trx_transient = false );
 
-         void add_pending_ram_usage( const account_name account, int64_t ram_delta );
+         void add_pending_ram_usage( const account_name account, int64_t ram_delta, bool is_trx_transient = false );
          void verify_account_ram_usage( const account_name accunt )const;
 
          /// set_account_limits returns true if new ram_bytes limit is more restrictive than the previously set one
-         bool set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight);
+         bool set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight, bool is_trx_transient);
          void get_account_limits( const account_name& account, int64_t& ram_bytes, int64_t& net_weight, int64_t& cpu_weight) const;
 
          bool is_unlimited_cpu( const account_name& account ) const;
@@ -114,7 +114,7 @@ namespace eosio { namespace chain {
 
       private:
          chainbase::database&         _db;
-         std::function<deep_mind_handler*()> _get_deep_mind_logger;
+         std::function<deep_mind_handler*(bool is_trx_transient)> _get_deep_mind_logger;
    };
 } } } /// eosio::chain
 

libraries/chain/include/eosio/chain/thread_utils.hpp

@@ -25,6 +25,7 @@ namespace eosio { namespace chain {
    class named_thread_pool {
    public:
       using on_except_t = std::function<void(const fc::exception& e)>;
+      using init_t = std::function<void()>;
 
       named_thread_pool() = default;
 
@@ -39,14 +40,15 @@ namespace eosio { namespace chain {
       /// @param num_threads is number of threads spawned
       /// @param on_except is the function to call if io_context throws an exception, is called from thread pool thread.
       ///                  if an empty function then logs and rethrows exception on thread which will terminate.
+      /// @param init is an optional function to call at startup to initialize any data.
       /// @throw assert_exception if already started and not stopped.
-      void start( size_t num_threads, on_except_t on_except ) {
+      void start( size_t num_threads, on_except_t on_except, init_t init = {} ) {
          FC_ASSERT( !_ioc_work, "Thread pool already started" );
          _ioc_work.emplace( boost::asio::make_work_guard( _ioc ) );
          _ioc.restart();
          _thread_pool.reserve( num_threads );
          for( size_t i = 0; i < num_threads; ++i ) {
-            _thread_pool.emplace_back( std::thread( &named_thread_pool::run_thread, this, i, on_except )  );
+            _thread_pool.emplace_back( std::thread( &named_thread_pool::run_thread, this, i, on_except, init )  );
          }
       }
 
@@ -61,7 +63,7 @@ namespace eosio { namespace chain {
       }
 
    private:
-      void run_thread( size_t i, const on_except_t& on_except ) {
+      void run_thread( size_t i, const on_except_t& on_except, const init_t& init ) {
          std::string tn = boost::core::demangle(typeid(this).name());
          auto offset = tn.rfind("::");
          if (offset != std::string::npos)
@@ -69,6 +71,8 @@ namespace eosio { namespace chain {
          tn = tn.substr(0, tn.find('>')) + "-" + std::to_string( i );
          try {
             fc::set_os_thread_name( tn );
+            if ( init )
+               init();
             _ioc.run();
          } catch( const fc::exception& e ) {
             if( on_except ) {

libraries/chain/include/eosio/chain/transaction_context.hpp

@@ -84,6 +84,8 @@ namespace eosio { namespace chain {
          void validate_referenced_accounts( const transaction& trx, bool enforce_actor_whitelist_blacklist )const;
 
          bool is_dry_run()const { return trx_type == transaction_metadata::trx_type::dry_run; };
+         bool is_read_only()const { return trx_type == transaction_metadata::trx_type::read_only; };
+         bool is_transient()const { return trx_type == transaction_metadata::trx_type::read_only || trx_type == transaction_metadata::trx_type::dry_run; };
 
       private:
 

libraries/chain/include/eosio/chain/transaction_metadata.hpp

@@ -24,7 +24,8 @@ class transaction_metadata {
          input,
          implicit,
          scheduled,
-         dry_run
+         dry_run,
+         read_only
       };
 
    private:
@@ -73,6 +74,8 @@ class transaction_metadata {
       bool implicit() const { return _trx_type == trx_type::implicit; };
       bool scheduled() const { return _trx_type == trx_type::scheduled; };
       bool is_dry_run() const { return _trx_type == trx_type::dry_run; };
+      bool is_read_only() const { return _trx_type == trx_type::read_only; };
+      bool is_transient() const { return _trx_type == trx_type::read_only || _trx_type == trx_type::dry_run; };
 
       /// Thread safe.
       /// @returns transaction_metadata_ptr or exception via future

libraries/chain/include/eosio/chain/wasm_interface.hpp

@@ -45,6 +45,9 @@ namespace eosio { namespace chain {
          wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile);
          ~wasm_interface();
 
+         // initialize exec per thread
+         void init_thread_local_data();
+
          //call before dtor to skip what can be minutes of dtor overhead with some runtimes; can cause leaks
          void indicate_shutting_down();
 
@@ -72,6 +75,7 @@ namespace eosio { namespace chain {
             const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, apply_context& context)> substitute_apply;
       private:
          unique_ptr<struct wasm_interface_impl> my;
+         vm_type vm;
    };
 
 } } // eosio::chain

libraries/chain/include/eosio/chain/wasm_interface_private.hpp

@@ -50,11 +50,21 @@ namespace eosio { namespace chain {
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       struct eosvmoc_tier {
          eosvmoc_tier(const boost::filesystem::path& d, const eosvmoc::config& c, const chainbase::database& db)
-          : cc(d, c, db), exec(cc),
-            mem(wasm_constraints::maximum_linear_memory/wasm_constraints::wasm_page_size) {}
+          : cc(d, c, db) {
+             // construct exec for the main thread
+             init_thread_local_data();
+          }
+
+         // Support multi-threaded execution.
+         void init_thread_local_data() {
+            exec = std::make_unique<eosvmoc::executor>(cc);
+         }
+
          eosvmoc::code_cache_async cc;
-         eosvmoc::executor exec;
-         eosvmoc::memory mem;
+
+         // Each thread requires its own exec and mem. Defined in wasm_interface.cpp
+         thread_local static std::unique_ptr<eosvmoc::executor> exec;
+         thread_local static eosvmoc::memory mem;
       };
 #endif
 

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp

@@ -34,11 +34,16 @@ class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface {
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
       void immediately_exit_currently_running_module() override;
+      void init_thread_local_data() override;
 
       friend eosvmoc_instantiated_module;
       eosvmoc::code_cache_sync cc;
       eosvmoc::executor exec;
       eosvmoc::memory mem;
+
+      // Defined in eos-vm-oc.cpp. Used for non-main thread in multi-threaded execution
+      thread_local static std::unique_ptr<eosvmoc::executor> exec_thread_local;
+      thread_local static eosvmoc::memory mem_thread_local;
 };
 
 /**

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -15,6 +15,7 @@
 
 
 #include <thread>
+#include <shared_mutex>
 
 namespace std {
     template<> struct hash<eosio::chain::eosvmoc::code_tuple> {
@@ -84,6 +85,9 @@ class code_cache_base {
 
       template <typename T>
       void serialize_cache_index(fc::datastream<T>& ds);
+
+      std::thread::id _main_thread_id;
+      bool is_main_thread() const;
 };
 
 class code_cache_async : public code_cache_base {
@@ -114,4 +118,4 @@ class code_cache_sync : public code_cache_base {
       const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version);
 };
 
-}}}
+}}}

libraries/chain/include/eosio/chain/webassembly/runtime_interface.hpp

@@ -27,6 +27,9 @@ class wasm_runtime_interface {
       virtual void immediately_exit_currently_running_module() = 0;
 
       virtual ~wasm_runtime_interface();
+
+      // eosvmoc_runtime needs this
+      virtual void init_thread_local_data() {};
 };
 
 }}

libraries/chain/protocol_feature_manager.cpp

@@ -569,7 +569,7 @@ Enables new `get_block_num` intrinsic which returns the current block number.
 
    protocol_feature_manager::protocol_feature_manager(
       protocol_feature_set&& pfs,
-      std::function<deep_mind_handler*()> get_deep_mind_logger
+      std::function<deep_mind_handler*(bool is_trx_transient)> get_deep_mind_logger
    ):_protocol_feature_set( std::move(pfs) ), _get_deep_mind_logger(get_deep_mind_logger)
    {
       _builtin_protocol_features.resize( _protocol_feature_set._recognized_builtin_protocol_features.size() );
@@ -750,7 +750,8 @@ Enables new `get_block_num` intrinsic which returns the current block number.
                   ("digest", feature_digest)
       );
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      // activate_feature is called by init. no transaction specific logging is possible
+      if (auto dm_logger = _get_deep_mind_logger(false)) {
          dm_logger->on_activate_feature(*itr);
       }
 

libraries/chain/resource_limits.cpp

@@ -65,7 +65,8 @@ void resource_limits_manager::initialize_database() {
       state.virtual_net_limit = config.net_limit_parameters.max;
    });
 
-   if (auto dm_logger = _get_deep_mind_logger()) {
+   // At startup, no transaction specific logging is possible
+   if (auto dm_logger = _get_deep_mind_logger(false)) {
       dm_logger->on_init_resource_limits(config, state);
    }
 }
@@ -93,7 +94,7 @@ void resource_limits_manager::read_from_snapshot( const snapshot_reader_ptr& sna
    });
 }
 
-void resource_limits_manager::initialize_account(const account_name& account) {
+void resource_limits_manager::initialize_account(const account_name& account, bool is_trx_transient) {
    const auto& limits = _db.create<resource_limits_object>([&]( resource_limits_object& bl ) {
       bl.owner = account;
    });
@@ -101,7 +102,7 @@ void resource_limits_manager::initialize_account(const account_name& account) {
    const auto& usage = _db.create<resource_usage_object>([&]( resource_usage_object& bu ) {
       bu.owner = account;
    });
-   if (auto dm_logger = _get_deep_mind_logger()) {
+   if (auto dm_logger = _get_deep_mind_logger(is_trx_transient)) {
       dm_logger->on_newaccount_resource_limits(limits, usage);
    }
 }
@@ -116,7 +117,9 @@ void resource_limits_manager::set_block_parameters(const elastic_limit_parameter
       c.cpu_limit_parameters = cpu_limit_parameters;
       c.net_limit_parameters = net_limit_parameters;
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      // set_block_parameters is called by controller::finalize_block,
+      // where transaction specific logging is not possible
+      if (auto dm_logger = _get_deep_mind_logger(false)) {
          dm_logger->on_update_resource_limits_config(c);
       }
    });
@@ -133,7 +136,7 @@ void resource_limits_manager::update_account_usage(const flat_set<account_name>&
    }
 }
 
-void resource_limits_manager::add_transaction_usage(const flat_set<account_name>& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t time_slot ) {
+void resource_limits_manager::add_transaction_usage(const flat_set<account_name>& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t time_slot, bool is_trx_transient ) {
    const auto& state = _db.get<resource_limits_state_object>();
    const auto& config = _db.get<resource_limits_config_object>();
 
@@ -149,7 +152,7 @@ void resource_limits_manager::add_transaction_usage(const flat_set<account_name>
           bu.net_usage.add( net_usage, time_slot, config.account_net_usage_average_window );
           bu.cpu_usage.add( cpu_usage, time_slot, config.account_cpu_usage_average_window );
 
-         if (auto dm_logger = _get_deep_mind_logger()) {
+         if (auto dm_logger = _get_deep_mind_logger(is_trx_transient)) {
             dm_logger->on_update_account_usage(bu);
          }
       });
@@ -205,7 +208,7 @@ void resource_limits_manager::add_transaction_usage(const flat_set<account_name>
    EOS_ASSERT( state.pending_net_usage <= config.net_limit_parameters.max, block_resource_exhausted, "Block has insufficient net resources" );
 }
 
-void resource_limits_manager::add_pending_ram_usage( const account_name account, int64_t ram_delta ) {
+void resource_limits_manager::add_pending_ram_usage( const account_name account, int64_t ram_delta, bool is_trx_transient ) {
    if (ram_delta == 0) {
       return;
    }
@@ -220,7 +223,7 @@ void resource_limits_manager::add_pending_ram_usage( const account_name account,
    _db.modify( usage, [&]( auto& u ) {
       u.ram_usage += ram_delta;
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      if (auto dm_logger = _get_deep_mind_logger(is_trx_transient)) {
          dm_logger->on_ram_event(account, u.ram_usage, ram_delta);
       }
    });
@@ -243,7 +246,7 @@ int64_t resource_limits_manager::get_account_ram_usage( const account_name& name
 }
 
 
-bool resource_limits_manager::set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight) {
+bool resource_limits_manager::set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight, bool is_trx_transient) {
    //const auto& usage = _db.get<resource_usage_object,by_owner>( account );
    /*
     * Since we need to delay these until the next resource limiting boundary, these are created in a "pending"
@@ -289,7 +292,7 @@ bool resource_limits_manager::set_account_limits( const account_name& account, i
       pending_limits.net_weight = net_weight;
       pending_limits.cpu_weight = cpu_weight;
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      if (auto dm_logger = _get_deep_mind_logger(is_trx_transient)) {
          dm_logger->on_set_account_limits(pending_limits);
       }
    });
@@ -356,7 +359,9 @@ void resource_limits_manager::process_account_limit_updates() {
          multi_index.remove(*itr);
       }
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      // process_account_limit_updates is called by controller::finalize_block,
+      // where transaction specific logging is not possible
+      if (auto dm_logger = _get_deep_mind_logger(false)) {
          dm_logger->on_update_resource_limits_state(state);
       }
    });
@@ -376,7 +381,9 @@ void resource_limits_manager::process_block_usage(uint32_t block_num) {
       state.update_virtual_net_limit(config);
       state.pending_net_usage = 0;
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      // process_block_usage is called by controller::finalize_block,
+      // where transaction specific logging is not possible
+      if (auto dm_logger = _get_deep_mind_logger(false)) {
          dm_logger->on_update_resource_limits_state(state);
       }
    });

libraries/chain/transaction.cpp

@@ -62,16 +62,19 @@ fc::microseconds transaction::get_signature_keys( const vector<signature_type>&
 { try {
    auto start = fc::time_point::now();
    recovered_pub_keys.clear();
-   const digest_type digest = sig_digest(chain_id, cfd);
 
-   for(const signature_type& sig : signatures) {
-      auto now = fc::time_point::now();
-      EOS_ASSERT( now < deadline, tx_cpu_usage_exceeded, "transaction signature verification executed for too long ${time}us",
-                  ("time", now - start)("now", now)("deadline", deadline)("start", start) );
-      auto[ itr, successful_insertion ] = recovered_pub_keys.emplace( sig, digest );
-      EOS_ASSERT( allow_duplicate_keys || successful_insertion, tx_duplicate_sig,
-                  "transaction includes more than one signature signed using the same key associated with public key: ${key}",
-                  ("key", *itr ) );
+   if ( !signatures.empty() ) {
+      const digest_type digest = sig_digest(chain_id, cfd);
+
+      for(const signature_type& sig : signatures) {
+         auto now = fc::time_point::now();
+         EOS_ASSERT( now < deadline, tx_cpu_usage_exceeded, "transaction signature verification executed for too long ${time}us",
+                     ("time", now - start)("now", now)("deadline", deadline)("start", start) );
+         auto[ itr, successful_insertion ] = recovered_pub_keys.emplace( sig, digest );
+         EOS_ASSERT( allow_duplicate_keys || successful_insertion, tx_duplicate_sig,
+                     "transaction includes more than one signature signed using the same key associated with public key: ${key}",
+                     ("key", *itr ) );
+      }
    }
 
    return fc::time_point::now() - start;

libraries/chain/transaction_context.cpp

@@ -59,7 +59,7 @@ namespace eosio { namespace chain {
    ,net_usage(trace->net_usage)
    ,pseudo_start(s)
    {
-      if (!c.skip_db_sessions()) {
+      if (!c.skip_db_sessions() && !is_read_only()) {
          undo_session.emplace(c.mutable_db().start_undo_session(true));
       }
       trace->id = packed_trx.id();
@@ -67,7 +67,7 @@ namespace eosio { namespace chain {
       trace->block_time = c.pending_block_time();
       trace->producer_block_id = c.pending_producer_block_id();
 
-      if(auto dm_logger = control.get_deep_mind_logger())
+      if(auto dm_logger = c.get_deep_mind_logger(is_transient()))
       {
          dm_logger->on_start_transaction();
       }
@@ -75,7 +75,7 @@ namespace eosio { namespace chain {
 
    transaction_context::~transaction_context()
    {
-      if(auto dm_logger = control.get_deep_mind_logger())
+      if(auto dm_logger = control.get_deep_mind_logger(is_transient()))
       {
          dm_logger->on_end_transaction();
       }
@@ -105,13 +105,13 @@ namespace eosio { namespace chain {
       _deadline = start + objective_duration_limit;
 
       // Possibly lower net_limit to the maximum net usage a transaction is allowed to be billed
-      if( cfg.max_transaction_net_usage <= net_limit ) {
+      if( cfg.max_transaction_net_usage <= net_limit && !is_read_only() ) {
          net_limit = cfg.max_transaction_net_usage;
          net_limit_due_to_block = false;
       }
 
       // Possibly lower objective_duration_limit to the maximum cpu usage a transaction is allowed to be billed
-      if( cfg.max_transaction_cpu_usage <= objective_duration_limit.count() ) {
+      if( cfg.max_transaction_cpu_usage <= objective_duration_limit.count() && !is_read_only() ) {
          objective_duration_limit = fc::microseconds(cfg.max_transaction_cpu_usage);
          billing_timer_exception_code = tx_cpu_usage_exceeded::code_value;
          tx_cpu_usage_reason = tx_cpu_usage_exceeded_reason::on_chain_consensus_max_transaction_cpu_usage;
@@ -138,53 +138,57 @@ namespace eosio { namespace chain {
       }
 
       initial_objective_duration_limit = objective_duration_limit;
-
-      if( explicit_billed_cpu_time )
-         validate_cpu_usage_to_bill( billed_cpu_time_us, std::numeric_limits<int64_t>::max(), false, subjective_cpu_bill_us); // Fail early if the amount to be billed is too high
-
-      // Record accounts to be billed for network and CPU usage
-      if( control.is_builtin_activated(builtin_protocol_feature_t::only_bill_first_authorizer) ) {
-         bill_to_accounts.insert( trx.first_authorizer() );
-      } else {
-         for( const auto& act : trx.actions ) {
-            for( const auto& auth : act.authorization ) {
-               bill_to_accounts.insert( auth.actor );
-            }
-         }
-      }
-      validate_ram_usage.reserve( bill_to_accounts.size() );
-
-      // Update usage values of accounts to reflect new time
-      rl.update_account_usage( bill_to_accounts, block_timestamp_type(control.pending_block_time()).slot );
-
-      // Calculate the highest network usage and CPU time that all of the billed accounts can afford to be billed
       int64_t account_net_limit = 0;
       int64_t account_cpu_limit = 0;
-      bool greylisted_net = false, greylisted_cpu = false;
-      std::tie( account_net_limit, account_cpu_limit, greylisted_net, greylisted_cpu) = max_bandwidth_billed_accounts_can_pay();
-      net_limit_due_to_greylist |= greylisted_net;
-      cpu_limit_due_to_greylist |= greylisted_cpu;
+
+      if ( !is_read_only() ) {
+         if( explicit_billed_cpu_time )
+            validate_cpu_usage_to_bill( billed_cpu_time_us, std::numeric_limits<int64_t>::max(), false, subjective_cpu_bill_us); // Fail early if the amount to be billed is too high
+
+         // Record accounts to be billed for network and CPU usage
+         if( control.is_builtin_activated(builtin_protocol_feature_t::only_bill_first_authorizer) ) {
+            bill_to_accounts.insert( trx.first_authorizer() );
+         } else {
+            for( const auto& act : trx.actions ) {
+               for( const auto& auth : act.authorization ) {
+                  bill_to_accounts.insert( auth.actor );
+               }
+            }
+         }
+         validate_ram_usage.reserve( bill_to_accounts.size() );
+
+         // Update usage values of accounts to reflect new time
+         rl.update_account_usage( bill_to_accounts, block_timestamp_type(control.pending_block_time()).slot );
+
+         // Calculate the highest network usage and CPU time that all of the billed accounts can afford to be billed
+         bool greylisted_net = false, greylisted_cpu = false;
+         std::tie( account_net_limit, account_cpu_limit, greylisted_net, greylisted_cpu) = max_bandwidth_billed_accounts_can_pay();
+         net_limit_due_to_greylist |= greylisted_net;
+         cpu_limit_due_to_greylist |= greylisted_cpu;
+      }
 
       eager_net_limit = net_limit;
 
-      // Possibly lower eager_net_limit to what the billed accounts can pay plus some (objective) leeway
-      auto new_eager_net_limit = std::min( eager_net_limit, static_cast<uint64_t>(account_net_limit + cfg.net_usage_leeway) );
-      if( new_eager_net_limit < eager_net_limit ) {
-         eager_net_limit = new_eager_net_limit;
-         net_limit_due_to_block = false;
-      }
+      if ( !is_read_only() ) {
+         // Possibly lower eager_net_limit to what the billed accounts can pay plus some (objective) leeway
+         auto new_eager_net_limit = std::min( eager_net_limit, static_cast<uint64_t>(account_net_limit + cfg.net_usage_leeway) );
+         if( new_eager_net_limit < eager_net_limit ) {
+            eager_net_limit = new_eager_net_limit;
+            net_limit_due_to_block = false;
+         }
 
-      // Possibly limit deadline if the duration accounts can be billed for (+ a subjective leeway) does not exceed current delta
-      if( (fc::microseconds(account_cpu_limit) + leeway) <= (_deadline - start) ) {
-         _deadline = start + fc::microseconds(account_cpu_limit) + leeway;
-         billing_timer_exception_code = leeway_deadline_exception::code_value;
+         // Possibly limit deadline if the duration accounts can be billed for (+ a subjective leeway) does not exceed current delta
+         if( (fc::microseconds(account_cpu_limit) + leeway) <= (_deadline - start) ) {
+            _deadline = start + fc::microseconds(account_cpu_limit) + leeway;
+            billing_timer_exception_code = leeway_deadline_exception::code_value;
+         }
       }
 
       // Possibly limit deadline to subjective max_transaction_time
       if( max_transaction_time_subjective != fc::microseconds::maximum() && (start + max_transaction_time_subjective) <= _deadline ) {
          _deadline = start + max_transaction_time_subjective;
-         tx_cpu_usage_reason = billed_cpu_time_us > 0 ? tx_cpu_usage_exceeded_reason::speculative_executed_adjusted_max_transaction_time :
-                                                        tx_cpu_usage_exceeded_reason::node_configured_max_transaction_time;
+         tx_cpu_usage_reason = billed_cpu_time_us > 0 ?
+            tx_cpu_usage_exceeded_reason::speculative_executed_adjusted_max_transaction_time : tx_cpu_usage_exceeded_reason::node_configured_max_transaction_time;
          billing_timer_exception_code = tx_cpu_usage_exceeded::code_value;
       }
 
@@ -194,20 +198,22 @@ namespace eosio { namespace chain {
          billing_timer_exception_code = deadline_exception::code_value;
       }
 
-      if( !explicit_billed_cpu_time ) {
-         int64_t validate_account_cpu_limit = account_cpu_limit - subjective_cpu_bill_us + leeway.count(); // Add leeway to allow powerup
-         // Possibly limit deadline to account subjective cpu left
-         if( subjective_cpu_bill_us > 0 && (start + fc::microseconds(validate_account_cpu_limit) < _deadline) ) {
-            _deadline = start + fc::microseconds(validate_account_cpu_limit);
-            billing_timer_exception_code = tx_cpu_usage_exceeded::code_value;
-            tx_cpu_usage_reason = tx_cpu_usage_exceeded_reason::account_cpu_limit;
-         }
+      if ( !is_read_only() ) {
+         if( !explicit_billed_cpu_time ) {
+            int64_t validate_account_cpu_limit = account_cpu_limit - subjective_cpu_bill_us + leeway.count(); // Add leeway to allow powerup
+            // Possibly limit deadline to account subjective cpu left
+            if( subjective_cpu_bill_us > 0 && (start + fc::microseconds(validate_account_cpu_limit) < _deadline) ) {
+               _deadline = start + fc::microseconds(validate_account_cpu_limit);
+               billing_timer_exception_code = tx_cpu_usage_exceeded::code_value;
+               tx_cpu_usage_reason = tx_cpu_usage_exceeded_reason::account_cpu_limit;
+            }
 
-         // Fail early if amount of the previous speculative execution is within 10% of remaining account cpu available
-         if( validate_account_cpu_limit > 0 )
-            validate_account_cpu_limit -= EOS_PERCENT( validate_account_cpu_limit, 10 * config::percent_1 );
-         if( validate_account_cpu_limit < 0 ) validate_account_cpu_limit = 0;
-         validate_account_cpu_usage_estimate( billed_cpu_time_us, validate_account_cpu_limit, subjective_cpu_bill_us );
+            // Fail early if amount of the previous speculative execution is within 10% of remaining account cpu available
+            if( validate_account_cpu_limit > 0 )
+               validate_account_cpu_limit -= EOS_PERCENT( validate_account_cpu_limit, 10 * config::percent_1 );
+            if( validate_account_cpu_limit < 0 ) validate_account_cpu_limit = 0;
+            validate_account_cpu_usage_estimate( billed_cpu_time_us, validate_account_cpu_limit, subjective_cpu_bill_us );
+         }
       }
 
       // Explicit billed_cpu_time_us should be used, block_deadline will be maximum unless in test code
@@ -248,6 +254,9 @@ namespace eosio { namespace chain {
                                                  uint64_t packed_trx_prunable_size )
    {
       const transaction& trx = packed_trx.get_transaction();
+      if ( is_transient() ) {
+         EOS_ASSERT( trx.delay_sec.value == 0, transaction_exception, "dry-run or read-only transaction cannot be delayed" );
+      }
       if( trx.transaction_extensions.size() > 0 ) {
          disallow_transaction_extensions( "no transaction extensions supported yet for input transactions" );
       }
@@ -277,12 +286,17 @@ namespace eosio { namespace chain {
       published = control.pending_block_time();
       is_input = true;
       if (!control.skip_trx_checks()) {
-         control.validate_expiration(trx);
-         control.validate_tapos(trx);
+         if ( !is_read_only() ) {
+            control.validate_expiration(trx);
+            control.validate_tapos(trx);
+         }
          validate_referenced_accounts( trx, enforce_whiteblacklist && control.is_speculative_block() );
       }
-      init( initial_net_usage);
-      record_transaction( packed_trx.id(), trx.expiration ); /// checks for dupes
+
+      init( initial_net_usage );
+      if ( !is_read_only() ) {
+         record_transaction( packed_trx.id(), trx.expiration );
+      }
    }
 
    void transaction_context::init_for_deferred_trx( fc::time_point p )
@@ -332,6 +346,13 @@ namespace eosio { namespace chain {
    void transaction_context::finalize() {
       EOS_ASSERT( is_initialized, transaction_exception, "must first initialize" );
 
+      // read-only transactions only need net_usage and elapsed in the trace
+      if ( is_read_only() ) {
+         net_usage = ((net_usage + 7)/8)*8; // Round up to nearest multiple of word size (8 bytes)
+         trace->elapsed = fc::time_point::now() - start;
+         return;
+      }
+                                         
       if( is_input ) {
          const transaction& trx = packed_trx.get_transaction();
          auto& am = control.get_mutable_authorization_manager();
@@ -383,7 +404,7 @@ namespace eosio { namespace chain {
       validate_cpu_usage_to_bill( billed_cpu_time_us, account_cpu_limit, true, subjective_cpu_bill_us );
 
       rl.add_transaction_usage( bill_to_accounts, static_cast<uint64_t>(billed_cpu_time_us), net_usage,
-                                block_timestamp_type(control.pending_block_time()).slot ); // Should never fail
+                                block_timestamp_type(control.pending_block_time()).slot, is_transient() ); // Should never fail
    }
 
    void transaction_context::squash() {
@@ -588,7 +609,7 @@ namespace eosio { namespace chain {
 
    void transaction_context::add_ram_usage( account_name account, int64_t ram_delta ) {
       auto& rl = control.get_mutable_resource_limits_manager();
-      rl.add_pending_ram_usage( account, ram_delta );
+      rl.add_pending_ram_usage( account, ram_delta, is_transient() );
       if( ram_delta > 0 ) {
          validate_ram_usage.insert( account );
       }
@@ -709,7 +730,7 @@ namespace eosio { namespace chain {
       apply_context acontext( control, *this, action_ordinal, recurse_depth );
 
       if (recurse_depth == 0) {
-         if (auto dm_logger = control.get_deep_mind_logger()) {
+         if (auto dm_logger = control.get_deep_mind_logger(is_transient())) {
             dm_logger->on_input_action();
          }
       }
@@ -741,7 +762,7 @@ namespace eosio { namespace chain {
         gto.expiration  = gto.delay_until + fc::seconds(control.get_global_properties().configuration.deferred_trx_expiration_window);
         trx_size = gto.set( trx );
 
-        if (auto dm_logger = control.get_deep_mind_logger()) {
+        if (auto dm_logger = control.get_deep_mind_logger(is_transient())) {
            std::string event_id = RAM_EVENT_ID("${id}", ("id", gto.id));
 
            dm_logger->on_create_deferred(deep_mind_handler::operation_qualifier::push, gto, packed_trx);
@@ -789,6 +810,10 @@ namespace eosio { namespace chain {
          auto* code = db.find<account_object, by_name>(a.account);
          EOS_ASSERT( code != nullptr, transaction_exception,
                      "action's code account '${account}' does not exist", ("account", a.account) );
+         if ( is_read_only() ) {
+            EOS_ASSERT( a.authorization.size() == 0, transaction_exception,
+                       "read-only action '${name}' cannot have authorizations", ("name", a.name) );
+         }
          for( const auto& auth : a.authorization ) {
             one_auth = true;
             auto* actor = db.find<account_object, by_name>(auth.actor);
@@ -801,7 +826,7 @@ namespace eosio { namespace chain {
                actors.insert( auth.actor );
          }
       }
-      EOS_ASSERT( one_auth || is_dry_run(), tx_no_auths, "transaction must have at least one authorization" );
+      EOS_ASSERT( one_auth || is_transient(), tx_no_auths, "transaction must have at least one authorization" );
 
       if( enforce_actor_whitelist_blacklist ) {
          control.check_actor_list( actors );

libraries/chain/wasm_interface.cpp

@@ -23,7 +23,6 @@
 #include <softfloat.hpp>
 #include <compiler_builtins.hpp>
 #include <boost/asio.hpp>
-#include <boost/bind.hpp>
 #include <fstream>
 #include <string.h>
 
@@ -34,10 +33,19 @@
 namespace eosio { namespace chain {
 
    wasm_interface::wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile)
-     : my( new wasm_interface_impl(vm, eosvmoc_tierup, d, data_dir, eosvmoc_config, profile) ) {}
+     : my( new wasm_interface_impl(vm, eosvmoc_tierup, d, data_dir, eosvmoc_config, profile) ), vm( vm ) {}
 
    wasm_interface::~wasm_interface() {}
 
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+   void wasm_interface::init_thread_local_data() {
+      if (my->eosvmoc)
+         my->eosvmoc->init_thread_local_data();
+      else if (vm == wasm_interface::vm_type::eos_vm_oc && my->runtime_interface)
+         my->runtime_interface->init_thread_local_data();
+   }
+#endif
+
    void wasm_interface::validate(const controller& control, const bytes& code) {
       const auto& pso = control.db().get<protocol_state_object>();
 
@@ -96,7 +104,7 @@ namespace eosio { namespace chain {
             once_is_enough = true;
          }
          if(cd) {
-            my->eosvmoc->exec.execute(*cd, my->eosvmoc->mem, context);
+            my->eosvmoc->exec->execute(*cd, my->eosvmoc->mem, context);
             return;
          }
       }
@@ -115,6 +123,11 @@ namespace eosio { namespace chain {
    wasm_instantiated_module_interface::~wasm_instantiated_module_interface() {}
    wasm_runtime_interface::~wasm_runtime_interface() {}
 
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+   thread_local std::unique_ptr<eosvmoc::executor> wasm_interface_impl::eosvmoc_tier::exec {};
+   thread_local eosvmoc::memory wasm_interface_impl::eosvmoc_tier::mem{ wasm_constraints::maximum_linear_memory/wasm_constraints::wasm_page_size };
+#endif
+
 std::istream& operator>>(std::istream& in, wasm_interface::vm_type& runtime) {
    std::string s;
    in >> s;

libraries/chain/webassembly/action.cpp

@@ -24,9 +24,10 @@ namespace eosio { namespace chain { namespace webassembly {
    void interface::set_action_return_value( span<const char> packed_blob ) {
       auto max_action_return_value_size = 
          context.control.get_global_properties().configuration.max_action_return_value_size;
-      EOS_ASSERT(packed_blob.size() <= max_action_return_value_size,
-                 action_return_value_exception, 
-                 "action return value size must be less or equal to ${s} bytes", ("s", max_action_return_value_size));
+      if( !context.trx_context.is_read_only() )
+         EOS_ASSERT(packed_blob.size() <= max_action_return_value_size,
+                    action_return_value_exception,
+                    "action return value size must be less or equal to ${s} bytes", ("s", max_action_return_value_size));
       context.action_return_value.assign( packed_blob.data(), packed_blob.data() + packed_blob.size() );
    }
 }}} // ns eosio::chain::webassembly

libraries/chain/webassembly/privileged.cpp

@@ -17,14 +17,16 @@ namespace eosio { namespace chain { namespace webassembly {
    }
 
    void interface::preactivate_feature( legacy_ptr<const digest_type> feature_digest ) {
-      context.control.preactivate_feature( *feature_digest );
+      EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "preactivate_feature not allowed in a readonly transaction");
+      context.control.preactivate_feature( *feature_digest, context.trx_context.is_transient() );
    }
 
    void interface::set_resource_limits( account_name account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight ) {
+      EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_resource_limits not allowed in a readonly transaction");
       EOS_ASSERT(ram_bytes >= -1, wasm_execution_error, "invalid value for ram resource limit expected [-1,INT64_MAX]");
       EOS_ASSERT(net_weight >= -1, wasm_execution_error, "invalid value for net resource weight expected [-1,INT64_MAX]");
       EOS_ASSERT(cpu_weight >= -1, wasm_execution_error, "invalid value for cpu resource weight expected [-1,INT64_MAX]");
-      if( context.control.get_mutable_resource_limits_manager().set_account_limits(account, ram_bytes, net_weight, cpu_weight) ) {
+      if( context.control.get_mutable_resource_limits_manager().set_account_limits(account, ram_bytes, net_weight, cpu_weight, context.trx_context.is_transient()) ) {
          context.trx_context.validate_ram_usage.insert( account );
       }
    }
@@ -99,6 +101,7 @@ namespace eosio { namespace chain { namespace webassembly {
       return s;
    }
    void interface::set_wasm_parameters_packed( span<const char> packed_parameters ) {
+      EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_wasm_parameters_packed not allowed in a readonly transaction");
       datastream<const char*> ds( packed_parameters.data(), packed_parameters.size() );
       uint32_t version;
       chain::wasm_config cfg;
@@ -113,6 +116,7 @@ namespace eosio { namespace chain { namespace webassembly {
       );
    }
    int64_t interface::set_proposed_producers( legacy_span<const char> packed_producer_schedule) {
+      EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_proposed_producers not allowed in a readonly transaction");
       datastream<const char*> ds( packed_producer_schedule.data(), packed_producer_schedule.size() );
       std::vector<producer_authority> producers;
       std::vector<legacy::producer_key> old_version;
@@ -129,6 +133,7 @@ namespace eosio { namespace chain { namespace webassembly {
    }
 
    int64_t interface::set_proposed_producers_ex( uint64_t packed_producer_format, legacy_span<const char> packed_producer_schedule) {
+      EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_proposed_producers_ex not allowed in a readonly transaction");
       if (packed_producer_format == 0) {
          return set_proposed_producers(std::move(packed_producer_schedule));
       } else if (packed_producer_format == 1) {
@@ -157,6 +162,7 @@ namespace eosio { namespace chain { namespace webassembly {
    }
 
    void interface::set_blockchain_parameters_packed( legacy_span<const char> packed_blockchain_parameters ) {
+      EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_blockchain_parameters_packed not allowed in a readonly transaction");
       datastream<const char*> ds( packed_blockchain_parameters.data(), packed_blockchain_parameters.size() );
       chain::chain_config_v0 cfg;
       fc::raw::unpack(ds, cfg);
@@ -188,6 +194,7 @@ namespace eosio { namespace chain { namespace webassembly {
    }
 
    void interface::set_parameters_packed( span<const char> packed_parameters ){
+      EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_parameters_packed not allowed in a readonly transaction");
       datastream<const char*> ds( packed_parameters.data(), packed_parameters.size() );
 
       chain::chain_config cfg = context.control.get_global_properties().configuration;
@@ -207,6 +214,7 @@ namespace eosio { namespace chain { namespace webassembly {
    }
 
    void interface::set_privileged( account_name n, bool is_priv ) {
+      EOS_ASSERT(!context.trx_context.is_read_only(), wasm_execution_error, "set_privileged not allowed in a readonly transaction");
       const auto& a = context.db.get<account_metadata_object, by_name>( n );
       context.db.modify( a, [&]( auto& ma ){
          ma.set_privileged( is_priv );

libraries/chain/webassembly/runtimes/eos-vm-oc.cpp

@@ -15,7 +15,8 @@ class eosvmoc_instantiated_module : public wasm_instantiated_module_interface {
       eosvmoc_instantiated_module(const digest_type& code_hash, const uint8_t& vm_version, eosvmoc_runtime& wr) :
          _code_hash(code_hash),
          _vm_version(vm_version),
-         _eosvmoc_runtime(wr)
+         _eosvmoc_runtime(wr),
+         _main_thread_id(std::this_thread::get_id())
       {
 
       }
@@ -24,16 +25,22 @@ class eosvmoc_instantiated_module : public wasm_instantiated_module_interface {
          _eosvmoc_runtime.cc.free_code(_code_hash, _vm_version);
       }
 
+      bool is_main_thread() { return _main_thread_id == std::this_thread::get_id(); };
+
       void apply(apply_context& context) override {
          const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version);
          EOS_ASSERT(cd, wasm_execution_error, "EOS VM OC instantiation failed");
 
-         _eosvmoc_runtime.exec.execute(*cd, _eosvmoc_runtime.mem, context);
+         if ( is_main_thread() )
+            _eosvmoc_runtime.exec.execute(*cd, _eosvmoc_runtime.mem, context);
+         else
+            _eosvmoc_runtime.exec_thread_local->execute(*cd, _eosvmoc_runtime.mem_thread_local, context);
       }
 
       const digest_type              _code_hash;
       const uint8_t                  _vm_version;
       eosvmoc_runtime&               _eosvmoc_runtime;
+      std::thread::id                _main_thread_id;
 };
 
 eosvmoc_runtime::eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db)
@@ -45,11 +52,17 @@ eosvmoc_runtime::~eosvmoc_runtime() {
 
 std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
                                                                                         const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) {
-
    return std::make_unique<eosvmoc_instantiated_module>(code_hash, vm_type, *this);
 }
 
 //never called. EOS VM OC overrides eosio_exit to its own implementation
 void eosvmoc_runtime::immediately_exit_currently_running_module() {}
 
+void eosvmoc_runtime::init_thread_local_data() {
+   exec_thread_local = std::make_unique<eosvmoc::executor>(cc);
+}
+
+thread_local std::unique_ptr<eosvmoc::executor> eosvmoc_runtime::exec_thread_local {};
+thread_local eosvmoc::memory eosvmoc_runtime::mem_thread_local{ wasm_constraints::maximum_linear_memory/wasm_constraints::wasm_page_size };
+
 }}}}

libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp

@@ -106,9 +106,11 @@ std::tuple<size_t, size_t> code_cache_async::consume_compile_thread_queue() {
    return {gotsome, bytes_remaining};
 }
 
+
 const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version) {
    //if there are any outstanding compiles, process the result queue now
-   if(_outstanding_compiles_and_poison.size()) {
+   //do this only on main thread (which is in single threaded write window)
+   if(is_main_thread() && _outstanding_compiles_and_poison.size()) {
       auto [count_processed, bytes_remaining] = consume_compile_thread_queue();
 
       if(count_processed)
@@ -134,9 +136,12 @@ const code_descriptor* const code_cache_async::get_descriptor_for_code(const dig
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
+      if (is_main_thread())
+         _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
+   if(!is_main_thread()) // on read-only thread
+      return nullptr;
 
    const code_tuple ct = code_tuple{code_id, vm_version};
 
@@ -178,9 +183,12 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
+      if (is_main_thread())
+         _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
+   if(!is_main_thread())
+      return nullptr;
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
    if(!codeobject) //should be impossible right?
@@ -204,7 +212,8 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const
 
 code_cache_base::code_cache_base(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
    _db(db),
-   _cache_file_path(data_dir/"code_cache.bin")
+   _cache_file_path(data_dir/"code_cache.bin"),
+   _main_thread_id(std::this_thread::get_id())
 {
    static_assert(sizeof(allocator_t) <= header_offset, "header offset intersects with allocator");
 
@@ -382,4 +391,7 @@ void code_cache_base::check_eviction_threshold(size_t free_bytes) {
       run_eviction_round();
 }
 
+bool code_cache_base::is_main_thread() const {
+   return _main_thread_id == std::this_thread::get_id();
+}
 }}}

libraries/custom_appbase/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library(custom_appbase INTERFACE)
+target_include_directories(custom_appbase INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include")
+target_link_libraries(custom_appbase INTERFACE appbase)
+
+add_subdirectory(tests)

libraries/custom_appbase/include/eosio/chain/application.hpp

@@ -0,0 +1,111 @@
+#pragma once
+
+#include <appbase/application_base.hpp>
+#include <appbase/execution_priority_queue.hpp>
+
+/*
+ * Custmomize appbase to support two-queue exector.
+*/
+namespace appbase { 
+
+enum class exec_window {
+   read,  // the window during which operations from read_only_trx_safe queue
+          // can be executed in app thread in parallel with read-only transactions
+          // in read-only transaction excuting threads.
+   write, // the window during which operations from both general and
+          // read_only_trx_safe queues can be executed in app thread,
+          // while read-only transactions are not executed in read-only
+          // transaction excuting threads.
+};
+
+enum class exec_queue {
+   read_only_trx_safe, // the queue storing operations which are safe to execute
+                       // on app thread in parallel with read-only transactions
+                       // in read-only transaction excuting threads.
+   general             // the queue storing operations which can be only executed
+                       // on the app thread while read-only transactions are
+                       // not being executed in read-only threads
+};
+
+class two_queue_executor {
+public:
+
+   template <typename Func>
+   auto post( int priority, exec_queue q, Func&& func ) {
+      if ( q == exec_queue::general )
+         return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      else
+         return boost::asio::post(io_serv_, read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+   }
+
+   // Legacy and deprecated. To be removed after cleaning up its uses in base appbase
+   template <typename Func>
+   auto post( int priority, Func&& func ) {
+      // safer to use general queue for unknown type of operation since operations
+      // from general queue are not executed in parallel with read-only transactions
+      return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+   }
+
+   boost::asio::io_service& get_io_service() { return io_serv_; }
+
+   bool execute_highest() {
+      if ( exec_window_ == exec_window::write ) {
+         if( !general_queue_.empty() && ( read_only_trx_safe_queue_.empty() || *read_only_trx_safe_queue_.top() < *general_queue_.top()) )  {
+            // general_queue_'s top function's priority greater than read_only_trx_safe_queue_'s top function's, or general_queue_ empty
+            general_queue_.execute_highest();
+         } else if( !read_only_trx_safe_queue_.empty() ) {
+            read_only_trx_safe_queue_.execute_highest();
+         }
+         return !read_only_trx_safe_queue_.empty() || !general_queue_.empty();
+      } else {
+         return read_only_trx_safe_queue_.execute_highest();
+      }
+   }
+
+   template <typename Function>
+   boost::asio::executor_binder<Function, appbase::execution_priority_queue::executor>
+   wrap(int priority, exec_queue q, Function&& func ) {
+      if ( q == exec_queue::general )
+         return general_queue_.wrap(priority, --order_, std::forward<Function>(func));
+      else
+         return read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Function>(func));
+   }
+     
+   void clear() {
+      read_only_trx_safe_queue_.clear();
+      general_queue_.clear();
+   }
+
+   void set_to_read_window() {
+      exec_window_ = exec_window::read;
+   }
+
+   void set_to_write_window() {
+      exec_window_ = exec_window::write;
+   }
+
+   bool is_read_window() const {
+      return exec_window_ == exec_window::read;
+   }
+
+   bool is_write_window() const {
+      return exec_window_ == exec_window::write;
+   }
+
+   auto& read_only_trx_safe_queue() { return read_only_trx_safe_queue_; }
+
+   auto& general_queue() { return general_queue_; }
+
+   // members are ordered taking into account that the last one is destructed first
+private:
+   boost::asio::io_service            io_serv_;
+   appbase::execution_priority_queue  read_only_trx_safe_queue_;
+   appbase::execution_priority_queue  general_queue_;
+   std::atomic<std::size_t>           order_ { std::numeric_limits<size_t>::max() }; // to maintain FIFO ordering in both queues within priority
+   exec_window                        exec_window_ { exec_window::write };
+};
+
+using application = application_t<two_queue_executor>;
+}
+
+#include <appbase/application_instance.hpp>

libraries/custom_appbase/tests/CMakeLists.txt

@@ -0,0 +1,13 @@
+if(CMAKE_CXX_STANDARD EQUAL 98 OR CMAKE_CXX_STANDARD LESS 17)
+   message(FATAL_ERROR "appbase requires c++17 or newer")
+elseif(NOT CMAKE_CXX_STANDARD)
+   set(CMAKE_CXX_STANDARD 17)
+   set(CMAKE_CXX_STANDARD_REQUIRED ON)
+endif()
+
+file(GLOB UNIT_TESTS "*.cpp")
+add_executable( custom_appbase_test ${UNIT_TESTS} )
+target_link_libraries( custom_appbase_test appbase ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} )
+target_include_directories( custom_appbase_test PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include" "${CMAKE_CURRENT_SOURCE_DIR}/../../appbase/include" )
+
+add_test( custom_appbase_test custom_appbase_test )

libraries/custom_appbase/tests/custom_appbase_tests.cpp

@@ -0,0 +1,215 @@
+#define BOOST_TEST_MODULE custom_appbase_tests
+#include <boost/test/included/unit_test.hpp>
+#include <thread>
+#include <iostream>
+
+#include <eosio/chain/application.hpp>
+
+using namespace appbase;
+
+BOOST_AUTO_TEST_SUITE(custom_appbase_tests)
+
+std::thread start_app_thread(appbase::scoped_app& app) {
+   const char* argv[] = { boost::unit_test::framework::current_test_case().p_name->c_str() };
+   BOOST_CHECK(app->initialize(sizeof(argv) / sizeof(char*), const_cast<char**>(argv)));
+   app->startup();
+   std::thread app_thread( [&]() {
+      app->exec();
+   } );
+   return app_thread;
+}
+
+// verify functions from both queues are executed when execution window is not explictly set
+BOOST_AUTO_TEST_CASE( default_exec_window ) {
+   appbase::scoped_app app;
+   auto app_thread = start_app_thread(app);
+   
+   // post functions
+   std::map<int, int> rslts {};
+   int seq_num = 0;
+   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+
+   // Stop app. Use the lowest priority to make sure this function to execute the last
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_only_trx_safe_queue should only contain the current lambda function,
+      // and general_queue should have execute all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+      app->quit();
+      } );
+   app_thread.join();
+
+   // both queues are cleared after execution 
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+
+   // exactly number of both queues' functions processed
+   BOOST_REQUIRE_EQUAL( rslts.size(), 8 );
+
+   // same priority of functions executed by the post order
+   BOOST_CHECK_LT( rslts[0], rslts[1] );  // medium
+   BOOST_CHECK_LT( rslts[2], rslts[3] );  // high
+   BOOST_CHECK_LT( rslts[3], rslts[7] );  // high
+   BOOST_CHECK_LT( rslts[4], rslts[5] );  // low
+
+   // higher priority posted earlier executed earlier
+   BOOST_CHECK_LT( rslts[3], rslts[4] );
+   BOOST_CHECK_LT( rslts[6], rslts[7] );
+}
+
+// verify functions only from read_only_trx_safe queue are processed during read window
+BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
+   appbase::scoped_app app;
+   auto app_thread = start_app_thread(app);
+   
+   // set to run functions from read_only_trx_safe queue only
+   app->executor().set_to_read_window();
+
+   // post functions
+   std::map<int, int> rslts {};
+   int seq_num = 0;
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+
+   // stop application. Use lowest at the end to make sure this executes the last
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_queue should have current function and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 4 );
+      app->quit();
+      } );
+   app_thread.join();
+
+   // both queues are cleared after execution 
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+
+   // exactly number of posts processed
+   BOOST_REQUIRE_EQUAL( rslts.size(), 6 );
+
+   // same priority (high) of functions in read_queue executed by the post order
+   BOOST_CHECK_LT( rslts[1], rslts[3] );
+
+   // higher priority posted earlier in read_queue executed earlier
+   BOOST_CHECK_LT( rslts[3], rslts[4] );
+}
+
+// verify no functions are executed during  read window if read_only_trx_safe queue is empty
+BOOST_AUTO_TEST_CASE( execute_from_empty_read_queue ) {
+   appbase::scoped_app app;
+   auto app_thread = start_app_thread(app);
+   
+   // set to run functions from read_only_trx_safe queue only
+   app->executor().set_to_read_window();
+
+   // post functions
+   std::map<int, int> rslts {};
+   int seq_num = 0;
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+
+   // Stop application. Use lowest at the end to make sure this executes the last
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_queue should have current function and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 10 );
+      app->quit();
+      } );
+   app_thread.join();
+
+   // both queues are cleared after execution 
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+
+   // no results
+   BOOST_REQUIRE_EQUAL( rslts.size(), 0 );
+}
+
+// verify functions from both queues are processed in write window
+BOOST_AUTO_TEST_CASE( execute_from_both_queues ) {
+   appbase::scoped_app app;
+   auto app_thread = start_app_thread(app);
+
+   // set to run functions from both queues
+   app->executor().is_write_window();
+
+   // post functions
+   std::map<int, int> rslts {};
+   int seq_num = 0;
+   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[10]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[11]=seq_num; ++seq_num; } );
+
+   // stop application. Use lowest at the end to make sure this executes the last
+   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+      // read_queue should have current function and write_queue's functions are all executed 
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
+      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+      app->quit();
+      } );
+
+   app_thread.join();
+
+   // queues are emptied after quit
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+
+   // exactly number of posts processed
+   BOOST_REQUIRE_EQUAL( rslts.size(), 12 );
+
+   // all low must be processed the in order of posting
+   BOOST_CHECK_LT( rslts[4], rslts[5] );
+   BOOST_CHECK_LT( rslts[5], rslts[7] );
+   BOOST_CHECK_LT( rslts[7], rslts[10] );
+
+   // all medium must be processed the in order of posting
+   BOOST_CHECK_LT( rslts[0], rslts[1] );
+   BOOST_CHECK_LT( rslts[1], rslts[11] );
+
+   // all functions posted after high before highest must be processed after high
+   BOOST_CHECK_LT( rslts[2], rslts[3] );
+   BOOST_CHECK_LT( rslts[2], rslts[4] );
+   BOOST_CHECK_LT( rslts[2], rslts[5] );
+
+   // all functions posted after highest must be processed after it
+   BOOST_CHECK_LT( rslts[6], rslts[7] );
+   BOOST_CHECK_LT( rslts[6], rslts[8] );
+   BOOST_CHECK_LT( rslts[6], rslts[9] );
+   BOOST_CHECK_LT( rslts[6], rslts[10] );
+   BOOST_CHECK_LT( rslts[6], rslts[11] );
+   BOOST_CHECK_LT( rslts[6], rslts[11] );
+}
+
+BOOST_AUTO_TEST_SUITE_END()

libraries/libfc/CMakeLists.txt

@@ -140,7 +140,7 @@ if(APPLE)
   find_library(corefoundation_framework CoreFoundation)
 endif()
 target_link_libraries( fc PUBLIC Boost::date_time Boost::filesystem Boost::chrono Boost::iostreams Threads::Threads
-                                 OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB ${PLATFORM_SPECIFIC_LIBS} ${CMAKE_DL_LIBS} secp256k1 ${security_framework} ${corefoundation_framework})
+                                 OpenSSL::Crypto ZLIB::ZLIB ${PLATFORM_SPECIFIC_LIBS} ${CMAKE_DL_LIBS} secp256k1 ${security_framework} ${corefoundation_framework})
 
 # Critically, this ensures that OpenSSL 1.1 & 3.0 both have a variant of BN_zero() with void return value. But it also allows access
 #  to some obsoleted AES functions in 3.0 too, since 3.0's API_COMPAT is effectively 3.0 by default

libraries/libfc/include/fc/crypto/sha256.hpp

@@ -22,6 +22,10 @@ class sha256
     char*       data();
     size_t      data_size() const { return 256 / 8; }
 
+    bool empty()const {
+       return (_hash[0] | _hash[1] | _hash[2] | _hash[3]) == 0;
+    }
+
     static sha256 hash( const char* d, uint32_t dlen );
     static sha256 hash( const string& );
     static sha256 hash( const sha256& );

libraries/libfc/include/fc/io/cfile.hpp

@@ -190,8 +190,8 @@ public:
 
    bool eof() const { return feof(_file.get()) != 0; }
 
-   int getc() { 
-      int ret = fgetc(_file.get());  
+   int getc() {
+      int ret = fgetc(_file.get());
       if (ret == EOF) {
          throw std::ios_base::failure( "cfile: " + _file_path.generic_string() +
                                        " unable to read 1 byte");
@@ -210,6 +210,15 @@ public:
 
    cfile_datastream create_datastream();
 
+
+   friend void swap(cfile& lhs, cfile& rhs) {
+      using std::swap;
+      swap(lhs._open, rhs._open);
+      swap(lhs._file_path, rhs._file_path);
+      swap(lhs._file_blk_size, rhs._file_blk_size);
+      swap(lhs._file, rhs._file);
+   }
+
 private:
    bool                  _open = false;
    fc::path              _file_path;
@@ -264,6 +273,11 @@ class datastream<fc::cfile, void> : public fc::cfile {
 
    fc::cfile&       storage() { return *this; }
    const fc::cfile& storage() const { return *this; }
+
+   friend void swap(datastream<fc::cfile, void>& lhs, datastream<fc::cfile, void>& rhs) {
+      using std::swap;
+      swap(static_cast<fc::cfile&>(lhs), static_cast<fc::cfile&>(rhs));
+   }
 };
 
 

libraries/libfc/include/fc/io/datastream.hpp

@@ -97,10 +97,12 @@ class datastream<size_t, void> {
 };
 
 template <typename Streambuf>
-class datastream<Streambuf, typename std::enable_if_t<std::is_base_of_v<std::streambuf, Streambuf>>> {
+class datastream<Streambuf, typename std::enable_if_t<std::is_base_of_v<std::streambuf, std::remove_reference_t<Streambuf>>>> {
  private:
    Streambuf buf;
 
+   using reference_type = std::add_lvalue_reference_t<Streambuf>;
+
  public:
    template <typename... Args>
    datastream(Args&&... args)
@@ -120,8 +122,8 @@ class datastream<Streambuf, typename std::enable_if_t<std::is_base_of_v<std::str
    }
    bool remaining() { return buf.in_avail(); }
 
-   Streambuf&       storage() { return buf; }
-   const Streambuf& storage() const { return buf; }
+   reference_type       storage() { return buf; }
+   const reference_type storage() const { return buf; }
 };
 
 template <typename Container>

libraries/libfc/include/fc/io/json.hpp

@@ -35,7 +35,6 @@ namespace fc
          static constexpr uint64_t max_length_limit = std::numeric_limits<uint64_t>::max();
          static constexpr size_t escape_string_yield_check_count = 128;
          static variant  from_string( const string& utf8_str, const parse_type ptype = parse_type::legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH );
-         static variants variants_from_string( const string& utf8_str, const parse_type ptype = parse_type::legacy_parser, uint32_t max_depth = DEFAULT_MAX_RECURSION_DEPTH );
          static string   to_string( const variant& v, const yield_function_t& yield, const output_formatting format = output_formatting::stringify_large_ints_and_doubles);
          static string   to_pretty_string( const variant& v, const yield_function_t& yield, const output_formatting format = output_formatting::stringify_large_ints_and_doubles );
 

libraries/libfc/include/fc/log/appender.hpp

@@ -36,7 +36,7 @@ namespace fc {
       public:
          typedef std::shared_ptr<appender> ptr;
 
-         virtual void initialize( boost::asio::io_service& io_service ) = 0;
+         virtual void initialize() = 0;
          virtual void log( const log_message& m ) = 0;
    };
 }

libraries/libfc/include/fc/log/console_appender.hpp

@@ -3,12 +3,12 @@
 #include <fc/log/logger.hpp>
 #include <vector>
 
-namespace fc 
+namespace fc
 {
-   class console_appender final : public appender 
+   class console_appender final : public appender
    {
        public:
-            struct color 
+            struct color
             {
                 enum type {
                    red,
@@ -24,9 +24,9 @@ namespace fc
 
             struct stream { enum type { std_out, std_error }; };
 
-            struct level_color 
+            struct level_color
             {
-               level_color( log_level l=log_level::all, 
+               level_color( log_level l=log_level::all,
                             color::type c=color::console_default )
                :level(l),color(c){}
 
@@ -34,7 +34,7 @@ namespace fc
                console_appender::color::type     color;
             };
 
-            struct config 
+            struct config
             {
                config()
                :format( "${timestamp} ${thread_name} ${context} ${file}:${line} ${method} ${level}]  ${message}" ),
@@ -52,10 +52,10 @@ namespace fc
             console_appender();
 
             ~console_appender();
-            void initialize( boost::asio::io_service& io_service ) {}
-            virtual void log( const log_message& m );
-            
-            void print( const std::string& text_to_print, 
+            void initialize() override {}
+            virtual void log( const log_message& m ) override;
+
+            void print( const std::string& text_to_print,
                         color::type text_color = color::console_default );
 
             void configure( const config& cfg );

libraries/libfc/include/fc/log/dmlog_appender.hpp

@@ -20,7 +20,7 @@ namespace fc {
             explicit dmlog_appender( const std::optional<config>& args) ;
 
             virtual ~dmlog_appender();
-            virtual void initialize( boost::asio::io_service& io_service ) override;
+            virtual void initialize() override;
 
             virtual void log( const log_message& m ) override;
 

libraries/libfc/include/fc/log/gelf_appender.hpp

@@ -6,14 +6,14 @@
 
 #include <regex>
 
-namespace fc 
+namespace fc
 {
   // Log appender that sends log messages in JSON format over UDP
   // https://www.graylog2.org/resources/gelf/specification
-  class gelf_appender final : public appender 
+  class gelf_appender final : public appender
   {
   public:
-    struct config 
+    struct config
     {
       static const std::vector<std::string> reserved_field_names;
       static const std::regex user_field_name_pattern;
@@ -31,11 +31,11 @@ namespace fc
      * In a single-threaded world with a boost::io_service that's not owned
      * by this library, ugly things are required.  Tough.
      */
-    void initialize(boost::asio::io_service& io_service) override;
+    void initialize() override;
     virtual void log(const log_message& m) override;
 
-  private:
     class impl;
+  private:
     std::shared_ptr<impl> my;
   };
 } // namespace fc

libraries/libfc/include/fc/log/logger.hpp

@@ -1,7 +1,7 @@
 #pragma once
-#include <fc/string.hpp>
 #include <fc/time.hpp>
 #include <fc/log/log_message.hpp>
+#include <string>
 
 #ifndef DEFAULT_LOGGER
 #define DEFAULT_LOGGER "default"
@@ -25,11 +25,11 @@ namespace fc
    class logger 
    {
       public:
-         static logger get( const fc::string& name = DEFAULT_LOGGER );
-         static void update( const fc::string& name, logger& log );
+         static logger get( const std::string& name = DEFAULT_LOGGER );
+         static void update( const std::string& name, logger& log );
 
          logger();
-         logger( const string& name, const logger& parent = nullptr );
+         logger( const std::string& name, const logger& parent = nullptr );
          logger( std::nullptr_t );
          logger( const logger& c );
          logger( logger&& c );
@@ -44,8 +44,8 @@ namespace fc
          logger&    set_parent( const logger& l );
          logger     get_parent()const;
 
-         void  set_name( const fc::string& n );
-         const fc::string& name()const;
+         void  set_name( const std::string& n );
+         std::string get_name()const;
 
          bool is_enabled( log_level e )const;
          void log( log_message m );
@@ -74,6 +74,12 @@ namespace fc
 # define FC_MULTILINE_MACRO_END  } while (0)
 #endif
 
+#define fc_tlog( LOGGER, FORMAT, ... ) \
+  FC_MULTILINE_MACRO_BEGIN \
+   if( (LOGGER).is_enabled( fc::log_level::all ) ) \
+      (LOGGER).log( FC_LOG_MESSAGE( all, FORMAT, __VA_ARGS__ ) ); \
+  FC_MULTILINE_MACRO_END
+
 #define fc_dlog( LOGGER, FORMAT, ... ) \
   FC_MULTILINE_MACRO_BEGIN \
    if( (LOGGER).is_enabled( fc::log_level::debug ) ) \
@@ -98,6 +104,12 @@ namespace fc
       (LOGGER).log( FC_LOG_MESSAGE( error, FORMAT, __VA_ARGS__ ) ); \
   FC_MULTILINE_MACRO_END
 
+#define tlog( FORMAT, ... ) \
+  FC_MULTILINE_MACRO_BEGIN \
+   if( (fc::logger::get(DEFAULT_LOGGER)).is_enabled( fc::log_level::all ) ) \
+      (fc::logger::get(DEFAULT_LOGGER)).log( FC_LOG_MESSAGE( all, FORMAT, __VA_ARGS__ ) ); \
+  FC_MULTILINE_MACRO_END
+
 #define dlog( FORMAT, ... ) \
   FC_MULTILINE_MACRO_BEGIN \
    if( (fc::logger::get(DEFAULT_LOGGER)).is_enabled( fc::log_level::debug ) ) \

libraries/libfc/include/fc/log/logger_config.hpp

@@ -54,7 +54,7 @@ namespace fc {
       static logger get_logger( const fc::string& name );
       static void update_logger( const fc::string& name, logger& log );
 
-      static void initialize_appenders( boost::asio::io_service& ios );
+      static void initialize_appenders();
 
       static bool configure_logging( const logging_config& l );
 

libraries/libfc/src/io/json.cpp

@@ -9,6 +9,8 @@
 #include <sstream>
 
 #include <boost/filesystem/fstream.hpp>
+#include <boost/iostreams/device/array.hpp>
+#include <boost/iostreams/stream.hpp>
 
 namespace fc
 {
@@ -88,7 +90,7 @@ namespace fc
    template<typename T>
    std::string stringFromStream( T& in )
    {
-      std::stringstream token;
+      std::string token;
       try
       {
          char c = in.peek();
@@ -103,28 +105,29 @@ namespace fc
             switch( c = in.peek() )
             {
                case '\\':
-                  token << parseEscape( in );
+                  token += parseEscape( in );
                   break;
                case 0x04:
                   FC_THROW_EXCEPTION( parse_error_exception, "EOF before closing '\"' in string '${token}'",
-                                                   ("token", token.str() ) );
+                                                   ("token", token ) );
                case '"':
                   in.get();
-                  return token.str();
+                  return token;
                default:
-                  token << c;
+                  token += c;
                   in.get();
             }
          }
          FC_THROW_EXCEPTION( parse_error_exception, "EOF before closing '\"' in string '${token}'",
-                                          ("token", token.str() ) );
+                                          ("token", token ) );
        } FC_RETHROW_EXCEPTIONS( warn, "while parsing token '${token}'",
-                                          ("token", token.str() ) );
+                                          ("token", token ) );
    }
+
    template<typename T>
    std::string stringFromToken( T& in )
    {
-      std::stringstream token;
+      std::string token;
       try
       {
          char c = in.peek();
@@ -134,37 +137,36 @@ namespace fc
             switch( c = in.peek() )
             {
                case '\\':
-                  token << parseEscape( in );
+                  token += parseEscape( in );
                   break;
                case '\t':
                case ' ':
                case '\n':
                   in.get();
-                  return token.str();
+                  return token;
                case '\0':
                   FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" );
                default:
                 if( isalnum( c ) || c == '_' || c == '-' || c == '.' || c == ':' || c == '/' )
                 {
-                  token << c;
+                  token += c;
                   in.get();
                 }
-                else return token.str();
+                else return token;
             }
          }
-         return token.str();
+         return token;
       }
       catch( const fc::eof_exception& eof )
       {
-         return token.str();
+         return token;
       }
       catch (const std::ios_base::failure&)
       {
-         return token.str();
+         return token;
       }
 
-      FC_RETHROW_EXCEPTIONS( warn, "while parsing token '${token}'",
-                                          ("token", token.str() ) );
+      FC_RETHROW_EXCEPTIONS( warn, "while parsing token '${token}'", ("token", token) );
    }
 
    template<typename T, json::parse_type parser_type>
@@ -252,14 +254,14 @@ namespace fc
    template<typename T, json::parse_type parser_type>
    variant number_from_stream( T& in )
    {
-      std::stringstream ss;
+      std::string s;
 
       bool  dot = false;
       bool  neg = false;
       if( in.peek() == '-')
       {
         neg = true;
-        ss.put( in.get() );
+        s += in.get();
       }
       bool done = false;
 
@@ -284,14 +286,15 @@ namespace fc
               case '7':
               case '8':
               case '9':
-                 ss.put( in.get() );
+                 s += in.get();
                  break;
               case '\0':
                  FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" );
               default:
                  if( isalnum( c ) )
                  {
-                    return ss.str() + stringFromToken( in );
+                    s += stringFromToken( in );
+                    return s;
                  }
                 done = true;
                 break;
@@ -304,7 +307,7 @@ namespace fc
       catch (const std::ios_base::failure&)
       {
       }
-      std::string str = ss.str();
+      const std::string& str = s;
       if (str == "-." || str == "." || str == "-") // check the obviously wrong things we could have encountered
         FC_THROW_EXCEPTION(parse_error_exception, "Can't parse token \"${token}\" as a JSON numeric constant", ("token", str));
       if( dot )
@@ -313,11 +316,11 @@ namespace fc
         return to_int64(str);
       return to_uint64(str);
    }
+
    template<typename T>
    variant token_from_stream( T& in )
    {
-      std::stringstream ss;
-      ss.exceptions( std::ifstream::badbit );
+      std::string s;
       bool received_eof = false;
       bool done = false;
 
@@ -337,7 +340,7 @@ namespace fc
               case 'f':
               case 'a':
               case 's':
-                 ss.put( in.get() );
+                 s += in.get();
                  break;
               default:
                  done = true;
@@ -356,7 +359,7 @@ namespace fc
 
       // we can get here either by processing a delimiter as in "null,"
       // an EOF like "null<EOF>", or an invalid token like "nullZ"
-      std::string str = ss.str();
+      const std::string& str = s;
       if( str == "null" )
         return variant();
       if( str == "true" )
@@ -441,50 +444,23 @@ namespace fc
 
    variant json::from_string( const std::string& utf8_str, const json::parse_type ptype, const uint32_t max_depth )
    { try {
-      std::stringstream in( utf8_str );
-      //in.exceptions( std::ifstream::eofbit );
+      using stream_t = boost::iostreams::stream<boost::iostreams::array_source>;
+      stream_t in(utf8_str.c_str(), utf8_str.size());
       switch( ptype )
       {
           case parse_type::legacy_parser:
-             return variant_from_stream<std::stringstream, json::parse_type::legacy_parser>( in, max_depth );
+             return variant_from_stream<stream_t, json::parse_type::legacy_parser>( in, max_depth );
           case parse_type::legacy_parser_with_string_doubles:
-              return variant_from_stream<std::stringstream, json::parse_type::legacy_parser_with_string_doubles>( in, max_depth );
+              return variant_from_stream<stream_t, json::parse_type::legacy_parser_with_string_doubles>( in, max_depth );
           case parse_type::strict_parser:
-              return json_relaxed::variant_from_stream<std::stringstream, true>( in, max_depth );
+              return json_relaxed::variant_from_stream<stream_t, true>( in, max_depth );
           case parse_type::relaxed_parser:
-              return json_relaxed::variant_from_stream<std::stringstream, false>( in, max_depth );
+              return json_relaxed::variant_from_stream<stream_t, false>( in, max_depth );
           default:
               FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", static_cast<int>(ptype)) );
       }
    } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) }
 
-   variants json::variants_from_string( const std::string& utf8_str, const json::parse_type ptype, const uint32_t max_depth )
-   { try {
-      variants result;
-      std::stringstream in( utf8_str );
-      //in.exceptions( std::ifstream::eofbit );
-      try {
-         while( true )
-         {
-           // result.push_back( variant_from_stream( in ));
-           result.push_back(json_relaxed::variant_from_stream<std::stringstream, false>( in, max_depth ));
-         }
-      } catch ( const fc::eof_exception& ){}
-      return result;
-   } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) }
-   /*
-   void toUTF8( const char str, std::ostream& os )
-   {
-      // validate str == valid utf8
-      utf8::replace_invalid( &str, &str + 1, std::ostream_iterator<char>(os) );
-   }
-
-   void toUTF8( const wchar_t c, std::ostream& os )
-   {
-      utf8::utf16to8( &c, (&c)+1, std::ostream_iterator<char>(os) );
-   }
-   */
-
    /**
     *  Convert '\t', '\r', '\n', '\\' and '"'  to "\t\r\n\\\"" if escape_control_chars == true
     *  Convert all other < 32 & 127 ascii to escaped unicode "\u00xx"

libraries/libfc/src/log/dmlog_appender.cpp

@@ -56,7 +56,7 @@ namespace fc {
       }
    }
 
-   void dmlog_appender::initialize( boost::asio::io_service& io_service ) {}
+   void dmlog_appender::initialize() {}
 
    void dmlog_appender::log( const log_message& m ) {
       FILE* out = my->out;

libraries/libfc/src/log/gelf_appender.cpp

@@ -8,6 +8,7 @@
 #include <fc/io/json.hpp>
 #include <fc/crypto/city.hpp>
 #include <fc/compress/zlib.hpp>
+#include <fc/log/logger_config.hpp>
 
 #include <boost/lexical_cast.hpp>
 #include <iomanip>
@@ -42,10 +43,15 @@ namespace fc
   class gelf_appender::impl
   {
   public:
+    using work_guard_t = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
     config                                         cfg;
     std::optional<boost::asio::ip::udp::endpoint>  gelf_endpoint;
+    std::thread                                    thread;
+    boost::asio::io_context                        io_context;
+    work_guard_t work_guard =                      boost::asio::make_work_guard(io_context);
     udp_socket                                     gelf_socket;
 
+
     impl(const variant& c)
     {
       mutable_variant_object mvo;
@@ -73,6 +79,10 @@ namespace fc
 
     ~impl()
     {
+       if (thread.joinable()) {
+          work_guard.reset();
+          thread.join();
+       }
     }
   };
 
@@ -81,7 +91,7 @@ namespace fc
   {
   }
 
-  void gelf_appender::initialize(boost::asio::io_service &io_service)
+  void gelf_appender::initialize()
   {
     try
     {
@@ -100,14 +110,19 @@ namespace fc
         string::size_type colon_pos = my->cfg.endpoint.find(':');
         try
         {
-          uint16_t port = boost::lexical_cast<uint16_t>(my->cfg.endpoint.substr(colon_pos + 1, my->cfg.endpoint.size()));
+          FC_ASSERT(colon_pos != std::string::npos, "The logging destination port is not specified");
+          string port = my->cfg.endpoint.substr(colon_pos + 1);
 
           string hostname = my->cfg.endpoint.substr( 0, colon_pos );
-          auto endpoints = resolve(io_service, hostname, port);
+
+          boost::asio::ip::udp::resolver resolver{ my->io_context };
+          auto endpoints = resolver.resolve(hostname, port);
+
           if (endpoints.empty())
               FC_THROW_EXCEPTION(unknown_host_exception, "The logging destination host name can not be resolved: ${hostname}",
                                  ("hostname", hostname));
-          my->gelf_endpoint = endpoints.back();
+
+          my->gelf_endpoint = *endpoints.begin();
         }
         catch (const boost::bad_lexical_cast&)
         {
@@ -117,9 +132,20 @@ namespace fc
 
       if (my->gelf_endpoint)
       {
-        my->gelf_socket.initialize(io_service);
+        my->gelf_socket.initialize(my->io_context);
         my->gelf_socket.open();
         std::cerr << "opened GELF socket to endpoint " << my->cfg.endpoint << "\n";
+
+        my->thread = std::thread([this] {
+          try {
+            fc::set_os_thread_name("gelf");
+            my->io_context.run();
+          } catch (std::exception& ex) {
+            fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
+          } catch (...) {
+            fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
+          }
+        });
       }
     }
     catch (...)
@@ -131,11 +157,8 @@ namespace fc
   gelf_appender::~gelf_appender()
   {}
 
-  void gelf_appender::log(const log_message& message)
+  void do_log(gelf_appender::impl* my, uint64_t time_ns, const log_message& message)
   {
-    if (!my->gelf_endpoint)
-      return;
-
     log_context context = message.get_context();
 
     mutable_variant_object gelf_message;
@@ -143,8 +166,6 @@ namespace fc
     gelf_message["host"] = my->cfg.host;
     gelf_message["short_message"] = format_string(message.get_format(), message.get_data(), true);
 
-    // use now() instead of context.get_timestamp() because log_message construction can include user provided long running calls
-    const auto time_ns = time_point::now().time_since_epoch().count();
     gelf_message["timestamp"] = time_ns / 1000000.;
     gelf_message["_timestamp_ns"] = time_ns;
 
@@ -248,4 +269,20 @@ namespace fc
       FC_ASSERT(number_of_packets_sent == total_number_of_packets);
     }
   }
+
+  void gelf_appender::log(const log_message& message) {
+    if (!my->gelf_endpoint)
+      return;
+
+    // use now() instead of context.get_timestamp() because log_message construction can include user provided long running calls
+    boost::asio::post(my->io_context, [impl = my.get(), time_ns = time_point::now().time_since_epoch().count(), message] () {
+      try {
+        do_log(impl, time_ns, message);
+      } catch (std::exception& ex) {
+        fprintf(stderr, "GELF logger caught exception at %s:%d : %s\n", __FILE__, __LINE__, ex.what());
+      } catch (...) {
+        fprintf(stderr, "GELF logger caught exception unknown exception %s:%d\n", __FILE__, __LINE__);
+      }
+    });
+  }
 } // fc

libraries/libfc/src/log/logger.cpp

@@ -3,9 +3,8 @@
 #include <fc/log/appender.hpp>
 #include <fc/exception/exception.hpp>
 #include <fc/filesystem.hpp>
-#include <unordered_map>
-#include <string>
 #include <fc/log/logger_config.hpp>
+#include <unordered_map>
 
 namespace fc {
 
@@ -13,7 +12,7 @@ namespace fc {
       public:
          impl()
          :_parent(nullptr),_enabled(true),_additivity(false),_level(log_level::warn){}
-         fc::string       _name;
+         std::string      _name;
          logger           _parent;
          bool             _enabled;
          bool             _additivity;
@@ -28,7 +27,7 @@ namespace fc {
 
     logger::logger(nullptr_t){}
 
-    logger::logger( const string& name, const logger& parent )
+    logger::logger( const std::string& name, const logger& parent )
     :my( new impl() )
     {
        my->_name = name;
@@ -40,7 +39,7 @@ namespace fc {
     :my(l.my){}
 
     logger::logger( logger&& l )
-    :my(fc::move(l.my)){}
+    :my(std::move(l.my)){}
 
     logger::~logger(){}
 
@@ -82,14 +81,14 @@ namespace fc {
        }
     }
 
-    void logger::set_name( const fc::string& n ) { my->_name = n; }
-    const fc::string& logger::name()const { return my->_name; }
+    void logger::set_name( const std::string& n ) { my->_name = n; }
+    std::string logger::get_name()const { return my->_name; }
 
-    logger logger::get( const fc::string& s ) {
+    logger logger::get( const std::string& s ) {
        return log_config::get_logger( s );
     }
 
-    void logger::update( const fc::string& name, logger& log ) {
+    void logger::update( const std::string& name, logger& log ) {
        log_config::update_logger( name, log );
     }
 

libraries/libfc/src/log/logger_config.cpp

@@ -43,10 +43,10 @@ namespace fc {
       }
    }
 
-   void log_config::initialize_appenders( boost::asio::io_service& ios ) {
+   void log_config::initialize_appenders() {
       std::lock_guard g( log_config::get().log_mutex );
       for( auto& iter : log_config::get().appender_map )
-         iter.second->initialize( ios );
+         iter.second->initialize();
    }
 
    void configure_logging( const fc::path& lc ) {

libraries/libfc/src/network/http/http_client.cpp

@@ -8,15 +8,12 @@
 #include <boost/beast/version.hpp>
 #include <boost/asio/connect.hpp>
 #include <boost/asio/ip/tcp.hpp>
-#include <boost/asio/ssl/error.hpp>
-#include <boost/asio/ssl/stream.hpp>
+#include <boost/asio/deadline_timer.hpp>
 #include <boost/asio/local/stream_protocol.hpp>
-#include <boost/asio/ssl/rfc2818_verification.hpp>
 #include <boost/filesystem.hpp>
 
 using tcp = boost::asio::ip::tcp;       // from <boost/asio/ip/tcp.hpp>
 namespace http = boost::beast::http;    // from <boost/beast/http.hpp>
-namespace ssl = boost::asio::ssl;       // from <boost/asio/ssl.hpp>
 namespace local = boost::asio::local;
 
 namespace fc {
@@ -25,17 +22,15 @@ namespace fc {
  * mapping of protocols to their standard ports
  */
 static const std::map<string,uint16_t> default_proto_ports = {
-   {"http", 80},
-   {"https", 443}
+   {"http", 80}
 };
 
 class http_client_impl {
 public:
    using host_key = std::tuple<std::string, std::string, uint16_t>;
    using raw_socket_ptr = std::unique_ptr<tcp::socket>;
-   using ssl_socket_ptr = std::unique_ptr<ssl::stream<tcp::socket>>;
    using unix_socket_ptr = std::unique_ptr<local::stream_protocol::socket>;
-   using connection = std::variant<raw_socket_ptr, ssl_socket_ptr, unix_socket_ptr>;
+   using connection = std::variant<raw_socket_ptr, unix_socket_ptr>;
    using connection_map = std::map<host_key, connection>;
    using unix_url_split_map = std::map<string, fc::url>;
    using error_code = boost::system::error_code;
@@ -43,24 +38,13 @@ public:
 
    http_client_impl()
    :_ioc()
-   ,_sslc(ssl::context::sslv23_client)
    {
       set_verify_peers(true);
    }
 
-   void add_cert(const std::string& cert_pem_string) {
-      error_code ec;
-      _sslc.add_certificate_authority(boost::asio::buffer(cert_pem_string.data(), cert_pem_string.size()), ec);
-      FC_ASSERT(!ec, "Failed to add cert: ${msg}", ("msg", ec.message()));
-   }
+   void add_cert(const std::string& cert_pem_string) {}
 
-   void set_verify_peers(bool enabled) {
-      if (enabled) {
-         _sslc.set_verify_mode(ssl::verify_peer);
-      } else {
-         _sslc.set_verify_mode(ssl::verify_none);
-      }
-   }
+   void set_verify_peers(bool enabled) {}
 
    template<typename SyncReadStream, typename Fn, typename CancelFn>
    error_code sync_do_with_deadline( SyncReadStream& s, deadline_type deadline, Fn f, CancelFn cf ) {
@@ -189,41 +173,9 @@ public:
       return res.first;
    }
 
-   connection_map::iterator create_ssl_connection( const url& dest, const deadline_type& deadline ) {
-      auto key = url_to_host_key(dest);
-      auto ssl_socket = std::make_unique<ssl::stream<tcp::socket>>(_ioc, _sslc);
-
-      // Set SNI Hostname (many hosts need this to handshake successfully)
-      if(!SSL_set_tlsext_host_name(ssl_socket->native_handle(), dest.host()->c_str()))
-      {
-         error_code ec{static_cast<int>(::ERR_get_error()), boost::asio::error::get_ssl_category()};
-         FC_THROW("Unable to set SNI Host Name: ${msg}", ("msg", ec.message()));
-      }
-
-      ssl_socket->set_verify_callback(boost::asio::ssl::rfc2818_verification(*dest.host()));
-
-      error_code ec = sync_connect_with_timeout(ssl_socket->next_layer(), *dest.host(), dest.port() ? std::to_string(*dest.port()) : "443", deadline);
-      if (!ec) {
-         ec = sync_do_with_deadline(ssl_socket->next_layer(), deadline, [&ssl_socket](std::optional<error_code>& final_ec) {
-            ssl_socket->async_handshake(ssl::stream_base::client, [&final_ec](const error_code& ec) {
-               final_ec.emplace(ec);
-            });
-         });
-      }
-      FC_ASSERT(!ec, "Failed to connect: ${message}", ("message",ec.message()));
-
-      auto res = _connections.emplace(std::piecewise_construct,
-                                      std::forward_as_tuple(key),
-                                      std::forward_as_tuple(std::move(ssl_socket)));
-
-      return res.first;
-   }
-
    connection_map::iterator create_connection( const url& dest, const deadline_type& deadline ) {
       if (dest.proto() == "http") {
          return create_raw_connection(dest, deadline);
-      } else if (dest.proto() == "https") {
-         return create_ssl_connection(dest, deadline);
       } else if (dest.proto() == "unix") {
          return create_unix_connection(dest, deadline);
       } else {
@@ -236,10 +188,6 @@ public:
          return !ptr->is_open();
       }
 
-      bool operator() ( const ssl_socket_ptr& ptr ) const {
-         return !ptr->lowest_layer().is_open();
-      }
-
       bool operator() ( const unix_socket_ptr& ptr) const {
          return !ptr->is_open();
       }
@@ -420,7 +368,6 @@ public:
    }
 
    boost::asio::io_context  _ioc;
-   ssl::context             _sslc;
    connection_map           _connections;
    unix_url_split_map       _unix_url_paths;
 };

libraries/prometheus/CMakeLists.txt

@@ -0,0 +1,20 @@
+add_library(prometheus-core
+        prometheus-cpp/core/src/check_names.cc
+        prometheus-cpp/core/src/counter.cc
+        prometheus-cpp/core/src/detail/builder.cc
+        prometheus-cpp/core/src/detail/ckms_quantiles.cc
+        prometheus-cpp/core/src/detail/time_window_quantiles.cc
+        prometheus-cpp/core/src/detail/utils.cc
+        prometheus-cpp/core/src/family.cc
+        prometheus-cpp/core/src/gauge.cc
+        prometheus-cpp/core/src/histogram.cc
+        prometheus-cpp/core/src/info.cc
+        prometheus-cpp/core/src/registry.cc
+        prometheus-cpp/core/src/serializer.cc
+        prometheus-cpp/core/src/summary.cc
+        prometheus-cpp/core/src/text_serializer.cc
+        )
+
+target_include_directories( prometheus-core PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}/include
+        ${CMAKE_CURRENT_SOURCE_DIR}/prometheus-cpp/core/include)

libraries/prometheus/include/prometheus/detail/core_export.h

@@ -0,0 +1,10 @@
+#pragma once
+
+// For prometheus-cpp library, this normally a generated file.  we aren't following their process, so
+// we have it here.  The macros are used in prometheus-cpp in relation to shared libraries, which isn't
+// relevant for us.
+
+#define PROMETHEUS_CPP_CORE_EXPORT
+#define PROMETHEUS_CPP_CORE_NO_EXPORT
+
+

libraries/state_history/compression.cpp

@@ -18,15 +18,17 @@ bytes zlib_compress_bytes(const bytes& in) {
    return out;
 }
 
-bytes zlib_decompress(const bytes& in) {
+bytes zlib_decompress(std::string_view data) {
    bytes                  out;
    bio::filtering_ostream decomp;
    decomp.push(bio::zlib_decompressor());
    decomp.push(bio::back_inserter(out));
-   bio::write(decomp, in.data(), in.size());
+   bio::write(decomp, data.data(), data.size());
    bio::close(decomp);
    return out;
 }
 
+
+
 } // namespace state_history
 } // namespace eosio

libraries/state_history/create_deltas.cpp

@@ -57,8 +57,10 @@ bool include_delta(const chain::protocol_state_object& old, const chain::protoco
    return old.activated_protocol_features != curr.activated_protocol_features;
 }
 
-std::vector<table_delta> create_deltas(const chainbase::database& db, bool full_snapshot) {
-   std::vector<table_delta>                          deltas;
+void pack_deltas(boost::iostreams::filtering_ostreambuf& obuf, const chainbase::database& db, bool full_snapshot) {
+
+   fc::datastream<boost::iostreams::filtering_ostreambuf&> ds{obuf};
+
    const auto&                                       table_id_index = db.get_index<chain::table_id_multi_index>();
    std::map<uint64_t, const chain::table_id_object*> removed_table_id;
    for (auto& rem : table_id_index.last_undo_session().removed_values)
@@ -73,72 +75,118 @@ std::vector<table_delta> create_deltas(const chainbase::database& db, bool full_
       return *it->second;
    };
 
-   auto pack_row          = [&](auto& row) { return fc::raw::pack(make_history_serial_wrapper(db, row)); };
-   auto pack_contract_row = [&](auto& row) {
-      return fc::raw::pack(make_history_context_wrapper(db, get_table_id(row.t_id._id), row));
+   auto pack_row          = [&](auto& ds, auto& row) { fc::raw::pack(ds, make_history_serial_wrapper(db, row)); };
+   auto pack_contract_row = [&](auto& ds, auto& row) {
+      fc::raw::pack(ds, make_history_context_wrapper(db, get_table_id(row.t_id._id), row));
    };
 
-   auto process_table = [&](auto* name, auto& index, auto& pack_row) {
+   auto process_table = [&](auto& ds, auto* name, auto& index, auto& pack_row) {
+
+      auto pack_row_v0 = [&](auto& ds, bool present, auto& row) {
+         fc::raw::pack(ds, present);
+         fc::datastream<size_t> ps;
+         pack_row(ps, row);
+         fc::raw::pack(ds, fc::unsigned_int(ps.tellp()));
+         pack_row(ds, row);
+      };
+
       if (full_snapshot) {
          if (index.indices().empty())
             return;
-         deltas.push_back({});
-         auto& delta = deltas.back();
-         delta.name  = name;
-         for (auto& row : index.indices())
-            delta.rows.obj.emplace_back(true, pack_row(row));
+
+         fc::raw::pack(ds, fc::unsigned_int(0)); // table_delta = std::variant<table_delta_v0> and fc::unsigned_int struct_version
+         fc::raw::pack(ds, name);
+         fc::raw::pack(ds, fc::unsigned_int(index.indices().size()));
+         for (auto& row : index.indices()) {
+            pack_row_v0(ds, true, row);
+         }
       } else {
          auto undo = index.last_undo_session();
-         if (undo.old_values.empty() && undo.new_values.empty() && undo.removed_values.empty())
-            return;
-         deltas.push_back({});
-         auto& delta = deltas.back();
-         delta.name  = name;
-         for (auto& old : undo.old_values) {
-            auto& row = index.get(old.id);
-            if (include_delta(old, row))
-               delta.rows.obj.emplace_back(true, pack_row(row));
-         }
-         for (auto& old : undo.removed_values)
-            delta.rows.obj.emplace_back(false, pack_row(old));
-         for (auto& row : undo.new_values) {
-            delta.rows.obj.emplace_back(true, pack_row(row));
-         }
 
-         if(delta.rows.obj.empty()) {
-            deltas.pop_back();
+         size_t num_entries =
+             std::count_if(undo.old_values.begin(), undo.old_values.end(),
+                           [&index](const auto& old) { return include_delta(old, index.get(old.id)); }) +
+             std::distance(undo.removed_values.begin(), undo.removed_values.end()) +
+             std::distance(undo.new_values.begin(), undo.new_values.end());
+
+         if (num_entries) {
+            fc::raw::pack(ds, fc::unsigned_int(0)); // table_delta = std::variant<table_delta_v0> and fc::unsigned_int struct_version
+            fc::raw::pack(ds, name);
+            fc::raw::pack(ds, fc::unsigned_int((uint32_t)num_entries));
+
+            for (auto& old : undo.old_values) {
+               auto& row = index.get(old.id);
+               if (include_delta(old, row)) {
+                  pack_row_v0(ds, true, row);
+               }
+            }
+
+            for (auto& old : undo.removed_values) {
+               pack_row_v0(ds, false, old);
+            }
+
+            for (auto& row : undo.new_values) {
+               pack_row_v0(ds, true, row);
+            }
          }
       }
    };
 
-   process_table("account", db.get_index<chain::account_index>(), pack_row);
-   process_table("account_metadata", db.get_index<chain::account_metadata_index>(), pack_row);
-   process_table("code", db.get_index<chain::code_index>(), pack_row);
+   auto has_table = [&](auto x) -> int {
+      auto& index = db.get_index<std::remove_pointer_t<decltype(x)>>();
+      if (full_snapshot) {
+         return !index.indices().empty();
+      } else {
+         auto undo = index.last_undo_session();
+         return std::find_if(undo.old_values.begin(), undo.old_values.end(),
+                           [&index](const auto& old) { return include_delta(old, index.get(old.id)); }) != undo.old_values.end() ||
+             !undo.removed_values.empty() || !undo.new_values.empty();
+      }
+   };
 
-   process_table("contract_table", db.get_index<chain::table_id_multi_index>(), pack_row);
-   process_table("contract_row", db.get_index<chain::key_value_index>(), pack_contract_row);
-   process_table("contract_index64", db.get_index<chain::index64_index>(), pack_contract_row);
-   process_table("contract_index128", db.get_index<chain::index128_index>(), pack_contract_row);
-   process_table("contract_index256", db.get_index<chain::index256_index>(), pack_contract_row);
-   process_table("contract_index_double", db.get_index<chain::index_double_index>(), pack_contract_row);
-   process_table("contract_index_long_double", db.get_index<chain::index_long_double_index>(), pack_contract_row);
+   int num_tables = std::apply(
+       [&has_table](auto... args) { return (has_table(args) + ... ); },
+       std::tuple<chain::account_index*, chain::account_metadata_index*, chain::code_index*,
+                  chain::table_id_multi_index*, chain::key_value_index*, chain::index64_index*, chain::index128_index*,
+                  chain::index256_index*, chain::index_double_index*, chain::index_long_double_index*,
+                  chain::global_property_multi_index*, chain::generated_transaction_multi_index*,
+                  chain::protocol_state_multi_index*, chain::permission_index*, chain::permission_link_index*,
+                  chain::resource_limits::resource_limits_index*, chain::resource_limits::resource_usage_index*,
+                  chain::resource_limits::resource_limits_state_index*,
+                  chain::resource_limits::resource_limits_config_index*>());
 
-   process_table("global_property", db.get_index<chain::global_property_multi_index>(), pack_row);
-   process_table("generated_transaction", db.get_index<chain::generated_transaction_multi_index>(), pack_row);
-   process_table("protocol_state", db.get_index<chain::protocol_state_multi_index>(), pack_row);
+   fc::raw::pack(ds, fc::unsigned_int(num_tables));
 
-   process_table("permission", db.get_index<chain::permission_index>(), pack_row);
-   process_table("permission_link", db.get_index<chain::permission_link_index>(), pack_row);
+   process_table(ds, "account", db.get_index<chain::account_index>(), pack_row);
+   process_table(ds, "account_metadata", db.get_index<chain::account_metadata_index>(), pack_row);
+   process_table(ds, "code", db.get_index<chain::code_index>(), pack_row);
 
-   process_table("resource_limits", db.get_index<chain::resource_limits::resource_limits_index>(), pack_row);
-   process_table("resource_usage", db.get_index<chain::resource_limits::resource_usage_index>(), pack_row);
-   process_table("resource_limits_state", db.get_index<chain::resource_limits::resource_limits_state_index>(),
+   process_table(ds, "contract_table", db.get_index<chain::table_id_multi_index>(), pack_row);
+   process_table(ds, "contract_row", db.get_index<chain::key_value_index>(), pack_contract_row);
+   process_table(ds, "contract_index64", db.get_index<chain::index64_index>(), pack_contract_row);
+   process_table(ds, "contract_index128", db.get_index<chain::index128_index>(), pack_contract_row);
+   process_table(ds, "contract_index256", db.get_index<chain::index256_index>(), pack_contract_row);
+   process_table(ds, "contract_index_double", db.get_index<chain::index_double_index>(), pack_contract_row);
+   process_table(ds, "contract_index_long_double", db.get_index<chain::index_long_double_index>(), pack_contract_row);
+
+   process_table(ds, "global_property", db.get_index<chain::global_property_multi_index>(), pack_row);
+   process_table(ds, "generated_transaction", db.get_index<chain::generated_transaction_multi_index>(), pack_row);
+   process_table(ds, "protocol_state", db.get_index<chain::protocol_state_multi_index>(), pack_row);
+
+   process_table(ds, "permission", db.get_index<chain::permission_index>(), pack_row);
+   process_table(ds, "permission_link", db.get_index<chain::permission_link_index>(), pack_row);
+
+   process_table(ds, "resource_limits", db.get_index<chain::resource_limits::resource_limits_index>(), pack_row);
+   process_table(ds, "resource_usage", db.get_index<chain::resource_limits::resource_usage_index>(), pack_row);
+   process_table(ds, "resource_limits_state", db.get_index<chain::resource_limits::resource_limits_state_index>(),
                  pack_row);
-   process_table("resource_limits_config", db.get_index<chain::resource_limits::resource_limits_config_index>(),
+   process_table(ds, "resource_limits_config", db.get_index<chain::resource_limits::resource_limits_config_index>(),
                  pack_row);
 
-   return deltas;
+   obuf.pubsync();
+
 }
 
+
 } // namespace state_history
 } // namespace eosio

libraries/state_history/include/eosio/state_history/compression.hpp

@@ -8,7 +8,7 @@ namespace state_history {
 using chain::bytes;
 
 bytes zlib_compress_bytes(const bytes& in);
-bytes zlib_decompress(const bytes& in);
+bytes zlib_decompress(std::string_view);
 
 } // namespace state_history
 } // namespace eosio

libraries/state_history/include/eosio/state_history/create_deltas.hpp

@@ -1,11 +1,13 @@
 #pragma once
 
 #include <eosio/state_history/types.hpp>
+#include <boost/iostreams/filtering_streambuf.hpp>
 
 namespace eosio {
 namespace state_history {
 
-std::vector<table_delta> create_deltas(const chainbase::database& db, bool full_snapshot);
+void pack_deltas(boost::iostreams::filtering_ostreambuf& ds, const chainbase::database& db, bool full_snapshot);
+
 
 } // namespace state_history
 } // namespace eosio

libraries/state_history/include/eosio/state_history/log.hpp

@@ -1,19 +1,34 @@
 #pragma once
 
-#include <boost/filesystem.hpp>
-#include <fstream>
-#include <stdint.h>
-
-#include <boost/asio.hpp>
-
+#include <eosio/state_history/compression.hpp>
 #include <eosio/chain/block_header.hpp>
 #include <eosio/chain/exceptions.hpp>
 #include <eosio/chain/types.hpp>
+#include <eosio/chain/log_catalog.hpp>
+#include <eosio/chain/log_data_base.hpp>
+#include <eosio/chain/log_index.hpp>
+
 #include <fc/io/cfile.hpp>
 #include <fc/log/logger.hpp>
 #include <fc/log/logger_config.hpp> //set_thread_name
+#include <fc/bitutil.hpp>
+
+#include <boost/asio.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/iostreams/device/file.hpp>
+#include <boost/iostreams/device/file_descriptor.hpp>
+#include <boost/iostreams/filter/zlib.hpp>
+#include <boost/iostreams/filtering_streambuf.hpp>
+#include <boost/iostreams/restrict.hpp>
+
+#include <fstream>
+#include <cstdint>
+
+
+struct state_history_test_fixture;
 
 namespace eosio {
+namespace bio = boost::iostreams;
 
 /*
  *   *.log:
@@ -62,18 +77,216 @@ struct state_history_log_header {
 static const int state_history_log_header_serial_size = sizeof(state_history_log_header::magic) +
                                                         sizeof(state_history_log_header::block_id) +
                                                         sizeof(state_history_log_header::payload_size);
-struct state_history_log_prune_config {
-   uint32_t                prune_blocks;                  //number of blocks to prune to when doing a prune
-   size_t                  prune_threshold = 4*1024*1024; //(approximately) how many bytes need to be added before a prune is performed
-   std::optional<size_t>   vacuum_on_close;               //when set, a vacuum is performed on dtor if log contains less than this many bytes
+
+namespace state_history {
+   struct prune_config {
+      uint32_t                prune_blocks;                  //number of blocks to prune to when doing a prune
+      size_t                  prune_threshold = 4*1024*1024; //(approximately) how many bytes need to be added before a prune is performed
+      std::optional<size_t>   vacuum_on_close;               //when set, a vacuum is performed on dtor if log contains less than this many bytes
+   };
+
+   struct partition_config {
+      fc::path retained_dir       = "retained";
+      fc::path archive_dir        = "archive";
+      uint32_t stride             = 1000000;
+      uint32_t max_retained_files = 10;
+   };
+} // namespace state_history
+
+using state_history_log_config = std::variant<std::monostate, state_history::prune_config, state_history::partition_config>;
+
+struct locked_decompress_stream {
+   std::unique_lock<std::mutex> lock; // state_history_log mutex
+   std::variant<std::vector<char>, std::unique_ptr<bio::filtering_istreambuf>> buf;
+
+   locked_decompress_stream() = delete;
+   locked_decompress_stream(locked_decompress_stream&&) = default;
+
+   explicit locked_decompress_stream(std::unique_lock<std::mutex> l)
+   : lock(std::move(l)) {};
+
+   template <typename StateHistoryLog>
+   void init(StateHistoryLog&& log, fc::cfile& stream, uint64_t compressed_size) {
+      auto istream = std::make_unique<bio::filtering_istreambuf>();
+      istream->push(bio::zlib_decompressor());
+      istream->push(bio::restrict(bio::file_source(stream.get_file_path().string()), stream.tellp(), compressed_size));
+      buf = std::move(istream);
+   }
+
+   template <typename LogData>
+   void init(LogData&& log, fc::datastream<const char*>& stream, uint64_t compressed_size) {
+      auto istream = std::make_unique<bio::filtering_istreambuf>();
+      istream->push(bio::zlib_decompressor());
+      istream->push(bio::restrict(bio::file_source(log.filename), stream.pos() - log.data(), compressed_size));
+      buf = std::move(istream);
+   }
+
+   size_t init(std::vector<char> cbuf) {
+      buf.emplace<std::vector<char>>( std::move(cbuf) );
+      return std::get<std::vector<char>>(buf).size();
+   }
 };
 
+namespace detail {
+
+std::vector<char> zlib_decompress(fc::cfile& file, uint64_t compressed_size) {
+   if (compressed_size) {
+      std::vector<char> compressed(compressed_size);
+      file.read(compressed.data(), compressed_size);
+      return state_history::zlib_decompress({compressed.data(), compressed_size});
+   }
+   return {};
+}
+
+std::vector<char> zlib_decompress(fc::datastream<const char*>& strm, uint64_t compressed_size) {
+   if (compressed_size) {
+      return state_history::zlib_decompress({strm.pos(), compressed_size});
+   }
+   return {};
+}
+
+template <typename Log, typename Stream>
+uint64_t read_unpacked_entry(Log&& log, Stream& stream, uint64_t payload_size, locked_decompress_stream& result) {
+   // result has state_history_log mutex locked
+
+   uint32_t s;
+   stream.read((char*)&s, sizeof(s));
+   if (s == 1 && payload_size > (s + sizeof(uint32_t))) {
+      uint64_t compressed_size = payload_size - sizeof(uint32_t) - sizeof(uint64_t);
+      uint64_t decompressed_size;
+      stream.read((char*)&decompressed_size, sizeof(decompressed_size));
+      result.init(log, stream, compressed_size);
+      return decompressed_size;
+   } else {
+      // Compressed deltas now exceeds 4GB on one of the public chains. This length prefix
+      // was intended to support adding additional fields in the future after the
+      // packed deltas or packed traces. For now we're going to ignore on read.
+
+      uint64_t compressed_size = payload_size - sizeof(uint32_t);
+      return result.init( zlib_decompress(stream, compressed_size) );
+   }
+}
+
+class state_history_log_data : public chain::log_data_base<state_history_log_data> {
+   uint32_t version_;
+   bool     is_currently_pruned_;
+   uint64_t size_;
+
+ public:
+   state_history_log_data() = default;
+   explicit state_history_log_data(const fc::path& path) { open(path); }
+
+   void open(const fc::path& path) {
+      if (file.is_open())
+         file.close();
+      file.set_file_path(path);
+      file.open("rb");
+      uint64_t v           = chain::read_data_at<uint64_t>(file, 0);
+      version_             = get_ship_version(v);
+      is_currently_pruned_ = is_ship_log_pruned(v);
+      file.seek_end(0);
+      size_ = file.tellp();
+   }
+
+   uint64_t size() const { return size_; }
+   uint32_t version() const { return version_; }
+   uint32_t first_block_num() { return block_num_at(0); }
+   uint32_t first_block_position() const { return 0; }
+
+   bool is_currently_pruned() const { return is_currently_pruned_; }
+
+   uint64_t ro_stream_at(uint64_t pos, locked_decompress_stream& result) {
+      uint64_t                    payload_size = payload_size_at(pos);
+      file.seek(pos + sizeof(state_history_log_header));
+      // fc::datastream<const char*> stream(file.const_data() + pos + sizeof(state_history_log_header), payload_size);
+      return read_unpacked_entry(*this, file, payload_size, result);
+   }
+
+   uint32_t block_num_at(uint64_t position) {
+      return fc::endian_reverse_u32(
+          chain::read_data_at<uint32_t>(file, position + offsetof(state_history_log_header, block_id)));
+   }
+
+   chain::block_id_type block_id_at(uint64_t position) {
+      return chain::read_data_at<chain::block_id_type>(file, position +
+                                                      offsetof(state_history_log_header, block_id));
+   }
+
+   uint64_t payload_size_at(uint64_t pos) {
+      std::string filename = file.get_file_path().generic_string();
+      EOS_ASSERT(size() >= pos + sizeof(state_history_log_header), chain::plugin_exception,
+                 "corrupt ${name}: invalid entry size at at position ${pos}", ("name", filename)("pos", pos));
+
+      state_history_log_header header = chain::read_data_at<state_history_log_header>(file, pos);
+
+      EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic), chain::plugin_exception,
+                 "corrupt ${name}: invalid header for entry at position ${pos}", ("name", filename)("pos", pos));
+
+      EOS_ASSERT(size() >= pos + sizeof(state_history_log_header) + header.payload_size, chain::plugin_exception,
+                 "corrupt ${name}: invalid payload size for entry at position ${pos}", ("name", filename)("pos", pos));
+      return header.payload_size;
+   }
+
+   void construct_index(const fc::path& index_file_name) {
+      fc::cfile index_file;
+      index_file.set_file_path(index_file_name);
+      index_file.open("w+b");
+
+      uint64_t pos = 0;
+      while (pos < size()) {
+         uint64_t payload_size = payload_size_at(pos);
+         index_file.write(reinterpret_cast<const char*>(&pos), sizeof(pos));
+         pos += (sizeof(state_history_log_header) + payload_size + sizeof(uint64_t));
+      }
+   }
+};
+
+// directly adapt from boost/iostreams/filter/counter.hpp and change the type of chars_ to uint64_t.
+class counter  {
+public:
+    typedef char char_type;
+    struct category
+        : bio::dual_use,
+          bio::filter_tag,
+          bio::multichar_tag,
+          bio::optimally_buffered_tag
+        { };
+    explicit counter(uint64_t first_char = 0)
+        : chars_(first_char)
+        { }
+    uint64_t characters() const { return chars_; }
+    std::streamsize optimal_buffer_size() const { return 0; }
+
+    template<typename Source>
+    std::streamsize read(Source& src, char_type* s, std::streamsize n)
+    {
+        std::streamsize result = bio::read(src, s, n);
+        if (result == -1)
+            return -1;
+        chars_ += result;
+        return result;
+    }
+
+    template<typename Sink>
+    std::streamsize write(Sink& snk, const char_type* s, std::streamsize n)
+    {
+        std::streamsize result = bio::write(snk, s, n);
+        chars_ += result;
+        return result;
+    }
+private:
+    uint64_t chars_;
+};
+
+} // namespace detail
+
 class state_history_log {
  private:
    const char* const       name = "";
-   std::string             log_filename;
-   std::string             index_filename;
-   std::optional<state_history_log_prune_config> prune_config; //is set, log is in pruned mode
+   state_history_log_config config;
+
+   // provide exclusive access to all data of this object since accessed from the main thread and the ship thread
+   mutable std::mutex      _mx;
    fc::cfile               log;
    fc::cfile               index;
    uint32_t                _begin_block = 0;        //always tracks the first block available even after pruning
@@ -81,31 +294,40 @@ class state_history_log {
    uint32_t                _end_block   = 0;
    chain::block_id_type    last_block_id;
 
-   std::thread                                                              thr;
-   std::atomic<bool>                                                        write_thread_has_exception = false;
-   std::exception_ptr                                                       eptr;
-   boost::asio::io_context                                                  ctx;
-   boost::asio::io_context::strand                                          work_strand{ctx};
-   boost::asio::executor_work_guard<boost::asio::io_context::executor_type> work_guard =
-       boost::asio::make_work_guard(ctx);
-   std::recursive_mutex                                                     mx;
+   using catalog_t = chain::log_catalog<detail::state_history_log_data, chain::log_index<chain::plugin_exception>>;
+   catalog_t catalog;
 
  public:
-   state_history_log(const char* const name, std::string log_filename, std::string index_filename,
-                     std::optional<state_history_log_prune_config> prune_conf = std::optional<state_history_log_prune_config>())
+   friend struct ::state_history_test_fixture;
+
+   state_history_log( const state_history_log&) = delete;
+
+   state_history_log(const char* name, const fc::path& log_dir,
+                     state_history_log_config conf = {})
        : name(name)
-       , log_filename(std::move(log_filename))
-       , index_filename(std::move(index_filename))
-       , prune_config(prune_conf) {
+       , config(std::move(conf)) {
+
+      log.set_file_path(log_dir/(std::string(name) + ".log"));
+      index.set_file_path(log_dir/(std::string(name) + ".index"));
+
       open_log();
       open_index();
-     
-      if(prune_config) {
-         EOS_ASSERT(prune_config->prune_blocks, chain::plugin_exception, "state history log prune configuration requires at least one block");
-         EOS_ASSERT(__builtin_popcount(prune_config->prune_threshold) == 1, chain::plugin_exception, "state history prune threshold must be power of 2");
-         //switch this over to the mask that will be used
-         prune_config->prune_threshold = ~(prune_config->prune_threshold-1);
-      }
+
+      std::visit(eosio::chain::overloaded{
+         [](std::monostate&) {},
+         [](state_history::prune_config& conf) {
+            EOS_ASSERT(conf.prune_blocks, chain::plugin_exception, "state history log prune configuration requires at least one block");
+            EOS_ASSERT(__builtin_popcount(conf.prune_threshold) == 1, chain::plugin_exception, "state history prune threshold must be power of 2");
+            //switch this over to the mask that will be used
+            conf.prune_threshold = ~(conf.prune_threshold-1);
+         }, [name, log_dir, this](state_history::partition_config& conf) {
+            catalog.open(log_dir, conf.retained_dir, conf.archive_dir, name);
+            catalog.max_retained_files = conf.max_retained_files;
+            if (_end_block == 0) {
+               _begin_block = _end_block = catalog.last_block_num() +1;
+            }
+         }
+      }, config);
 
       //check for conversions to/from pruned log, as long as log contains something
       if(_begin_block != _end_block) {
@@ -113,6 +335,8 @@ class state_history_log {
          log.seek(0);
          read_header(first_header);
 
+         auto prune_config = std::get_if<state_history::prune_config>(&config);
+
          if((is_ship_log_pruned(first_header.magic) == false) && prune_config) {
             //need to convert non-pruned to pruned; first prune any ranges we can (might be none)
             prune(fc::log_level::info);
@@ -131,36 +355,14 @@ class state_history_log {
             vacuum();
          }
       }
-
-       thr = std::thread([this] {
-         try {
-            fc::set_os_thread_name(this->name);
-            this->ctx.run();
-         } catch (...) {
-            elog("catched exception from ${name} write thread", ("name", this->name));
-            eptr                       = std::current_exception();
-            write_thread_has_exception = true;
-         }
-      });
-   }
-
-   void stop() {
-      if (thr.joinable()) {
-         work_guard.reset();
-         thr.join();
-      }
    }
 
    ~state_history_log() {
-      // complete execution before possible vacuuming
-      if (thr.joinable()) {
-         work_guard.reset();
-         thr.join();
-      }     
-
       //nothing to do if log is empty or we aren't pruning
       if(_begin_block == _end_block)
          return;
+
+      auto prune_config = std::get_if<state_history::prune_config>(&config);
       if(!prune_config || !prune_config->vacuum_on_close)
          return;
 
@@ -170,8 +372,96 @@ class state_history_log {
          vacuum();
    }
 
-   uint32_t begin_block() const { return _begin_block; }
-   uint32_t end_block() const { return _end_block; }
+   //        begin     end
+   std::pair<uint32_t, uint32_t> block_range() const {
+      std::lock_guard g(_mx);
+      return { std::min(catalog.first_block_num(), _begin_block), _end_block };
+   }
+
+   bool empty() const {
+      auto r = block_range();
+      return r.first == r.second;
+   }
+
+   locked_decompress_stream create_locked_decompress_stream() {
+      return locked_decompress_stream{ std::unique_lock<std::mutex>( _mx ) };
+   }
+
+   /// @return the decompressed entry size
+   uint64_t get_unpacked_entry(uint32_t block_num, locked_decompress_stream& result) {
+
+      // result has mx locked
+
+      auto opt_decompressed_size = catalog.ro_stream_for_block(block_num, result);
+      if (opt_decompressed_size)
+         return *opt_decompressed_size;
+
+      if (block_num < _begin_block || block_num >= _end_block)
+         return 0;
+
+      state_history_log_header header;
+      log.seek(get_pos(block_num));
+      read_header(header);
+
+      return detail::read_unpacked_entry(*this, log, header.payload_size, result);
+   }
+
+   template <typename F>
+   void pack_and_write_entry(state_history_log_header header, const chain::block_id_type& prev_id, F&& pack_to) {
+      std::lock_guard g(_mx);
+      write_entry(header, prev_id, [&, pack_to = std::forward<F>(pack_to)](auto& stream) {
+         size_t payload_pos = stream.tellp();
+
+         // In order to conserve memory usage for reading the chain state later, we need to
+         // encode the uncompressed data size to the disk so that the reader can send the
+         // decompressed data size before decompressing data. Here we use the number
+         // 1 indicates the format contains a 64 bits unsigned integer for decompressed data
+         // size and then the actually compressed data. The compressed data size can be
+         // computed from the payload size in the header minus sizeof(uint32_t) + sizeof(uint64_t).
+
+         uint32_t s = 1;
+         stream.write((char*)&s, sizeof(s));
+         uint64_t uncompressioned_size = 0;
+         stream.skip(sizeof(uncompressioned_size));
+
+         namespace bio = boost::iostreams;
+
+         detail::counter cnt;
+         {
+            bio::filtering_ostreambuf buf;
+            buf.push(boost::ref(cnt));
+            buf.push(bio::zlib_compressor());
+            buf.push(bio::file_descriptor_sink(stream.fileno(), bio::never_close_handle));
+            pack_to(buf);
+         }
+
+         // calculate the payload size and rewind back to header to write the payload size
+         stream.seek_end(0);
+         size_t   end_payload_pos = stream.tellp();
+         uint64_t payload_size    = end_payload_pos - payload_pos;
+         stream.seek(payload_pos - sizeof(uint64_t));
+         stream.write((char*)&payload_size, sizeof(payload_size));
+
+         // write the uncompressed data size
+         stream.skip(sizeof(s));
+         uncompressioned_size = cnt.characters();
+         stream.write((char*)&uncompressioned_size, sizeof(uncompressioned_size));
+
+         // make sure we reset the file position to end_payload_pos to preserve API behavior
+         stream.seek(end_payload_pos);
+      });
+   }
+
+   std::optional<chain::block_id_type> get_block_id(uint32_t block_num) {
+      std::lock_guard g(_mx);
+      return get_block_id_i(block_num);
+   }
+
+#ifdef BOOST_TEST_MODULE
+   fc::cfile& get_log_file() { return log;}
+#endif
+
+ private:
 
    void read_header(state_history_log_header& header, bool assert_version = true) {
       char bytes[state_history_log_header_serial_size];
@@ -194,12 +484,6 @@ class state_history_log {
 
    template <typename F>
    void write_entry(state_history_log_header header, const chain::block_id_type& prev_id, F write_payload) {
-      if (write_thread_has_exception) {
-         std::rethrow_exception(eptr);
-      }
-
-      std::unique_lock<std::recursive_mutex> lock(mx);
-      
       auto block_num = chain::block_header::num_from_id(header.block_id);
       EOS_ASSERT(_begin_block == _end_block || block_num <= _end_block, chain::plugin_exception,
                  "missed a block in ${name}.log", ("name", name));
@@ -216,24 +500,40 @@ class state_history_log {
          }
       }
 
-      if (block_num < _end_block)
-         truncate(block_num); //truncate is expected to always leave file pointer at the end
-      else if (!prune_config)
+      auto prune_config = std::get_if<state_history::prune_config>(&config);
+      if (block_num < _end_block) {
+         // This is typically because of a fork, and we need to truncate the log back to the beginning of the fork.
+         static uint32_t start_block_num = block_num;
+         // Guard agaisnt accidently starting a fresh chain with an existing ship log, require manual removal of ship logs.
+         EOS_ASSERT( block_num > 2, chain::plugin_exception, "Existing ship log with ${eb} blocks when starting from genesis block ${b}",
+                     ("eb", _end_block)("b", block_num) );
+         // block_num < _begin_block = pruned log, need to call truncate() to reset
+         // get_block_id_i check is an optimization to avoid writing a block that is already in the log (snapshot or replay)
+         if ( block_num < _begin_block || get_block_id_i(block_num) != header.block_id ) {
+            truncate(block_num); //truncate is expected to always leave file pointer at the end
+         } else {
+            if (start_block_num == block_num || block_num % 1000 == 0 )
+               ilog("log ${name}.log already contains block ${b}, end block ${eb}", ("name", name)("b", block_num)("eb", _end_block));
+            return;
+         }
+      } else if (!prune_config) {
          log.seek_end(0);
-      else if (prune_config && _begin_block != _end_block)
+      } else if (prune_config && _begin_block != _end_block) {
          log.seek_end(-sizeof(uint32_t));  //overwrite the trailing block count marker on this write
+      }
 
       //if we're operating on a pruned block log and this is the first entry in the log, make note of the feature in the header
       if(prune_config && _begin_block == _end_block)
          header.magic = ship_magic(get_ship_version(header.magic), ship_feature_pruned_log);
 
       uint64_t pos = log.tellp();
-            
+
       write_header(header);
       write_payload(log);
 
-      EOS_ASSERT(log.tellp() == pos + state_history_log_header_serial_size + header.payload_size, chain::plugin_exception,
-                 "wrote payload with incorrect size to ${name}.log", ("name", name));
+      if (header.payload_size != 0)
+         EOS_ASSERT(log.tellp() == pos + state_history_log_header_serial_size + header.payload_size, chain::plugin_exception,
+                    "wrote payload with incorrect size to ${name}.log", ("name", name));
       fc::raw::pack(log, pos);
 
       fc::raw::pack(index, pos);
@@ -249,28 +549,36 @@ class state_history_log {
          const uint32_t num_blocks_in_log = _end_block - _begin_block;
          fc::raw::pack(log, num_blocks_in_log);
       }
+
+      auto partition_config = std::get_if<state_history::partition_config>(&config);
+      if (partition_config && block_num % partition_config->stride == 0) {
+         split_log();
+      }
    }
 
-   // returns cfile positioned at payload
    fc::cfile& get_entry(uint32_t block_num, state_history_log_header& header) {
       EOS_ASSERT(block_num >= _begin_block && block_num < _end_block, chain::plugin_exception,
                  "read non-existing block in ${name}.log", ("name", name));
-      std::lock_guard lock(mx);
       log.seek(get_pos(block_num));
       read_header(header);
       return log;
    }
 
-   chain::block_id_type get_block_id(uint32_t block_num) {
-      std::lock_guard          lock(mx);
-      state_history_log_header header;
-      get_entry(block_num, header);
-      return header.block_id;
+   std::optional<chain::block_id_type> get_block_id_i(uint32_t block_num) {
+      auto result = catalog.id_for_block(block_num);
+      if (!result) {
+         if (block_num >= _begin_block && block_num < _end_block) {
+            state_history_log_header header;
+            get_entry(block_num, header);
+            return header.block_id;
+         }
+         return {};
+      }
+      return result;
    }
 
- private:
    //file position must be at start of last block's suffix (back pointer)
-   //called from open_log / ctor 
+   //called from open_log / ctor
    bool get_last_block() {
       state_history_log_header header;
       uint64_t                 suffix;
@@ -298,6 +606,8 @@ class state_history_log {
    }
 
    void prune(const fc::log_level& loglevel) {
+      auto prune_config = std::get_if<state_history::prune_config>(&config);
+
       if(!prune_config)
          return;
       if(_end_block - _begin_block <= prune_config->prune_blocks)
@@ -347,7 +657,7 @@ class state_history_log {
          }
       }
       log.flush();
-      boost::filesystem::resize_file(log_filename, pos);
+      boost::filesystem::resize_file(log.get_file_path().string(), pos);
       log.flush();
 
       log.seek_end(-sizeof(pos));
@@ -356,13 +666,12 @@ class state_history_log {
 
    // only called from constructor
    void open_log() {
-      log.set_file_path(log_filename);
-      log.open("a+b");
+      log.open(fc::cfile::create_or_update_rw_mode);
       log.seek_end(0);
       uint64_t size = log.tellp();
       log.close();
 
-      log.open("r+b");
+      log.open(fc::cfile::update_rw_mode);
       if (size >= state_history_log_header_serial_size) {
          state_history_log_header header;
          log.seek(0);
@@ -403,8 +712,7 @@ class state_history_log {
 
    // only called from constructor
    void open_index() {
-      index.set_file_path(index_filename);
-      index.open("a+b");
+      index.open(fc::cfile::create_or_update_rw_mode);
       index.seek_end(0);
       if (index.tellp() == (static_cast<int>(_end_block) - _index_begin_block) * sizeof(uint64_t))
          return;
@@ -444,7 +752,7 @@ class state_history_log {
       }
 
       index.close();
-      index.open("a+b");
+      index.open(fc::cfile::create_or_update_rw_mode);
    }
 
    uint64_t get_pos(uint32_t block_num) {
@@ -455,27 +763,48 @@ class state_history_log {
    }
 
    void truncate(uint32_t block_num) {
-      log.flush();
-      index.flush();
-      uint64_t num_removed = 0;
+      log.close();
+      index.close();
+
+      auto first_block_num     = catalog.empty() ? _begin_block : catalog.first_block_num();
+      auto new_begin_block_num = catalog.truncate(block_num, log.get_file_path());
+
+      // notice that catalog.truncate() can replace existing log and index files, so we have to
+      // close the files and reopen them again; otherwise we might operate on the obsolete files instead.
+
+      if (new_begin_block_num > 0) {
+         _begin_block = new_begin_block_num;
+         _index_begin_block = new_begin_block_num;
+      }
+
+      uint32_t num_removed;
+
       if (block_num <= _begin_block) {
-         num_removed = _end_block - _begin_block;
-         log.seek(0);
-         boost::filesystem::resize_file(log_filename, 0);
-         boost::filesystem::resize_file(index_filename, 0);
-         _begin_block = _end_block = 0;
+         num_removed = _end_block - first_block_num;
+         boost::filesystem::resize_file(log.get_file_path().string(), 0);
+         boost::filesystem::resize_file(index.get_file_path().string(), 0);
+         _begin_block = _end_block = block_num;
       } else {
          num_removed  = _end_block - block_num;
+
+         index.open("rb");
          uint64_t pos = get_pos(block_num);
-         log.seek(0);
-         boost::filesystem::resize_file(log_filename, pos);
-         boost::filesystem::resize_file(index_filename, (block_num - _index_begin_block) * sizeof(uint64_t));
+         index.close();
+
+         auto path = log.get_file_path().string();
+
+         boost::filesystem::resize_file(log.get_file_path().string(), pos);
+         boost::filesystem::resize_file(index.get_file_path().string(), (block_num - _index_begin_block) * sizeof(uint64_t));
          _end_block = block_num;
          //this will leave the end of the log with the last block's suffix no matter if the log is operating in pruned
          // mode or not. The assumption is truncate() is always immediately followed up with an append to the log thus
          // restoring the prune trailer if required
       }
+
+      log.open(fc::cfile::update_rw_mode);
       log.seek_end(0);
+      index.open(fc::cfile::create_or_update_rw_mode);
+
       ilog("fork or replay: removed ${n} blocks from ${name}.log", ("n", num_removed)("name", name));
    }
 
@@ -557,6 +886,50 @@ class state_history_log {
       _index_begin_block = _begin_block;
       ilog("Vacuum of pruned log ${n} complete",("n", name));
    }
+
+   void split_log() {
+
+      fc::path log_file_path = log.get_file_path();
+      fc::path index_file_path = index.get_file_path();
+
+      fc::datastream<fc::cfile>  new_log_file;
+      fc::datastream<fc::cfile> new_index_file;
+
+      fc::path tmp_log_file_path = log_file_path;
+      tmp_log_file_path.replace_extension("log.tmp");
+      fc::path tmp_index_file_path = index_file_path;
+      tmp_index_file_path.replace_extension("index.tmp");
+
+      new_log_file.set_file_path(tmp_log_file_path);
+      new_index_file.set_file_path(tmp_index_file_path);
+
+      try {
+         new_log_file.open(fc::cfile::truncate_rw_mode);
+         new_index_file.open(fc::cfile::truncate_rw_mode);
+
+      } catch (...) {
+         wlog("Unable to open new state history log or index file for writing during log spliting, "
+              "continue writing to existing block log file\n");
+         return;
+      }
+
+      index.close();
+      log.close();
+
+      catalog.add(_begin_block, _end_block - 1, log.get_file_path().parent_path(), name);
+
+      _begin_block = _end_block;
+
+      using std::swap;
+      swap(new_log_file, log);
+      swap(new_index_file, index);
+
+      fc::rename(tmp_log_file_path, log_file_path);
+      fc::rename(tmp_index_file_path, index_file_path);
+
+      log.set_file_path(log_file_path);
+      index.set_file_path(index_file_path);
+   }
 }; // state_history_log
 
 } // namespace eosio

libraries/state_history/include/eosio/state_history/serialization.hpp

@@ -715,4 +715,15 @@ datastream<ST>& operator<<(datastream<ST>& ds, const eosio::state_history::get_b
    return ds;
 }
 
+template <typename ST>
+datastream<ST>& operator<<(datastream<ST>& ds, const eosio::state_history::get_blocks_result_base& obj) {
+   fc::raw::pack(ds, obj.head);
+   fc::raw::pack(ds, obj.last_irreversible);
+   fc::raw::pack(ds, obj.this_block);
+   fc::raw::pack(ds, obj.prev_block);
+   history_pack_big_bytes(ds, obj.block);
+   return ds;
+}
+
+
 } // namespace fc

libraries/state_history/include/eosio/state_history/trace_converter.hpp

@@ -2,6 +2,7 @@
 
 #include <eosio/chain/block_state.hpp>
 #include <eosio/state_history/types.hpp>
+#include <boost/iostreams/filtering_streambuf.hpp>
 
 namespace eosio {
 namespace state_history {
@@ -13,8 +14,8 @@ struct trace_converter {
    std::map<transaction_id_type, augmented_transaction_trace> cached_traces;
    std::optional<augmented_transaction_trace>                 onblock_trace;
 
-   void  add_transaction(const transaction_trace_ptr& trace, const chain::packed_transaction_ptr& transaction);
-   bytes pack(const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state);
+   void add_transaction(const transaction_trace_ptr& trace, const chain::packed_transaction_ptr& transaction);
+   void pack(boost::iostreams::filtering_ostreambuf& ds, const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state);
 };
 
 } // namespace state_history

libraries/state_history/include/eosio/state_history/types.hpp

@@ -101,12 +101,15 @@ struct get_blocks_ack_request_v0 {
    uint32_t num_messages = 0;
 };
 
-struct get_blocks_result_v0 {
+struct get_blocks_result_base {
    block_position                head;
    block_position                last_irreversible;
    std::optional<block_position> this_block;
    std::optional<block_position> prev_block;
    std::optional<bytes>          block;
+};
+
+struct get_blocks_result_v0 : get_blocks_result_base {
    std::optional<bytes>          traces;
    std::optional<bytes>          deltas;
 };

libraries/state_history/trace_converter.cpp

@@ -15,7 +15,7 @@ void trace_converter::add_transaction(const transaction_trace_ptr& trace, const
    }
 }
 
-bytes trace_converter::pack(const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state) {
+void trace_converter::pack(boost::iostreams::filtering_ostreambuf& obuf, const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state) {
    std::vector<augmented_transaction_trace> traces;
    if (onblock_trace)
       traces.push_back(*onblock_trace);
@@ -33,7 +33,8 @@ bytes trace_converter::pack(const chainbase::database& db, bool trace_debug_mode
    cached_traces.clear();
    onblock_trace.reset();
 
-   return fc::raw::pack(make_history_context_wrapper(db, trace_debug_mode, traces));
+   fc::datastream<boost::iostreams::filtering_ostreambuf&> ds{obuf};
+   return fc::raw::pack(ds, make_history_context_wrapper(db, trace_debug_mode, traces));
 }
 
 } // namespace state_history

libraries/testing/contracts.cpp.in

@@ -8,7 +8,7 @@
 
 #define MAKE_EMBEDDED_WASM_ABI(CN,C, D)                                                                                                                                       \
 INCBIN(eosio_testing_contract_ ## CN ## _wasm, "${CMAKE_BINARY_DIR}/libraries/testing/" #D "/" #C "/" #C ".wasm");                                                            \
-INCBIN(eosio_testing_contract_ ## CN ## _abi,  "${CMAKE_BINARY_DIR}/libraries/testing/" #D "/" #C "/" #C ".abi");                                                             \
+INCTXT(eosio_testing_contract_ ## CN ## _abi,  "${CMAKE_BINARY_DIR}/libraries/testing/" #D "/" #C "/" #C ".abi");                                                             \
                                                                                                                                                                               \
 namespace eosio::testing {                                                                                                                                                    \
    std::vector<std::uint8_t> contracts:: CN ## _wasm() {                                                                                                                      \

libraries/testing/include/eosio/testing/tester.hpp

@@ -198,7 +198,7 @@ namespace eosio { namespace testing {
          unapplied_transaction_queue& get_unapplied_transaction_queue() { return unapplied_transactions; }
 
          transaction_trace_ptr    push_transaction( packed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US );
-         transaction_trace_ptr    push_transaction( signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US, bool no_throw = false );
+         transaction_trace_ptr    push_transaction( signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US, bool no_throw = false, transaction_metadata::trx_type trx_type = transaction_metadata::trx_type::input );
 
          [[nodiscard]]
          action_result            push_action(action&& cert_act, uint64_t authorizer); // TODO/QUESTION: Is this needed?
@@ -341,9 +341,8 @@ namespace eosio { namespace testing {
             return [this]( const account_name& name ) -> std::optional<abi_serializer> {
                try {
                   const auto& accnt = control->db().get<account_object, by_name>( name );
-                  abi_def abi;
-                  if( abi_serializer::to_abi( accnt.abi, abi )) {
-                     return abi_serializer( abi, abi_serializer::create_yield_function( abi_serializer_max_time ) );
+                  if( abi_def abi; abi_serializer::to_abi( accnt.abi, abi )) {
+                     return abi_serializer( std::move(abi), abi_serializer::create_yield_function( abi_serializer_max_time ) );
                   }
                   return std::optional<abi_serializer>();
                } FC_RETHROW_EXCEPTIONS( error, "Failed to find or parse ABI for ${name}", ("name", name))
@@ -490,6 +489,9 @@ namespace eosio { namespace testing {
          }
       }
 
+      tester(const std::function<void(controller&)>& control_setup, setup_policy policy = setup_policy::full,
+             db_read_mode read_mode = db_read_mode::HEAD);
+
       using base_tester::produce_block;
 
       signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override {

libraries/testing/tester.cpp

@@ -582,7 +582,8 @@ namespace eosio { namespace testing {
    transaction_trace_ptr base_tester::push_transaction( signed_transaction& trx,
                                                         fc::time_point deadline,
                                                         uint32_t billed_cpu_time_us,
-                                                        bool no_throw
+                                                        bool no_throw,
+                                                        transaction_metadata::trx_type trx_type
                                                       )
    { try {
       if( !control->is_building_block() )
@@ -597,7 +598,7 @@ namespace eosio { namespace testing {
             fc::microseconds::maximum() :
             fc::microseconds( deadline - fc::time_point::now() );
       auto ptrx = std::make_shared<packed_transaction>( trx, c );
-      auto fut = transaction_metadata::start_recover_keys( ptrx, control->get_thread_pool(), control->get_chain_id(), time_limit, transaction_metadata::trx_type::input );
+      auto fut = transaction_metadata::start_recover_keys( ptrx, control->get_thread_pool(), control->get_chain_id(), time_limit, trx_type );
       auto r = control->push_transaction( fut.get(), deadline, fc::microseconds::maximum(), billed_cpu_time_us, billed_cpu_time_us > 0, 0 );
       if (no_throw) return r;
       if( r->except_ptr ) std::rethrow_exception( r->except_ptr );
@@ -682,7 +683,7 @@ namespace eosio { namespace testing {
                                    const variant_object& data )const { try {
       const auto& acnt = control->get_account(code);
       auto abi = acnt.get_abi();
-      chain::abi_serializer abis(abi, abi_serializer::create_yield_function( abi_serializer_max_time ));
+      chain::abi_serializer abis(std::move(abi), abi_serializer::create_yield_function( abi_serializer_max_time ));
 
       string action_type_name = abis.get_action_type(acttype);
       FC_ASSERT( action_type_name != string(), "unknown action type ${a}", ("a",acttype) );
@@ -1206,6 +1207,20 @@ namespace eosio { namespace testing {
       preactivate_protocol_features( preactivations );
    }
 
+   tester::tester(const std::function<void(controller&)>& control_setup, setup_policy policy, db_read_mode read_mode) {
+      auto def_conf            = default_config(tempdir);
+      def_conf.first.read_mode = read_mode;
+      cfg                      = def_conf.first;
+
+      base_tester::open(make_protocol_feature_set(), def_conf.second.compute_chain_id(),
+                        [&genesis = def_conf.second, &control = this->control, &control_setup]() {
+                           control_setup(*control);
+                           control->startup([]() {}, []() { return false; }, genesis);
+                        });
+
+      execute_setup_policy(policy);
+   }
+
    bool fc_exception_message_is::operator()( const fc::exception& ex ) {
       auto message = ex.get_log().at( 0 ).get_message();
       bool match = (message == expected);

libraries/wasm-jit/Include/IR/Operators.h

@@ -12,10 +12,11 @@ namespace IR
 	struct NoImm {};
 	struct MemoryImm {};
 
+	PACKED_STRUCT(
 	struct ControlStructureImm
 	{
 		ResultType resultType{};
-	};
+	});
 
 	struct BranchImm
 	{
@@ -675,4 +676,19 @@ namespace IR
 	};
 
 	IR_API const char* getOpcodeName(Opcode opcode);
-}
+}
+
+//paranoia for future platforms
+static_assert(sizeof(IR::OpcodeAndImm<IR::NoImm>) == 2);
+static_assert(sizeof(IR::OpcodeAndImm<IR::MemoryImm>) == 2);
+static_assert(sizeof(IR::OpcodeAndImm<IR::ControlStructureImm>) == 3);
+static_assert(sizeof(IR::OpcodeAndImm<IR::BranchImm>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::BranchTableImm>) == 18);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<I32>>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<I64>>) == 10);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<F32>>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LiteralImm<F64>>) == 10);
+static_assert(sizeof(IR::OpcodeAndImm<IR::GetOrSetVariableImm<false>>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::CallImm>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::CallIndirectImm>) == 6);
+static_assert(sizeof(IR::OpcodeAndImm<IR::LoadOrStoreImm<0>>) == 10);

package.cmake

@@ -61,7 +61,7 @@ set(CPACK_DEBIAN_BASE_FILE_NAME "${CPACK_DEBIAN_FILE_NAME}.deb")
 string(REGEX REPLACE "^(${CMAKE_PROJECT_NAME})" "\\1-dev" CPACK_DEBIAN_DEV_FILE_NAME "${CPACK_DEBIAN_BASE_FILE_NAME}")
 
 #deb package tooling will be unable to detect deps for the dev package. llvm is tricky since we don't know what package could have been used; try to figure it out
-set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "libboost-all-dev, libssl-dev, libgmp-dev")
+set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "libboost-all-dev, libssl-dev, libgmp-dev, python3-numpy")
 find_program(DPKG_QUERY "dpkg-query")
 if(DPKG_QUERY AND OS_RELEASE MATCHES "\n?ID=\"?ubuntu" AND LLVM_CMAKE_DIR)
    execute_process(COMMAND "${DPKG_QUERY}" -S "${LLVM_CMAKE_DIR}" COMMAND cut -d: -f1 RESULT_VARIABLE LLVM_PKG_FIND_RESULT OUTPUT_VARIABLE LLVM_PKG_FIND_OUTPUT)
@@ -70,6 +70,7 @@ if(DPKG_QUERY AND OS_RELEASE MATCHES "\n?ID=\"?ubuntu" AND LLVM_CMAKE_DIR)
       string(APPEND CPACK_DEBIAN_DEV_PACKAGE_DEPENDS ", ${LLVM_PKG_FIND_OUTPUT}")
    endif()
 endif()
+set(CPACK_DEBIAN_DEV_PACKAGE_CONTROL_EXTRA "${CMAKE_BINARY_DIR}/scripts/postinst;${CMAKE_BINARY_DIR}/scripts/prerm")
 
 #since rpm packages aren't component based, make sure description picks up more detailed description instead of just summary
 set(CPACK_RPM_PACKAGE_DESCRIPTION "${CPACK_COMPONENT_BASE_DESCRIPTION}")

plugins/CMakeLists.txt

@@ -15,3 +15,4 @@ add_subdirectory(wallet_api_plugin)
 add_subdirectory(db_size_api_plugin)
 add_subdirectory(test_control_plugin)
 add_subdirectory(test_control_api_plugin)
+add_subdirectory(prometheus_plugin)

plugins/chain_api_plugin/chain.swagger.yaml

@@ -548,68 +548,6 @@ paths:
                     type: array
                     items: {}
 
-  /abi_json_to_bin:
-    post:
-      description: Returns an object containing the serialized action data.
-      operationId: abi_json_to_bin
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              title: AbiJsonToBinRequest
-                - code
-                - action
-                - args
-              properties:
-                code:
-                  $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
-                action:
-                  $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
-                args:
-                  type: object
-                  description: json object of the action parameters that will be serialized.
-
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  binargs:
-                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Hex.yaml"
-
-  /abi_bin_to_json:
-    post:
-      description: Returns an object containing the deserialized action data.
-      operationId: abi_bin_to_json
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              title: AbiBinToJsonRequest
-              required:
-                - code
-                - action
-                - binargs
-              properties:
-                code:
-                  $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
-                action:
-                  $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
-                binargs:
-                  $ref: "https://docs.eosnetwork.com/openapi/v2.0/Hex.yaml"
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                type: string
-
   /get_code:
     post:
       description: Returns an object containing the smart contract WASM code.
@@ -786,102 +724,3 @@ paths:
                         threshold:
                           type: "integer"
                           description: the sum of weights that must be met or exceeded to satisfy the permission
-
-  /send_transaction2:
-    post:
-      description: This method expects a transaction in JSON format and will attempt to apply it to the blockchain.
-      operationId: send_transaction2
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                signatures:
-                  type: array
-                  description: array of signatures required to authorize transaction
-                  items:
-                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Signature.yaml"
-                compression:
-                  type: boolean
-                  description: Compression used, usually false
-                packed_context_free_data:
-                  type: string
-                  description: json to hex
-                packed_trx:
-                  type: string
-                  description: Transaction object json to hex
-
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                description: Returns Nothing
-
-  /get_transaction_status:
-    post:
-      description: This method expects a transaction in JSON format and will attempt to apply it to the blockchain.
-      operationId: get_transaction_status
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                signatures:
-                  type: array
-                  description: array of signatures required to authorize transaction
-                  items:
-                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Signature.yaml"
-                compression:
-                  type: boolean
-                  description: Compression used, usually false
-                packed_context_free_data:
-                  type: string
-                  description: json to hex
-                packed_trx:
-                  type: string
-                  description: Transaction object json to hex
-
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                description: Returns Nothing
-
-  /compute_transaction:
-    post:
-      description: This method expects a transaction in JSON format and will attempt to apply it to the blockchain.
-      operationId: compute_transaction
-      requestBody:
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                signatures:
-                  type: array
-                  description: array of signatures required to authorize transaction
-                  items:
-                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Signature.yaml"
-                compression:
-                  type: boolean
-                  description: Compression used, usually false
-                packed_context_free_data:
-                  type: string
-                  description: json to hex
-                packed_trx:
-                  type: string
-                  description: Transaction object json to hex
-
-      responses:
-        "200":
-          description: OK
-          content:
-            application/json:
-              schema:
-                description: Returns Nothing

plugins/chain_api_plugin/chain_api_plugin.cpp

@@ -1,11 +1,11 @@
 #include <eosio/chain_api_plugin/chain_api_plugin.hpp>
 #include <eosio/chain/exceptions.hpp>
-
+#include <eosio/http_plugin/macros.hpp>
 #include <fc/io/json.hpp>
 
 namespace eosio {
 
-static appbase::abstract_plugin& _chain_api_plugin = app().register_plugin<chain_api_plugin>();
+   static auto _chain_api_plugin = application::register_plugin<chain_api_plugin>();
 
 using namespace eosio;
 
@@ -24,13 +24,6 @@ chain_api_plugin::~chain_api_plugin(){}
 void chain_api_plugin::set_program_options(options_description&, options_description&) {}
 void chain_api_plugin::plugin_initialize(const variables_map&) {}
 
-struct async_result_visitor : public fc::visitor<fc::variant> {
-   template<typename T>
-   fc::variant operator()(const T& v) const {
-      return fc::variant(v);
-   }
-};
-
 // Only want a simple 'Invalid transaction id' if unable to parse the body
 template<>
 chain_apis::read_only::get_transaction_status_params
@@ -50,7 +43,7 @@ parse_params<chain_apis::read_only::get_transaction_status_params, http_params_t
 
 #define CALL_WITH_400(api_name, api_handle, api_namespace, call_name, http_response_code, params_type) \
 {std::string("/v1/" #api_name "/" #call_name), \
-   [api_handle](string, string body, url_response_callback cb) mutable { \
+   [api_handle](string&&, string&& body, url_response_callback&& cb) mutable { \
           auto deadline = api_handle.start(); \
           try { \
              auto params = parse_params<api_namespace::call_name ## _params, params_type>(body);\
@@ -62,31 +55,6 @@ parse_params<chain_apis::read_only::get_transaction_status_params, http_params_t
           } \
        }}
 
-#define CALL_ASYNC_WITH_400(api_name, api_handle, api_namespace, call_name, call_result, http_response_code, params_type) \
-{std::string("/v1/" #api_name "/" #call_name), \
-   [api_handle](string, string body, url_response_callback cb) mutable { \
-      auto deadline = api_handle.start(); \
-      try { \
-         auto params = parse_params<api_namespace::call_name ## _params, params_type>(body);\
-         FC_CHECK_DEADLINE(deadline);\
-         api_handle.call_name( std::move(params), \
-            [cb, body](const std::variant<fc::exception_ptr, call_result>& result){\
-               if (std::holds_alternative<fc::exception_ptr>(result)) {\
-                  try {\
-                     std::get<fc::exception_ptr>(result)->dynamic_rethrow_exception();\
-                  } catch (...) {\
-                     http_plugin::handle_exception(#api_name, #call_name, body, cb);\
-                  }\
-               } else {\
-                  cb(http_response_code, fc::time_point::maximum(), std::visit(async_result_visitor(), result));\
-               }\
-            });\
-      } catch (...) { \
-         http_plugin::handle_exception(#api_name, #call_name, body, cb); \
-      } \
-   }\
-}
-
 #define CHAIN_RO_CALL(call_name, http_response_code, params_type) CALL_WITH_400(chain, ro_api, chain_apis::read_only, call_name, http_response_code, params_type)
 #define CHAIN_RW_CALL(call_name, http_response_code, params_type) CALL_WITH_400(chain, rw_api, chain_apis::read_write, call_name, http_response_code, params_type)
 #define CHAIN_RO_CALL_ASYNC(call_name, call_result, http_response_code, params_type) CALL_ASYNC_WITH_400(chain, ro_api, chain_apis::read_only, call_name, call_result, http_response_code, params_type)
@@ -98,25 +66,24 @@ void chain_api_plugin::plugin_startup() {
    ilog( "starting chain_api_plugin" );
    my.reset(new chain_api_plugin_impl(app().get_plugin<chain_plugin>().chain()));
    auto& chain = app().get_plugin<chain_plugin>();
-   auto& http = app().get_plugin<http_plugin>();
-   fc::microseconds max_response_time = http.get_max_response_time();
+   auto& _http_plugin = app().get_plugin<http_plugin>();
+   fc::microseconds max_response_time = _http_plugin.get_max_response_time();
 
    auto ro_api = chain.get_read_only_api(max_response_time);
    auto rw_api = chain.get_read_write_api(max_response_time);
 
-   auto& _http_plugin = app().get_plugin<http_plugin>();
    ro_api.set_shorten_abi_errors( !http_plugin::verbose_errors() );
 
    _http_plugin.add_api( {
-      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::priority::medium_high);
+      CHAIN_RO_CALL(get_info, 200, http_params_types::no_params)}, appbase::exec_queue::read_only_trx_safe, appbase::priority::medium_high);
    _http_plugin.add_api({
       CHAIN_RO_CALL(get_activated_protocol_features, 200, http_params_types::possible_no_params),
-      CHAIN_RO_CALL(get_block, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_block_info, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_block_header_state, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_account, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_code, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_code_hash, 200, http_params_types::params_required),
+      CHAIN_RO_CALL(get_consensus_parameters, 200, http_params_types::no_params),
       CHAIN_RO_CALL(get_abi, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_raw_code_and_abi, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_raw_abi, 200, http_params_types::params_required),
@@ -127,18 +94,21 @@ void chain_api_plugin::plugin_startup() {
       CHAIN_RO_CALL(get_producers, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_producer_schedule, 200, http_params_types::no_params),
       CHAIN_RO_CALL(get_scheduled_transactions, 200, http_params_types::params_required),
-      CHAIN_RO_CALL(abi_json_to_bin, 200, http_params_types::params_required),
-      CHAIN_RO_CALL(abi_bin_to_json, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_required_keys, 200, http_params_types::params_required),
       CHAIN_RO_CALL(get_transaction_id, 200, http_params_types::params_required),
+      CHAIN_RO_CALL_ASYNC(send_read_only_transaction, chain_apis::read_only::send_read_only_transaction_results, 200, http_params_types::params_required),
       CHAIN_RO_CALL_ASYNC(compute_transaction, chain_apis::read_only::compute_transaction_results, 200, http_params_types::params_required),
-      CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required),
+      // transaction related APIs will be posted to general queue after keys are recovered
       CHAIN_RW_CALL_ASYNC(push_transaction, chain_apis::read_write::push_transaction_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(push_transactions, chain_apis::read_write::push_transactions_results, 202, http_params_types::params_required),
       CHAIN_RW_CALL_ASYNC(send_transaction, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required),
-      CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required),
-      CHAIN_RO_CALL(get_consensus_parameters, 200, http_params_types::no_params)
-   });
+      CHAIN_RW_CALL_ASYNC(send_transaction2, chain_apis::read_write::send_transaction_results, 202, http_params_types::params_required)
+   }, appbase::exec_queue::read_only_trx_safe);
+
+   // Not safe to run in parallel with read-only transactions
+   _http_plugin.add_api({
+      CHAIN_RW_CALL_ASYNC(push_block, chain_apis::read_write::push_block_results, 202, http_params_types::params_required)
+   }, appbase::exec_queue::general, appbase::priority::medium_low );
 
    if (chain.account_queries_enabled()) {
       _http_plugin.add_async_api({
@@ -146,11 +116,51 @@ void chain_api_plugin::plugin_startup() {
       });
    }
 
+   _http_plugin.add_async_api({
+      CHAIN_RO_CALL_WITH_400(get_raw_block, 200, http_params_types::params_required),
+      CHAIN_RO_CALL_WITH_400(get_block_header, 200, http_params_types::params_required)
+   });
+
    if (chain.transaction_finality_status_enabled()) {
       _http_plugin.add_api({
          CHAIN_RO_CALL_WITH_400(get_transaction_status, 200, http_params_types::params_required),
-      });
+      }, appbase::exec_queue::read_only_trx_safe);
    }
+
+   _http_plugin.add_api({
+      { std::string("/v1/chain/get_block"),
+        [ro_api, &_http_plugin, max_time=std::min(chain.get_abi_serializer_max_time(),max_response_time)]
+              ( string&&, string&& body, url_response_callback&& cb ) mutable {
+           auto deadline = ro_api.start();
+           try {
+              auto start = fc::time_point::now();
+              auto params = parse_params<chain_apis::read_only::get_raw_block_params, http_params_types::params_required>(body);
+              FC_CHECK_DEADLINE( deadline );
+              chain::signed_block_ptr block = ro_api.get_raw_block( params, deadline );
+
+              auto abi_cache = ro_api.get_block_serializers( block, max_time );
+              FC_CHECK_DEADLINE( deadline );
+
+              auto post_time = fc::time_point::now();
+              auto remaining_time = max_time - (post_time - start);
+              _http_plugin.post_http_thread_pool(
+                    [ro_api, cb, deadline, post_time, remaining_time, abi_cache{std::move(abi_cache)}, block{std::move( block )}]() mutable {
+                 try {
+                    auto new_deadline = deadline + (fc::time_point::now() - post_time);
+
+                    fc::variant result = ro_api.convert_block( block, std::move(abi_cache), remaining_time );
+
+                    cb( 200, new_deadline, std::move( result ) );
+                 } catch( ... ) {
+                    http_plugin::handle_exception( "chain", "get_block", "", cb );
+                 }
+              } );
+           } catch( ... ) {
+              http_plugin::handle_exception("chain", "get_block", body, cb);
+           }
+        }
+      }
+   }, appbase::exec_queue::read_only_trx_safe);
 }
 
 void chain_api_plugin::plugin_shutdown() {}

plugins/chain_api_plugin/include/eosio/chain_api_plugin/chain_api_plugin.hpp

@@ -2,7 +2,7 @@
 #include <eosio/chain_plugin/chain_plugin.hpp>
 #include <eosio/http_plugin/http_plugin.hpp>
 
-#include <appbase/application.hpp>
+#include <eosio/chain/application.hpp>
 #include <eosio/chain/controller.hpp>
 
 namespace eosio {

plugins/chain_interface/include/eosio/chain/plugin_metrics.hpp

@@ -0,0 +1,62 @@
+#pragma once
+
+#include <fc/time.hpp>
+
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace eosio::chain::plugin_interface {
+
+   //
+   // prometheus metrics
+   //
+
+   enum class metric_type {
+      gauge = 1,
+      counter = 2
+   };
+
+   struct runtime_metric {
+      metric_type type = metric_type::gauge;
+      std::string family;
+      std::string label;
+      int64_t value = 0;
+   };
+
+   using metrics_listener = std::function<void(std::vector<runtime_metric>)>;
+
+   struct plugin_metrics {
+
+      virtual ~plugin_metrics() = default;
+      virtual std::vector<runtime_metric> metrics()=0;
+
+      bool should_post() {
+         return (_listener && (fc::time_point::now() > (_last_post + _min_post_interval_us)));
+      }
+
+      bool post_metrics() {
+         if (should_post()){
+            _listener(metrics());
+            _last_post = fc::time_point::now();
+            return true;
+         }
+
+         return false;
+      }
+
+      void register_listener(metrics_listener listener) {
+         _listener = std::move(listener);
+      }
+
+      explicit plugin_metrics(fc::microseconds min_post_interval_us = fc::milliseconds(250))
+      : _min_post_interval_us(min_post_interval_us)
+      , _listener(nullptr) {}
+
+   private:
+      fc::microseconds _min_post_interval_us;
+      metrics_listener _listener;
+      fc::time_point _last_post;
+   };
+
+}