Update appbase to branch tip

[4.0 -> main] Fix for loading snapshot with empty block log

.cicd/platforms.json

@@ -1,7 +1,4 @@
 {
-   "ubuntu18": {
-      "dockerfile": ".cicd/platforms/ubuntu18.Dockerfile"
-   },
    "ubuntu20": {
       "dockerfile": ".cicd/platforms/ubuntu20.Dockerfile"
    },

.cicd/platforms/ubuntu18.Dockerfile

@@ -1,40 +0,0 @@
-FROM ubuntu:bionic
-
-RUN apt-get update && apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential             \
-                                                      cmake                       \
-                                                      curl                        \
-                                                      g++-8                       \
-                                                      git                         \
-                                                      jq                          \
-                                                      libcurl4-openssl-dev        \
-                                                      libgmp-dev                  \
-                                                      libssl-dev                  \
-                                                      llvm-7-dev                  \
-                                                      ninja-build                 \
-                                                      python3                     \
-                                                      python3-numpy               \
-                                                      python3-pip                 \
-                                                      software-properties-common  \
-                                                      zlib1g-dev                  \
-                                                      zstd
-
-RUN python3 -m pip install dataclasses
-
-# GitHub's actions/checkout requires git 2.18+ but Ubuntu 18 only provides 2.17
-RUN add-apt-repository ppa:git-core/ppa && apt update && apt install -y git
-
-# Leap requires boost 1.67+ but Ubuntu 18 only provides 1.65
-# Probably need 1.70+ to work properly with old cmake provided in Ubuntu 18
-RUN curl -L https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 | tar jx && \
-   cd boost_* &&                                                                                              \
-   ./bootstrap.sh --prefix=/boost &&                                                                          \
-   ./b2 --with-iostreams --with-date_time --with-filesystem --with-system                                     \
-        --with-program_options --with-chrono --with-test -j$(nproc) install &&                                \
-   cd .. &&                                                                                                   \
-   rm -rf boost_*
-
-ENV CC=gcc-8
-ENV CXX=g++-8
-ENV BOOST_ROOT=/boost
-ENV LLVM_DIR=/usr/lib/llvm-7/

.cicd/platforms/ubuntu20.Dockerfile

@@ -1,16 +1,16 @@
 FROM ubuntu:focal
-
+ENV TZ="America/New_York"
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential      \
-                                                      cmake                \
-                                                      curl                 \
-                                                      git                  \
-                                                      jq                   \
-                                                      libboost-all-dev     \
-                                                      libcurl4-openssl-dev \
-                                                      libgmp-dev           \
-                                                      libssl-dev           \
-                                                      llvm-11-dev          \
-                                                      ninja-build          \
-                                                      python3-numpy        \
-                                                      zstd
+    apt-get install -y build-essential      \
+                       cmake                \
+                       git                  \
+                       jq                   \
+                       libboost-all-dev     \
+                       libcurl4-openssl-dev \
+                       libgmp-dev           \
+                       libssl-dev           \
+                       llvm-11-dev          \
+                       ninja-build          \
+                       python3-numpy        \
+                       zstd

.cicd/platforms/ubuntu22.Dockerfile

@@ -1,16 +1,16 @@
 FROM ubuntu:jammy
-
+ENV TZ="America/New_York"
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential      \
-                                                      cmake                \
-                                                      curl                 \
-                                                      git                  \
-                                                      jq                   \
-                                                      libboost-all-dev     \
-                                                      libcurl4-openssl-dev \
-                                                      libgmp-dev           \
-                                                      libssl-dev           \
-                                                      llvm-11-dev          \
-                                                      ninja-build          \
-                                                      python3-numpy        \
-                                                      zstd
+    apt-get install -y build-essential      \
+                       cmake                \
+                       git                  \
+                       jq                   \
+                       libboost-all-dev     \
+                       libcurl4-openssl-dev \
+                       libgmp-dev           \
+                       libssl-dev           \
+                       llvm-11-dev          \
+                       ninja-build          \
+                       python3-numpy        \
+                       zstd

.github/actions/parallel-ctest-containers/action.yml

@@ -9,6 +9,8 @@ inputs:
     required: true
   tests-label:
     required: true
+  test-timeout:
+    required: true
 runs:
   using: 'node16'
   main: 'dist/index.mjs'

.github/actions/parallel-ctest-containers/main.mjs

@@ -10,25 +10,27 @@ const container = core.getInput('container', {required: true});
 const error_log_paths = JSON.parse(core.getInput('error-log-paths', {required: true}));
 const log_tarball_prefix = core.getInput('log-tarball-prefix', {required: true});
 const tests_label = core.getInput('tests-label', {required: true});
+const test_timeout = core.getInput('test-timeout', {required: true});
+
+const repo_name = process.env.GITHUB_REPOSITORY.split('/')[1];
 
 try {
-   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", "/__w/leap/leap", container, "sh", "-c", "zstdcat /build.tar.zst | tar x"], {stdio:"inherit"}).status)
+   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", `/__w/${repo_name}/${repo_name}`, container, "sh", "-c", "zstdcat /build.tar.zst | tar x"], {stdio:"inherit"}).status)
       throw new Error("Failed to create base container");
    if(child_process.spawnSync("docker", ["commit", "base", "baseimage"], {stdio:"inherit"}).status)
       throw new Error("Failed to create base image");
    if(child_process.spawnSync("docker", ["rm", "base"], {stdio:"inherit"}).status)
       throw new Error("Failed to remove base container");
 
-   // the correct approach is by far "--show-only=json-v1" and then pluck out .tests[].name; but that doesn't work on U18's cmake 3.10 since it lacks json-v1 output
-   const test_query_result = child_process.spawnSync("docker", ["run", "--rm", "baseimage", "bash", "-e", "-o", "pipefail", "-c", `cd build; ctest -L '${tests_label}' --show-only | head -n -1 | cut -d ':' -f 2 -s | jq -cnR '[inputs | select(length>0)[1:]]'`]);
+   const test_query_result = child_process.spawnSync("docker", ["run", "--rm", "baseimage", "bash", "-e", "-o", "pipefail", "-c", `cd build; ctest -L '${tests_label}' --show-only=json-v1`]);
    if(test_query_result.status)
       throw new Error("Failed to discover tests with label")
-   const tests = JSON.parse(test_query_result.stdout);
+   const tests = JSON.parse(test_query_result.stdout).tests;
 
    let subprocesses = [];
    tests.forEach(t => {
       subprocesses.push(new Promise(resolve => {
-         child_process.spawn("docker", ["run", "--security-opt", "seccomp=unconfined", "-e", "GITHUB_ACTIONS=True", "--name", t, "--init", "baseimage", "bash", "-c", `cd build; ctest --output-on-failure -R '^${t}$'`], {stdio:"inherit"}).on('close', code => resolve(code));
+         child_process.spawn("docker", ["run", "--security-opt", "seccomp=unconfined", "-e", "GITHUB_ACTIONS=True", "--name", t.name, "--init", "baseimage", "bash", "-c", `cd build; ctest --output-on-failure -R '^${t.name}$' --timeout ${test_timeout}`], {stdio:"inherit"}).on('close', code => resolve(code));
       }));
    });
 
@@ -45,13 +47,13 @@ try {
       let packer = tar.pack();
 
       extractor.on('entry', (header, stream, next) => {
-         if(!header.name.startsWith(`__w/leap/leap/build`)) {
+         if(!header.name.startsWith(`__w/${repo_name}/${repo_name}/build`)) {
             stream.on('end', () => next());
             stream.resume();
             return;
          }
 
-         header.name = header.name.substring(`__w/leap/leap/`.length);
+         header.name = header.name.substring(`__w/${repo_name}/${repo_name}/`.length);
          if(header.name !== "build/" && error_log_paths.filter(p => header.name.startsWith(p)).length === 0) {
             stream.on('end', () => next());
             stream.resume();
@@ -61,8 +63,8 @@ try {
          stream.pipe(packer.entry(header, next));
       }).on('finish', () => {packer.finalize()});
 
-      child_process.spawn("docker", ["export", tests[i]]).stdout.pipe(extractor);
-      stream.promises.pipeline(packer, zlib.createGzip(), fs.createWriteStream(`${log_tarball_prefix}-${tests[i]}-logs.tar.gz`));
+      child_process.spawn("docker", ["export", tests[i].name]).stdout.pipe(extractor);
+      stream.promises.pipeline(packer, zlib.createGzip(), fs.createWriteStream(`${log_tarball_prefix}-${tests[i].name}-logs.tar.gz`));
    }
 } catch(e) {
    core.setFailed(`Uncaught exception ${e.message}`);

.github/workflows/build.yaml

@@ -63,7 +63,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu18, ubuntu20, ubuntu22]
+        platform: [ubuntu20, ubuntu22]
     runs-on: ["self-hosted", "enf-x86-beefy"]
     container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
     steps:
@@ -75,11 +75,9 @@ jobs:
           run: |
             # https://github.com/actions/runner/issues/2033
             chown -R $(id -u):$(id -g) $PWD
-            mkdir build
-            cd build
-            cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -GNinja ..
-            ninja
-            tar -pc -C .. --exclude "*.o" build | zstd --long -T0 -9 > ../build.tar.zst
+            cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -GNinja
+            cmake --build build
+            tar -pc --exclude "*.o" build | zstd --long -T0 -9 > build.tar.zst
         - name: Upload builddir
           uses: AntelopeIO/upload-artifact-large-chunks-action@v1
           with:
@@ -105,12 +103,18 @@ jobs:
           zstdcat build.tar.zst | tar x
           cd build
           cpack
+      - name: Install dev package
+        run: |
+          apt update && apt upgrade -y
+          apt install -y ./build/leap_*.deb ./build/leap-dev*.deb
+      - name: Test using TestHarness
+        run: |
+          python3 -c "from TestHarness import Cluster"
       - name: Upload dev package
         uses: actions/upload-artifact@v3
         with:
           name: leap-dev-ubuntu20-amd64
           path: build/leap-dev*.deb
-
   tests:
     name: Tests
     needs: [d, Build]
@@ -120,7 +124,7 @@ jobs:
       matrix:
         platform: [ubuntu20, ubuntu22]
     runs-on: ["self-hosted", "enf-x86-hightier"]
-    container: 
+    container:
       image: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
       options: --security-opt seccomp=unconfined
     steps:
@@ -133,12 +137,9 @@ jobs:
         run: |
           # https://github.com/actions/runner/issues/2033  -- need this because of full version label test looking at git revs
           chown -R $(id -u):$(id -g) $PWD
-          # jammy's boost 1.74 can stumble on an EXDEV via copy_file_range() it doesn't have a fallback for; re-eval once moving to std::filesystem
-          export TMPDIR="$PWD/tmp"
-          mkdir -p $TMPDIR
           zstdcat build.tar.zst | tar x
           cd build
-          ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)"
+          ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)" --timeout 420
 
   np-tests:
     name: NP Tests
@@ -147,7 +148,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu18, ubuntu20, ubuntu22]
+        platform: [ubuntu20, ubuntu22]
     runs-on: ["self-hosted", "enf-x86-midtier"]
     steps:
       - uses: actions/checkout@v3
@@ -162,7 +163,8 @@ jobs:
           error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
           log-tarball-prefix: ${{matrix.platform}}
           tests-label: nonparallelizable_tests
-      - name: Upload logs from failed tests  
+          test-timeout: 420
+      - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
@@ -176,7 +178,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu18, ubuntu20, ubuntu22]
+        platform: [ubuntu20, ubuntu22]
     runs-on: ["self-hosted", "enf-x86-lowtier"]
     steps:
       - uses: actions/checkout@v3
@@ -191,6 +193,7 @@ jobs:
           error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
           log-tarball-prefix: ${{matrix.platform}}
           tests-label: long_running_tests
+          test-timeout: 1800
       - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()

.github/workflows/pinned_build.yaml

@@ -0,0 +1,21 @@
+name: "Pinned Build"
+
+on:
+  workflow_dispatch:
+
+permissions:
+  packages: read
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  Temp:
+    name: temp
+    runs-on: ubuntu-latest
+    steps:
+      - name: test
+        run: |
+          pwd

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required( VERSION 3.8 )
+cmake_minimum_required( VERSION 3.16 )
 
 project( leap )
 include(CTest) # suppresses DartConfiguration.tcl error
@@ -37,12 +37,8 @@ set( LEAP_UTIL_EXECUTABLE_NAME leap-util )
 
 # http://stackoverflow.com/a/18369825
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0)
-        message(FATAL_ERROR "GCC version must be at least 8.0!")
-    endif()
-elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)
-        message(FATAL_ERROR "Clang version must be at least 5.0!")
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
+        message(FATAL_ERROR "GCC version must be at least 9.0!")
     endif()
 endif()
 
@@ -109,7 +105,7 @@ set(Boost_USE_MULTITHREADED      ON)
 set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 # Most boost deps get implictly picked up via fc, as just about everything links to fc. In addition we pick up
 # the pthread dependency through fc.
-find_package(Boost 1.67 REQUIRED COMPONENTS program_options unit_test_framework system)
+find_package(Boost 1.71 REQUIRED COMPONENTS program_options unit_test_framework system)
 
 if( APPLE AND UNIX )
 # Apple Specific Options Here
@@ -251,7 +247,6 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests/TestHarness DESTINATION ${CM
         PATTERN "*.json"
         PATTERN "__pycache__" EXCLUDE
         PATTERN "CMakeFiles" EXCLUDE)
-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tests/launcher.py DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/tests COMPONENT dev EXCLUDE_FROM_ALL)
 
 if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
    # Cmake versions < 3.21 did not support installing symbolic links to a directory via install(FILES ...)
@@ -268,18 +263,8 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib/python3/dist-packages/TestHarness DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages COMPONENT dev EXCLUDE_FROM_ALL)
    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/share/leap_testing/bin DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing COMPONENT dev EXCLUDE_FROM_ALL)
 else()
-   # The following install(CODE ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
-   # However, it can flag as an error if using `make package` instead of `make dev-install` for installation.
-   # This is due to those directories and symbolic links being created in the postinit script during the package install instead.
-
-   # Note If/when doing `make package`, it is not a true error if you see:
-   # Error creating directory "/<leap_install_dir>/lib/python3/dist-packages".
-   # CMake Error: failed to create symbolic link '/<leap_install_dir>/lib/python3/dist-packages/TestHarness': no such file or directory
-   # CMake Error: failed to create symbolic link '/<leap_install_dir>/share/leap_testing/bin': no such file or directory
-
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages)" COMPONENT dev EXCLUDE_FROM_ALL)
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages/TestHarness)" COMPONENT dev EXCLUDE_FROM_ALL)
-   install(CODE "execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/bin)" COMPONENT dev EXCLUDE_FROM_ALL)
+   # The following install(SCRIPT ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
+   install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/scripts/install_testharness_symlinks.cmake COMPONENT dev EXCLUDE_FROM_ALL)
 
    # The `make package` installation of symlinks happens via the `postinst` script installed in cmake.package via the line below
 endif()

CMakeModules/EosioTester.cmake.in

@@ -41,8 +41,6 @@ else ( APPLE )
    set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} -Wall")
 endif ( APPLE )
 
-### Remove after Boost 1.70 CMake fixes are in place
-set( Boost_NO_BOOST_CMAKE ON CACHE STRING "ON or OFF" )
 set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 find_package(Boost @Boost_MAJOR_VERSION@.@Boost_MINOR_VERSION@ EXACT REQUIRED COMPONENTS
    date_time
@@ -62,12 +60,9 @@ find_library(libwasm WASM @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libwast WAST @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libir IR     @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(liblogging Logging @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
-find_library(libruntime Runtime @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libsoftfloat softfloat @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 get_filename_component(cryptodir @OPENSSL_CRYPTO_LIBRARY@ DIRECTORY)
 find_library(liboscrypto crypto "${cryptodir}" NO_DEFAULT_PATH)
-get_filename_component(ssldir @OPENSSL_SSL_LIBRARY@ DIRECTORY)
-find_library(libosssl ssl "${ssldir}" NO_DEFAULT_PATH)
 find_library(libchainbase chainbase @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libbuiltins builtins @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 
@@ -89,11 +84,9 @@ macro(add_eosio_test_executable test_name)
        ${libfc}
        ${libwast}
        ${libwasm}
-       ${libruntime}
        ${libir}
        ${libsoftfloat}
        ${liboscrypto}
-       ${libosssl}
        ${liblogging}
        ${libchainbase}
        ${libbuiltins}

CMakeModules/EosioTesterBuild.cmake.in

@@ -38,8 +38,6 @@ else ( APPLE )
    set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} -Wall")
 endif ( APPLE )
 
-### Remove after Boost 1.70 CMake fixes are in place
-set( Boost_NO_BOOST_CMAKE ON CACHE STRING "ON or OFF" )
 set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 find_package(Boost @Boost_MAJOR_VERSION@.@Boost_MINOR_VERSION@ EXACT REQUIRED COMPONENTS
    date_time
@@ -59,12 +57,9 @@ find_library(libwasm WASM @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WASM NO_D
 find_library(libwast WAST @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WAST NO_DEFAULT_PATH)
 find_library(libir IR     @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/IR NO_DEFAULT_PATH)
 find_library(liblogging Logging @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/Logging NO_DEFAULT_PATH)
-find_library(libruntime Runtime @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/Runtime NO_DEFAULT_PATH)
 find_library(libsoftfloat softfloat @CMAKE_BINARY_DIR@/libraries/softfloat NO_DEFAULT_PATH)
 get_filename_component(cryptodir @OPENSSL_CRYPTO_LIBRARY@ DIRECTORY)
 find_library(liboscrypto crypto "${cryptodir}" NO_DEFAULT_PATH)
-get_filename_component(ssldir @OPENSSL_SSL_LIBRARY@ DIRECTORY)
-find_library(libosssl ssl "${ssldir}" NO_DEFAULT_PATH)
 find_library(libchainbase chainbase @CMAKE_BINARY_DIR@/libraries/chainbase NO_DEFAULT_PATH)
 find_library(libbuiltins builtins @CMAKE_BINARY_DIR@/libraries/builtins NO_DEFAULT_PATH)
 
@@ -86,11 +81,9 @@ macro(add_eosio_test_executable test_name)
        ${libfc}
        ${libwast}
        ${libwasm}
-       ${libruntime}
        ${libir}
        ${libsoftfloat}
        ${liboscrypto}
-       ${libosssl}
        ${liblogging}
        ${libchainbase}
        ${libbuiltins}

README.md

@@ -8,7 +8,6 @@ The `main` branch is the development branch; do not use it for production. Refer
 We currently support the following operating systems.
 - Ubuntu 22.04 Jammy
 - Ubuntu 20.04 Focal
-- Ubuntu 18.04 Bionic
 
 Other Unix derivatives such as macOS are tended to on a best-effort basis and may not be full featured. If you aren't using Ubuntu, please visit the "[Build Unsupported OS](./docs/00_install/01_build-from-source/00_build-unsupported-os.md)" page to explore your options.
 
@@ -45,12 +44,11 @@ You will need to build on a [supported operating system](#supported-operating-sy
 
 Requirements to build:
 - C++17 compiler and standard library
-- boost 1.67+
-- CMake 3.8+
+- boost 1.70+
+- CMake 3.16+
 - LLVM 7 - 11 - for Linux only
   - newer versions do not work
 - openssl 1.1+
-- curl
 - libcurl 7.40.0+
 - git
 - GMP
@@ -127,15 +125,12 @@ Now you can optionally [test](#step-4---test) your build, or [install](#step-5--
 #### Unpinned Build
 The following instructions are valid for this branch. Other release branches may have different requirements, so ensure you follow the directions in the branch or release you intend to build. If you are in an Ubuntu docker container, omit `sudo` because you run as `root` by default.
 
-<details> <summary>Ubuntu 22.04 Jammy & Ubuntu 20.04 Focal</summary>
-
 Install dependencies:
 ```bash
 sudo apt-get update
 sudo apt-get install -y \
         build-essential \
         cmake \
-        curl \
         git \
         libboost-all-dev \
         libcurl4-openssl-dev \
@@ -151,52 +146,6 @@ cd build
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/usr/lib/llvm-11 ..
 make -j "$(nproc)" package
 ```
-</details>
-
-<details> <summary>Ubuntu 18.04 Bionic</summary>
-
-Install dependencies:
-```bash
-sudo apt-get update
-sudo apt-get install -y \
-        build-essential \
-        cmake \
-        curl \
-        g++-8 \
-        git \
-        libcurl4-openssl-dev \
-        libgmp-dev \
-        libssl-dev \
-        llvm-7-dev \
-        python3 \
-        python3-numpy \
-        python3-pip \
-        zlib1g-dev
-
-python3 -m pip install dataclasses
-```
-You need to build Boost from source on this distribution:
-```bash
-curl -fL https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 -o ~/Downloads/boost_1_79_0.tar.bz2
-tar -jvxf ~/Downloads/boost_1_79_0.tar.bz2 -C ~/Downloads/
-pushd ~/Downloads/boost_1_79_0
-./bootstrap.sh --prefix="$HOME/boost1.79"
-./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -j "$(nproc)" install
-popd
-```
-The Boost `*.tar.bz2` download and `boost_1_79_0` folder can be removed now if you want more space.
-```bash
-rm -r ~/Downloads/boost_1_79_0.tar.bz2 ~/Downloads/boost_1_79_0
-```
-From a terminal in the root of the `leap` repo, build.
-```bash
-mkdir -p build
-cd build
-cmake -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 -DCMAKE_PREFIX_PATH="$HOME/boost1.79;/usr/lib/llvm-7/" -DCMAKE_BUILD_TYPE=Release ..
-make -j "$(nproc)" package
-```
-After building, you may remove the `~/boost1.79` directory or you may keep it around for your next build.
-</details>
 
 Now you can optionally [test](#step-4---test) your build, or [install](#step-5---install) the `*.deb` binary packages, which will be in the root of your build directory.
 

docs/00_install/index.md

@@ -12,7 +12,6 @@ Antelope currently supports the following operating systems:
 
 1. Ubuntu 22.04 Jammy
 2. Ubuntu 20.04 Focal
-3. Ubuntu 18.04 Bionic
 
 [[info | Note]]
 | It may be possible to build and install Antelope on other Unix-based operating systems. We gathered helpful information on the following page but please keep in mind that it is experimental and not officially supported. 

docs/01_nodeos/03_plugins/chain_plugin/index.md

@@ -120,7 +120,7 @@ Config Options for eosio::chain_plugin:
                                         applied to them (may specify multiple
                                         times)
   --read-mode arg (=head)               Database read mode ("head",
-                                        "irreversible").
+                                        "irreversible", "speculative").
                                         In "head" mode: database contains state
                                         changes up to the head block;
                                         transactions received by the node are
@@ -131,7 +131,14 @@ Config Options for eosio::chain_plugin:
                                         received via the P2P network are not
                                         relayed and transactions cannot be
                                         pushed via the chain API.
-
+                                        In "speculative" mode: (DEPRECATED:
+                                        head mode recommended) database
+                                        contains state changes by transactions
+                                        in the blockchain up to the head block
+                                        as well as some transactions not yet
+                                        included in the blockchain;
+                                        transactions received by the node are
+                                        relayed if valid.                                        
   --api-accept-transactions arg (=1)    Allow API transactions to be evaluated
                                         and relayed if valid.
   --validation-mode arg (=full)         Chain validation mode ("full" or

docs/01_nodeos/03_plugins/producer_plugin/index.md

@@ -122,8 +122,6 @@ Config Options for eosio::producer_plugin:
                                         transaction queue. Exceeding this value
                                         will subjectively drop transaction with
                                         resource exhaustion.
-  --disable-subjective-billing arg (=1) Disable subjective CPU billing for
-                                        API/P2P transactions
   --disable-subjective-account-billing arg
                                         Account which is excluded from
                                         subjective CPU billing

docs/01_nodeos/07_concepts/05_storage-and-read-modes.md

@@ -29,6 +29,7 @@ The `nodeos` service can be run in different "read" modes. These modes control h
 
 - `head`: this only includes the side effects of confirmed transactions, this mode processes unconfirmed transactions but does not include them.
 - `irreversible`: this mode also includes confirmed transactions only up to those included in the last irreversible block.
+- `speculative`: this includes the side effects of confirmed and unconfirmed transactions.
 
 A transaction is considered confirmed when a `nodeos` instance has received, processed, and written it to a block on the blockchain, i.e. it is in the head block or an earlier block.
 
@@ -44,6 +45,16 @@ When `nodeos` is configured to be in irreversible read mode, it will still track
 
 Clients such as `cleos` and the RPC API will see database state as of the current head block of the chain. It **will not** include changes made by transactions known to this node but not included in the chain, such as unconfirmed transactions.
 
+### Speculative Mode ( Deprecated )
+
+Clients such as `cleos` and the RPC API, will see database state as of the current head block plus changes made by all transactions known to this node but potentially not included in the chain, unconfirmed transactions for example.
+
+Speculative mode is low latency but fragile, there is no guarantee that the transactions reflected in the state will be included in the chain OR that they will reflected in the same order the state implies.
+
+This mode features the lowest latency, but is the least consistent.
+
+In speculative mode `nodeos` is able to execute transactions which have TaPoS (Transaction as Proof of Stake) pointing to any valid block in a fork considered to be the best fork by this node.
+
 ## How To Specify the Read Mode
 
 The mode in which `nodeos` is run can be specified using the `--read-mode` option from the `eosio::chain_plugin`.

libraries/chain/CMakeLists.txt

@@ -5,8 +5,8 @@ file(GLOB HEADERS "include/eosio/chain/*.hpp"
                   "include/eosio/chain/webassembly/*.hpp"
                   "${CMAKE_CURRENT_BINARY_DIR}/include/eosio/chain/core_symbol.hpp" )
 
-if(APPLE AND UNIX)
-   set(PLATFORM_TIMER_IMPL platform_timer_macos.cpp)
+if((APPLE AND UNIX) OR (${CMAKE_SYSTEM_NAME} STREQUAL "FreeBSD"))
+   set(PLATFORM_TIMER_IMPL platform_timer_kqueue.cpp)
 else()
    try_run(POSIX_TIMER_TEST_RUN_RESULT POSIX_TIMER_TEST_COMPILE_RESULT ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/platform_timer_posix_test.c)
    if(POSIX_TIMER_TEST_RUN_RESULT EQUAL 0)
@@ -107,6 +107,7 @@ add_library( eosio_chain
               abi_serializer.cpp
               asset.cpp
               snapshot.cpp
+              snapshot_scheduler.cpp
               deep_mind.cpp
 
              ${CHAIN_EOSVMOC_SOURCES}
@@ -121,6 +122,7 @@ add_library( eosio_chain
              protocol_feature_manager.cpp
              producer_schedule.cpp
              genesis_intrinsics.cpp
+             symbol.cpp
              whitelisted_intrinsics.cpp
              thread_utils.cpp
              platform_timer_accuracy.cpp
@@ -128,7 +130,7 @@ add_library( eosio_chain
              ${HEADERS}
              )
 
-target_link_libraries( eosio_chain PUBLIC bn256 fc chainbase eosio_rapidjson Logging IR WAST WASM Runtime
+target_link_libraries( eosio_chain PUBLIC bn256 fc chainbase eosio_rapidjson Logging IR WAST WASM
                        softfloat builtins ${CHAIN_EOSVM_LIBRARIES} ${LLVM_LIBS} ${CHAIN_RT_LINKAGE}
                      )
 target_include_directories( eosio_chain

libraries/chain/abi_serializer.cpp

@@ -4,6 +4,7 @@
 #include <fc/io/raw.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <fc/io/varint.hpp>
+#include <fc/time.hpp>
 
 namespace eosio { namespace chain {
 
@@ -23,11 +24,11 @@ namespace eosio { namespace chain {
 
    template <typename T>
    inline fc::variant variant_from_stream(fc::datastream<const char*>& stream, const abi_serializer::yield_function_t& yield) {
-      fc::yield_function_t y = [&yield](){ yield(0); }; // create yield function matching fc::variant requirements, 0 for recursive depth
       T temp;
       fc::raw::unpack( stream, temp );
-      y();
-      return fc::variant( temp, y );
+      yield(0);
+      // create yield function matching fc::variant requirements, 0 for recursive depth
+      return fc::variant( temp, [yield](){ yield(0); } );
    }
 
    template <typename T>
@@ -129,7 +130,7 @@ namespace eosio { namespace chain {
    }
 
    void abi_serializer::set_abi(abi_def abi, const yield_function_t& yield) {
-      impl::abi_traverse_context ctx(yield);
+      impl::abi_traverse_context ctx(yield, fc::microseconds{});
 
       EOS_ASSERT(starts_with(abi.version, "eosio::abi/1."), unsupported_abi_version_exception, "ABI has an unsupported version");
 
@@ -235,7 +236,7 @@ namespace eosio { namespace chain {
    }
 
    bool abi_serializer::is_type(const std::string_view& type, const yield_function_t& yield)const {
-      impl::abi_traverse_context ctx(yield);
+      impl::abi_traverse_context ctx(yield, fc::microseconds{});
       return _is_type(type, ctx);
    }
 
@@ -464,23 +465,27 @@ namespace eosio { namespace chain {
    }
 
    fc::variant abi_serializer::binary_to_variant( const std::string_view& type, const bytes& binary, const yield_function_t& yield, bool short_path )const {
-      impl::binary_to_variant_context ctx(*this, yield, type);
+      impl::binary_to_variant_context ctx(*this, yield, fc::microseconds{}, type);
       ctx.short_path = short_path;
       return _binary_to_variant(type, binary, ctx);
    }
 
-   fc::variant abi_serializer::binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path )const {
-      return binary_to_variant( type, binary, create_yield_function(max_serialization_time), short_path );
+   fc::variant abi_serializer::binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path )const {
+      impl::binary_to_variant_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type);
+      ctx.short_path = short_path;
+      return _binary_to_variant(type, binary, ctx);
    }
 
    fc::variant abi_serializer::binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const yield_function_t& yield, bool short_path )const {
-      impl::binary_to_variant_context ctx(*this, yield, type);
+      impl::binary_to_variant_context ctx(*this, yield, fc::microseconds{}, type);
       ctx.short_path = short_path;
       return _binary_to_variant(type, binary, ctx);
    }
 
-   fc::variant abi_serializer::binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const fc::microseconds& max_serialization_time, bool short_path )const {
-      return binary_to_variant( type, binary, create_yield_function(max_serialization_time), short_path );
+   fc::variant abi_serializer::binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path )const {
+      impl::binary_to_variant_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type);
+      ctx.short_path = short_path;
+      return _binary_to_variant(type, binary, ctx);
    }
 
    void abi_serializer::_variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char *>& ds, impl::variant_to_binary_context& ctx )const
@@ -603,23 +608,27 @@ namespace eosio { namespace chain {
    } FC_CAPTURE_AND_RETHROW() }
 
    bytes abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, const yield_function_t& yield, bool short_path )const {
-      impl::variant_to_binary_context ctx(*this, yield, type);
+      impl::variant_to_binary_context ctx(*this, yield, fc::microseconds{}, type);
       ctx.short_path = short_path;
       return _variant_to_binary(type, var, ctx);
    }
 
-   bytes abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path ) const {
-       return variant_to_binary( type, var, create_yield_function(max_serialization_time), short_path );
+   bytes abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_action_data_serialization_time, bool short_path ) const {
+      impl::variant_to_binary_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type);
+      ctx.short_path = short_path;
+      return _variant_to_binary(type, var, ctx);
    }
 
    void  abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const yield_function_t& yield, bool short_path )const {
-      impl::variant_to_binary_context ctx(*this, yield, type);
+      impl::variant_to_binary_context ctx(*this, yield, fc::microseconds{}, type);
       ctx.short_path = short_path;
       _variant_to_binary(type, var, ds, ctx);
    }
 
-   void  abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const fc::microseconds& max_serialization_time, bool short_path ) const {
-       variant_to_binary( type, var, create_yield_function(max_serialization_time), short_path );
+   void  abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const fc::microseconds& max_action_data_serialization_time, bool short_path ) const {
+      impl::variant_to_binary_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type);
+      ctx.short_path = short_path;
+      _variant_to_binary(type, var, ds, ctx);
    }
 
    type_name abi_serializer::get_action_type(name action)const {
@@ -650,9 +659,7 @@ namespace eosio { namespace chain {
    namespace impl {
 
       fc::scoped_exit<std::function<void()>> abi_traverse_context::enter_scope() {
-         std::function<void()> callback = [old_recursion_depth=recursion_depth, this](){
-            recursion_depth = old_recursion_depth;
-         };
+         std::function<void()> callback = [this](){ --recursion_depth; };
 
          ++recursion_depth;
          yield( recursion_depth );

libraries/chain/apply_context.cpp

@@ -19,15 +19,26 @@ namespace eosio { namespace chain {
 
 static inline void print_debug(account_name receiver, const action_trace& ar) {
    if (!ar.console.empty()) {
-      auto prefix = fc::format_string(
-                                      "\n[(${a},${n})->${r}]",
-                                      fc::mutable_variant_object()
-                                      ("a", ar.act.account)
-                                      ("n", ar.act.name)
-                                      ("r", receiver));
-      dlog(prefix + ": CONSOLE OUTPUT BEGIN =====================\n"
-           + ar.console
-           + prefix + ": CONSOLE OUTPUT END   =====================" );
+      if (fc::logger::get(DEFAULT_LOGGER).is_enabled( fc::log_level::debug )) {
+         std::string prefix;
+         prefix.reserve(3 + 13 + 1 + 13 + 3 + 13 + 1);
+         prefix += "\n[(";
+         prefix += ar.act.account.to_string();
+         prefix += ",";
+         prefix += ar.act.name.to_string();
+         prefix += ")->";
+         prefix += receiver.to_string();
+         prefix += "]";
+
+         std::string output;
+         output.reserve(512);
+         output += prefix;
+         output += ": CONSOLE OUTPUT BEGIN =====================\n";
+         output += ar.console;
+         output += prefix;
+         output += ": CONSOLE OUTPUT END   =====================";
+         dlog( std::move(output) );
+      }
    }
 }
 
@@ -467,7 +478,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
       trx.ref_block_num = 0;
       trx.ref_block_prefix = 0;
    } else {
-      trx.expiration = control.pending_block_time() + fc::microseconds(999'999); // Rounds up to nearest second (makes expiration check unnecessary)
+      trx.expiration = time_point_sec{control.pending_block_time() + fc::microseconds(999'999)}; // Rounds up to nearest second (makes expiration check unnecessary)
       trx.set_reference_block(control.head_block_id()); // No TaPoS check necessary
    }
 

libraries/chain/asset.cpp

@@ -1,9 +1,14 @@
+#include <string>
 #include <eosio/chain/asset.hpp>
 #include <boost/rational.hpp>
+#include <boost/algorithm/string.hpp>
 #include <fc/reflect/variant.hpp>
+#include <fc/safe.hpp>
 
 namespace eosio { namespace chain {
 
+using fc::safe;
+
 uint8_t asset::decimals()const {
    return sym.decimals();
 }
@@ -19,11 +24,11 @@ int64_t asset::precision()const {
 string asset::to_string()const {
    string sign = amount < 0 ? "-" : "";
    int64_t abs_amount = std::abs(amount);
-   string result = fc::to_string( static_cast<int64_t>(abs_amount) / precision());
+   string result = std::to_string( static_cast<int64_t>(abs_amount) / precision());
    if( decimals() )
    {
       auto fract = static_cast<int64_t>(abs_amount) % precision();
-      result += "." + fc::to_string(precision() + fract).erase(0,1);
+      result += "." + std::to_string(precision() + fract).erase(0,1);
    }
    return sign + result + " " + symbol_name();
 }
@@ -31,12 +36,12 @@ string asset::to_string()const {
 asset asset::from_string(const string& from)
 {
    try {
-      string s = fc::trim(from);
+      string s = boost::algorithm::trim_copy(from);
 
       // Find space in order to split amount and symbol
       auto space_pos = s.find(' ');
       EOS_ASSERT((space_pos != string::npos), asset_type_exception, "Asset's amount and symbol should be separated with space");
-      auto symbol_str = fc::trim(s.substr(space_pos + 1));
+      auto symbol_str = boost::algorithm::trim_copy(s.substr(space_pos + 1));
       auto amount_str = s.substr(0, space_pos);
 
       // Ensure that if decimal point is used (.), decimal fraction is specified

libraries/chain/authorization_manager.cpp

@@ -559,8 +559,8 @@ namespace eosio { namespace chain {
 
       }
 
-      if( !allow_unused_keys  || check_but_dont_fail) {
-         EOS_ASSERT( checker.all_keys_used(), tx_irrelevant_sig,
+      if( !allow_unused_keys ) {
+         EOS_ASSERT( checker.all_keys_used() || check_but_dont_fail, tx_irrelevant_sig,
                      "transaction bears irrelevant signatures from these keys: ${keys}",
                      ("keys", checker.unused_keys()) );
       }

libraries/chain/block_log.cpp

@@ -7,6 +7,7 @@
 #include <fc/bitutil.hpp>
 #include <fc/io/raw.hpp>
 #include <mutex>
+#include <string>
 
 #if defined(__BYTE_ORDER__)
 static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__);
@@ -67,14 +68,14 @@ namespace eosio { namespace chain {
             sizeof(uint32_t) + sizeof(first_block_num) + sizeof(chain_id_type) + sizeof(block_log::npos);
 
       template <typename Stream>
-      void read_from(Stream& ds, const fc::path& log_path) {
+      void read_from(Stream& ds, const std::filesystem::path& log_path) {
          ds.read((char*)&ver, sizeof(ver));
          EOS_ASSERT(version() > 0, block_log_exception, "Block log was not setup properly");
          EOS_ASSERT(block_log::is_supported_version(version()), block_log_unsupported_version,
                     "Unsupported version of block log. Block log version is ${version} while code supports version(s) "
                     "[${min},${max}], log file: ${log}",
                     ("version", version())("min", block_log::min_supported_version)(
-                          "max", block_log::max_supported_version)("log", log_path.generic_string()));
+                          "max", block_log::max_supported_version)("log", log_path));
 
          first_block_num = 1;
          if (version() != initial_version) {
@@ -150,7 +151,7 @@ namespace eosio { namespace chain {
 
       class index_writer {
        public:
-         index_writer(const fc::path& block_index_name, uint32_t blocks_expected, bool create = true) {
+         index_writer(const std::filesystem::path& block_index_name, uint32_t blocks_expected, bool create = true) {
             index_file.set_file_path(block_index_name);
             auto mode = create ? fc::cfile::truncate_rw_mode : fc::cfile::update_rw_mode;
             index_file.open(mode);
@@ -204,12 +205,12 @@ namespace eosio { namespace chain {
 
        public:
          block_log_data() = default;
-         block_log_data(const fc::path& path) { open(path); }
+         block_log_data(const std::filesystem::path& path) { open(path); }
          uint64_t first_block_position() const { return first_block_pos; }
 
          const block_log_preamble& get_preamble() const { return preamble; }
 
-         void open(const fc::path& path) {
+         void open(const std::filesystem::path& path) {
             if (file.is_open())
                file.close();
             file.set_file_path(path);
@@ -312,20 +313,20 @@ namespace eosio { namespace chain {
          }
 
          std::tuple<uint64_t, uint32_t, std::string>
-         full_validate_blocks(uint32_t last_block_num, const fc::path& blocks_dir, fc::time_point now);
+         full_validate_blocks(uint32_t last_block_num, const std::filesystem::path& blocks_dir, fc::time_point now);
 
-         void construct_index(const fc::path& index_file_path);
+         void construct_index(const std::filesystem::path& index_file_path);
       };
 
       using block_log_index = eosio::chain::log_index<block_log_exception>;
 
       /// Provide the read only view for both blocks.log and blocks.index files
       struct block_log_bundle {
-         fc::path        block_file_name, index_file_name; // full pathname for blocks.log and blocks.index
+         std::filesystem::path        block_file_name, index_file_name; // full pathname for blocks.log and blocks.index
          block_log_data  log_data;
          block_log_index log_index;
 
-         block_log_bundle(fc::path block_file, fc::path index_file)
+         block_log_bundle(std::filesystem::path block_file, std::filesystem::path index_file)
              : block_file_name(std::move(block_file)), index_file_name(std::move(index_file)) {
 
             log_data.open(block_file_name);
@@ -345,7 +346,7 @@ namespace eosio { namespace chain {
                         "index_num_blocks", index_num_blocks)("index_file_name", index_file_name));
          }
 
-         explicit block_log_bundle(const fc::path& block_dir)
+         explicit block_log_bundle(const std::filesystem::path& block_dir)
              : block_log_bundle(block_dir / "blocks.log", block_dir / "blocks.index") {}
       };
 
@@ -405,7 +406,7 @@ namespace eosio { namespace chain {
          return num_blocks;
       }
 
-      void block_log_data::construct_index(const fc::path& index_file_path) {
+      void block_log_data::construct_index(const std::filesystem::path& index_file_path) {
          std::string index_file_name = index_file_path.generic_string();
          ilog("Will write new blocks.index file ${file}", ("file", index_file_name));
 
@@ -438,12 +439,12 @@ namespace eosio { namespace chain {
    struct block_log_verifier {
       chain_id_type chain_id = chain_id_type::empty_chain_id();
 
-      void verify(block_log_data& log, const boost::filesystem::path& log_path) {
+      void verify(block_log_data& log, const std::filesystem::path& log_path) {
          if (chain_id.empty()) {
             chain_id = log.chain_id();
          } else {
             EOS_ASSERT(chain_id == log.chain_id(), block_log_exception,
-                       "block log file ${path} has a different chain id", ("path", log_path.generic_string()));
+                       "block log file ${path} has a different chain id", ("path", log_path));
          }
       }
    };
@@ -496,19 +497,21 @@ namespace eosio { namespace chain {
 
       /// Would remove pre-existing block log and index, never write blocks into disk.
       struct empty_block_log final : block_log_impl {
-         explicit empty_block_log(const bfs::path& log_dir) {
-            fc::remove(log_dir / "blocks.log");
-            fc::remove(log_dir / "blocks.index");
+         uint32_t first_block_number = std::numeric_limits<uint32_t>::max();
+
+         explicit empty_block_log(const std::filesystem::path& log_dir) {
+            std::filesystem::remove(log_dir / "blocks.log");
+            std::filesystem::remove(log_dir / "blocks.index");
          }
 
-         uint32_t first_block_num() final { return head ? head->block_num() : 1; }
+         uint32_t first_block_num() final { return head ? head->block_num() : first_block_number; }
          void append(const signed_block_ptr& b, const block_id_type& id, const std::vector<char>& packed_block) final {
             update_head(b, id);
          }
 
          uint64_t get_block_pos(uint32_t block_num) final { return block_log::npos; }
          void reset(const genesis_state& gs, const signed_block_ptr& first_block) final { update_head(first_block); }
-         void reset(const chain_id_type& chain_id, uint32_t first_block_num) final {}
+         void reset(const chain_id_type& chain_id, uint32_t first_block_num) final { first_block_number = first_block_num; }
          void flush() final {}
 
          signed_block_ptr read_block_by_num(uint32_t block_num) final { return {}; };
@@ -526,10 +529,10 @@ namespace eosio { namespace chain {
 
          basic_block_log() = default;
 
-         explicit basic_block_log(bfs::path log_dir) { open(log_dir); }
+         explicit basic_block_log(std::filesystem::path log_dir) { open(log_dir); }
 
          static void ensure_file_exists(fc::cfile& f) {
-            if (fc::exists(f.get_file_path()))
+            if (std::filesystem::exists(f.get_file_path()))
                return;
             f.open(fc::cfile::create_or_update_rw_mode);
             f.close();
@@ -544,7 +547,7 @@ namespace eosio { namespace chain {
                   index_file.open(fc::cfile::update_rw_mode);
                   vacuum(first_block_num_from_pruned_log(), preamble.first_block_num);
                } else {
-                  fc::resize_file(index_file.get_file_path(), 0);
+                  std::filesystem::resize_file(index_file.get_file_path(), 0);
                }
                preamble.ver = preamble.version();
             }
@@ -621,10 +624,10 @@ namespace eosio { namespace chain {
             FC_LOG_AND_RETHROW()
          }
 
-         void open(const fc::path& data_dir) {
+         void open(const std::filesystem::path& data_dir) {
 
-            if (!fc::is_directory(data_dir))
-               fc::create_directories(data_dir);
+            if (!std::filesystem::is_directory(data_dir))
+               std::filesystem::create_directories(data_dir);
 
             this->block_file.set_file_path(data_dir / "blocks.log");
             this->index_file.set_file_path(data_dir / "blocks.index");
@@ -649,8 +652,8 @@ namespace eosio { namespace chain {
              */
             ensure_file_exists(block_file);
             ensure_file_exists(index_file);
-            auto log_size   = fc::file_size(this->block_file.get_file_path());
-            auto index_size = fc::file_size(this->index_file.get_file_path());
+            auto log_size   = std::filesystem::file_size(this->block_file.get_file_path());
+            auto index_size = std::filesystem::file_size(this->index_file.get_file_path());
 
             if (log_size) {
                block_log_data log_data(block_file.get_file_path());
@@ -689,7 +692,7 @@ namespace eosio { namespace chain {
 
             } else if (index_size) {
                ilog("Log file is empty while the index file is nonempty, discard the index file");
-               boost::filesystem::resize_file(index_file.get_file_path(), 0);
+               std::filesystem::resize_file(index_file.get_file_path(), 0);
             }
 
             if (!block_file.is_open())
@@ -793,8 +796,8 @@ namespace eosio { namespace chain {
             //  and it'll mess this all up. Be sure to still remove the 4 byte trailer though.
             if (!head) {
                block_file.flush();
-               fc::resize_file(block_file.get_file_path(),
-                               fc::file_size(block_file.get_file_path()) - sizeof(uint32_t));
+               std::filesystem::resize_file(block_file.get_file_path(),
+                                            std::filesystem::file_size(block_file.get_file_path()) - sizeof(uint32_t));
                return;
             }
 
@@ -829,7 +832,7 @@ namespace eosio { namespace chain {
                }
             }
             block_file.flush();
-            fc::resize_file(block_file.get_file_path(), block_file.tellp());
+            std::filesystem::resize_file(block_file.get_file_path(), block_file.tellp());
 
             index_file.flush();
             {
@@ -847,7 +850,7 @@ namespace eosio { namespace chain {
                   block_file.write((char*)&new_pos, sizeof(new_pos));
                }
             }
-            fc::resize_file(index_file.get_file_path(), num_blocks_in_log * sizeof(uint64_t));
+            std::filesystem::resize_file(index_file.get_file_path(), num_blocks_in_log * sizeof(uint64_t));
 
             preamble.first_block_num = first_block_num;
          }
@@ -894,9 +897,9 @@ namespace eosio { namespace chain {
             return block_file.tellp();
          }
 
-         static void write_incomplete_block_data(const fc::path& blocks_dir, fc::time_point now, uint32_t block_num,
+         static void write_incomplete_block_data(const std::filesystem::path& blocks_dir, fc::time_point now, uint32_t block_num,
                                                  fc::cfile& strm) {
-            auto      tail_path = blocks_dir / std::string("blocks-bad-tail-").append(now).append(".log");
+            auto      tail_path = blocks_dir / std::string("blocks-bad-tail-").append(now.to_iso_string()).append(".log");
             fc::cfile tail;
             tail.set_file_path(tail_path);
             tail.open(fc::cfile::create_or_update_rw_mode);
@@ -932,7 +935,7 @@ namespace eosio { namespace chain {
 
                write_incomplete_block_data(block_file.get_file_path().parent_path(), fc::time_point::now(),
                                            expected_block_num + 1, ds);
-               boost::filesystem::resize_file(block_file.get_file_path(), trimmed_block_file_size);
+               std::filesystem::resize_file(block_file.get_file_path(), trimmed_block_file_size);
                return true;
             } catch (...) { return false; }
          }
@@ -944,12 +947,12 @@ namespace eosio { namespace chain {
          block_log_catalog catalog;
          const size_t      stride;
 
-         partitioned_block_log(const bfs::path& log_dir, const partitioned_blocklog_config& config) : stride(config.stride) {
+         partitioned_block_log(const std::filesystem::path& log_dir, const partitioned_blocklog_config& config) : stride(config.stride) {
             catalog.open(log_dir, config.retained_dir, config.archive_dir, "blocks");
             catalog.max_retained_files = config.max_retained_files;
 
             open(log_dir);
-            const auto log_size = fc::file_size(block_file.get_file_path());
+            const auto log_size = std::filesystem::file_size(block_file.get_file_path());
 
             if (log_size == 0 && !catalog.empty()) {
                basic_block_log::reset(catalog.verifier.chain_id, catalog.last_block_num() + 1);
@@ -965,12 +968,12 @@ namespace eosio { namespace chain {
             fc::datastream<fc::cfile> new_block_file;
             fc::datastream<fc::cfile> new_index_file;
 
-            fc::path block_file_path = block_file.get_file_path();
-            fc::path index_file_path = index_file.get_file_path();
+            std::filesystem::path block_file_path = block_file.get_file_path();
+            std::filesystem::path index_file_path = index_file.get_file_path();
 
-            fc::path tmp_block_file_path = block_file_path;
+            std::filesystem::path tmp_block_file_path = block_file_path;
             tmp_block_file_path.replace_extension("log.tmp");
-            fc::path tmp_index_file_path = index_file_path;
+            std::filesystem::path tmp_index_file_path = index_file_path;
             tmp_index_file_path.replace_extension("index.tmp");
 
             new_block_file.set_file_path(tmp_block_file_path);
@@ -996,8 +999,8 @@ namespace eosio { namespace chain {
             swap(new_block_file, block_file);
             swap(new_index_file, index_file);
 
-            fc::rename(tmp_block_file_path, block_file_path);
-            fc::rename(tmp_index_file_path, index_file_path);
+            std::filesystem::rename(tmp_block_file_path, block_file_path);
+            std::filesystem::rename(tmp_index_file_path, index_file_path);
 
             block_file.set_file_path(block_file_path);
             index_file.set_file_path(index_file_path);
@@ -1046,7 +1049,7 @@ namespace eosio { namespace chain {
          uint32_t              first_block_number = 0; // the first number available to read
          prune_blocklog_config prune_config;
 
-         punch_hole_block_log(const fc::path& data_dir, const prune_blocklog_config& prune_conf)
+         punch_hole_block_log(const std::filesystem::path& data_dir, const prune_blocklog_config& prune_conf)
              : prune_config(prune_conf) {
             EOS_ASSERT(__builtin_popcount(prune_config.prune_threshold) == 1, block_log_exception,
                        "block log prune threshold must be power of 2");
@@ -1168,7 +1171,7 @@ namespace eosio { namespace chain {
 
    } // namespace detail
 
-   block_log::block_log(const fc::path& data_dir, const block_log_config& config)
+   block_log::block_log(const std::filesystem::path& data_dir, const block_log_config& config)
        : my(std::visit(overloaded{ [&data_dir](const basic_blocklog_config& conf) -> detail::block_log_impl* {
                                      return new detail::basic_block_log(data_dir);
                                   },
@@ -1263,17 +1266,17 @@ namespace eosio { namespace chain {
    }
 
    // static
-   void block_log::construct_index(const fc::path& block_file_name, const fc::path& index_file_name) {
+   void block_log::construct_index(const std::filesystem::path& block_file_name, const std::filesystem::path& index_file_name) {
 
-      ilog("Will read existing blocks.log file ${file}", ("file", block_file_name.generic_string()));
-      ilog("Will write new blocks.index file ${file}", ("file", index_file_name.generic_string()));
+      ilog("Will read existing blocks.log file ${file}", ("file", block_file_name));
+      ilog("Will write new blocks.index file ${file}", ("file", index_file_name));
 
       block_log_data log_data(block_file_name);
       log_data.construct_index(index_file_name);
    }
 
    std::tuple<uint64_t, uint32_t, std::string>
-   block_log_data::full_validate_blocks(uint32_t last_block_num, const fc::path& blocks_dir, fc::time_point now) {
+   block_log_data::full_validate_blocks(uint32_t last_block_num, const std::filesystem::path& blocks_dir, fc::time_point now) {
       uint64_t      pos       = first_block_position();
       uint32_t      block_num = first_block_num() - 1;
       std::string   error_msg;
@@ -1306,32 +1309,32 @@ namespace eosio { namespace chain {
    }
 
    // static
-   fc::path block_log::repair_log(const fc::path& data_dir, uint32_t truncate_at_block,
+   std::filesystem::path block_log::repair_log(const std::filesystem::path& data_dir, uint32_t truncate_at_block,
                                   const char* reversible_block_dir_name) {
       ilog("Recovering Block Log...");
-      EOS_ASSERT(fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found,
+      EOS_ASSERT(std::filesystem::is_directory(data_dir) && std::filesystem::is_regular_file(data_dir / "blocks.log"), block_log_not_found,
                  "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir));
 
       if (truncate_at_block == 0)
          truncate_at_block = UINT32_MAX;
 
       auto now        = fc::time_point::now();
-      auto blocks_dir = fc::canonical(
+      auto blocks_dir = std::filesystem::canonical(
             data_dir); // canonical always returns an absolute path that has no symbolic link, dot, or dot-dot elements
       auto blocks_dir_name = blocks_dir.filename();
-      auto backup_dir      = blocks_dir.parent_path() / blocks_dir_name.generic_string().append("-").append(now);
+      auto backup_dir      = blocks_dir.parent_path() / blocks_dir_name.generic_string().append("-").append(now.to_iso_string());
 
-      EOS_ASSERT(!fc::exists(backup_dir), block_log_backup_dir_exist,
+      EOS_ASSERT(!std::filesystem::exists(backup_dir), block_log_backup_dir_exist,
                  "Cannot move existing blocks directory to already existing directory '${new_blocks_dir}'",
                  ("new_blocks_dir", backup_dir));
 
-      fc::create_directories(backup_dir);
-      fc::rename(blocks_dir / "blocks.log", backup_dir / "blocks.log");
-      if (fc::exists(blocks_dir / "blocks.index")) {
-         fc::rename(blocks_dir / "blocks.index", backup_dir / "blocks.index");
+      std::filesystem::create_directories(backup_dir);
+      std::filesystem::rename(blocks_dir / "blocks.log", backup_dir / "blocks.log");
+      if (std::filesystem::exists(blocks_dir / "blocks.index")) {
+         std::filesystem::rename(blocks_dir / "blocks.index", backup_dir / "blocks.index");
       }
-      if (strlen(reversible_block_dir_name) && fc::is_directory(blocks_dir / reversible_block_dir_name)) {
-         fc::rename(blocks_dir / reversible_block_dir_name, backup_dir / reversible_block_dir_name);
+      if (strlen(reversible_block_dir_name) && std::filesystem::is_directory(blocks_dir / reversible_block_dir_name)) {
+         std::filesystem::rename(blocks_dir / reversible_block_dir_name, backup_dir / reversible_block_dir_name);
       }
       ilog("Moved existing blocks directory to backup location: '${new_blocks_dir}'", ("new_blocks_dir", backup_dir));
 
@@ -1370,16 +1373,61 @@ namespace eosio { namespace chain {
    }
 
    // static
-   std::optional<genesis_state> block_log::extract_genesis_state(const fc::path& block_dir) {
-      boost::filesystem::path p(block_dir / "blocks.log");
-      for_each_file_in_dir_matches(block_dir, R"(blocks-1-\d+\.log)",
-                                   [&p](boost::filesystem::path log_path) { p = std::move(log_path); });
-      return block_log_data(p).get_genesis_state();
+   std::optional<block_log::chain_context> block_log::extract_chain_context(const std::filesystem::path& block_dir,
+                                                                            const std::filesystem::path& retained_dir) {
+      std::filesystem::path first_block_file;
+      if (!retained_dir.empty() && std::filesystem::exists(retained_dir)) {
+         for_each_file_in_dir_matches(retained_dir, R"(blocks-1-\d+\.log)",
+                                      [&](std::filesystem::path log_path) {
+                                          first_block_file = std::move(log_path);
+                                      });
+      }
+
+      if (first_block_file.empty() && std::filesystem::exists(block_dir / "blocks.log")) {
+         first_block_file = block_dir / "blocks.log";
+      }
+
+      if (!first_block_file.empty()) {
+         return block_log_data(first_block_file).get_preamble().chain_context;
+      }
+      
+      if (!retained_dir.empty() && std::filesystem::exists(retained_dir)) {
+         const std::regex        my_filter(R"(blocks-\d+-\d+\.log)");
+         std::smatch             what;
+         std::filesystem::directory_iterator end_itr; // Default ctor yields past-the-end
+         for (std::filesystem::directory_iterator p(retained_dir); p != end_itr; ++p) {
+            // Skip if not a file
+            if (!std::filesystem::is_regular_file(p->status()))
+               continue;
+            // skip if it does not match the pattern
+            std::string file = p->path().filename().string();
+            if (!std::regex_match(file, what, my_filter))
+               continue;
+            return block_log_data(p->path()).chain_id();
+         }
+      }
+      return {};
    }
 
    // static
-   chain_id_type block_log::extract_chain_id(const fc::path& data_dir) {
-      return block_log_data(data_dir / "blocks.log").chain_id();
+   std::optional<genesis_state> block_log::extract_genesis_state(const std::filesystem::path& block_dir,
+                                                                 const std::filesystem::path& retained_dir) {
+      auto context = extract_chain_context(block_dir, retained_dir);
+      if (!context || std::holds_alternative<chain_id_type>(*context))
+         return {};
+      return std::get<genesis_state>(*context);
+   }
+
+   // static
+   std::optional<chain_id_type> block_log::extract_chain_id(const std::filesystem::path& block_dir,
+                                                            const std::filesystem::path& retained_dir) {
+      auto context = extract_chain_context(block_dir, retained_dir);
+      if (!context)
+         return {};
+      return std::visit(overloaded{
+         [](const chain_id_type& id){ return id; },
+         [](const genesis_state& gs){ return gs.compute_chain_id(); }
+          } , *context);
    }
 
    // static
@@ -1398,7 +1446,7 @@ namespace eosio { namespace chain {
    }
 
    // static
-   bool block_log::is_pruned_log(const fc::path& data_dir) {
+   bool block_log::is_pruned_log(const std::filesystem::path& data_dir) {
       uint32_t version = 0;
       try {
          fc::cfile log_file;
@@ -1409,7 +1457,7 @@ namespace eosio { namespace chain {
       return detail::is_pruned_log_and_mask_version(version);
    }
 
-   void extract_blocklog_i(block_log_bundle& log_bundle, const fc::path& new_block_filename, const fc::path& new_index_filename,
+   void extract_blocklog_i(block_log_bundle& log_bundle, const std::filesystem::path& new_block_filename, const std::filesystem::path& new_index_filename,
                            uint32_t first_block_num, uint32_t num_blocks) {
 
       auto position_for_block = [&log_bundle](uint64_t block_num) {
@@ -1458,12 +1506,12 @@ namespace eosio { namespace chain {
    }
 
    // static
-   bool block_log::trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir,
+   bool block_log::trim_blocklog_front(const std::filesystem::path& block_dir, const std::filesystem::path& temp_dir,
                                        uint32_t truncate_at_block) {
       EOS_ASSERT(block_dir != temp_dir, block_log_exception, "block_dir and temp_dir need to be different directories");
 
       ilog("In directory ${dir} will trim all blocks before block ${n} from blocks.log and blocks.index.",
-           ("dir", block_dir.generic_string())("n", truncate_at_block));
+           ("dir", block_dir)("n", truncate_at_block));
 
       block_log_bundle log_bundle(block_dir);
 
@@ -1478,17 +1526,17 @@ namespace eosio { namespace chain {
       }
 
       // ****** create the new block log file and write out the header for the file
-      fc::create_directories(temp_dir);
-      fc::path new_block_filename = temp_dir / "blocks.log";
-      fc::path new_index_filename = temp_dir / "blocks.index";
+      std::filesystem::create_directories(temp_dir);
+      std::filesystem::path new_block_filename = temp_dir / "blocks.log";
+      std::filesystem::path new_index_filename = temp_dir / "blocks.index";
 
       extract_blocklog_i(log_bundle, new_block_filename, new_index_filename, truncate_at_block,
                          log_bundle.log_data.last_block_num() - truncate_at_block + 1);
 
-      fc::path old_log = temp_dir / "old.log";
+      std::filesystem::path old_log = temp_dir / "old.log";
       rename(log_bundle.block_file_name, old_log);
       rename(new_block_filename, log_bundle.block_file_name);
-      fc::path old_ind = temp_dir / "old.index";
+      std::filesystem::path old_ind = temp_dir / "old.index";
       rename(log_bundle.index_file_name, old_ind);
       rename(new_index_filename, log_bundle.index_file_name);
 
@@ -1496,13 +1544,13 @@ namespace eosio { namespace chain {
    }
 
    // static
-   int block_log::trim_blocklog_end(const fc::path& block_dir, uint32_t n) { // n is last block to keep (remove later blocks)
+   int block_log::trim_blocklog_end(const std::filesystem::path& block_dir, uint32_t n) { // n is last block to keep (remove later blocks)
 
       block_log_bundle log_bundle(block_dir);
 
       ilog("In directory ${block_dir} will trim all blocks after block ${n} from ${block_file} and ${index_file}",
-           ("block_dir", block_dir.generic_string())("n", n)("block_file", log_bundle.block_file_name.generic_string())(
-                 "index_file", log_bundle.index_file_name.generic_string()));
+           ("block_dir", block_dir)("n", n)("block_file", log_bundle.block_file_name)(
+                 "index_file", log_bundle.index_file_name));
 
       if (n < log_bundle.log_data.first_block_num()) {
          dlog("All blocks are after block ${n} so do nothing (trim_end would delete entire blocks.log)", ("n", n));
@@ -1519,21 +1567,25 @@ namespace eosio { namespace chain {
       const auto to_trim_block_position = log_bundle.log_index.nth_block_position(to_trim_block_index);
       const auto index_file_size        = to_trim_block_index * sizeof(uint64_t);
 
-      boost::filesystem::resize_file(log_bundle.block_file_name, to_trim_block_position);
-      boost::filesystem::resize_file(log_bundle.index_file_name, index_file_size);
+      std::filesystem::resize_file(log_bundle.block_file_name, to_trim_block_position);
+      std::filesystem::resize_file(log_bundle.index_file_name, index_file_size);
       ilog("blocks.index has been trimmed to ${index_file_size} bytes", ("index_file_size", index_file_size));
       return 0;
    }
 
    // static
-   void block_log::smoke_test(const fc::path& block_dir, uint32_t interval) {
+   void block_log::smoke_test(const std::filesystem::path& block_dir, uint32_t interval) {
 
       block_log_bundle log_bundle(block_dir);
 
+      ilog("block log version= ${version}",("version", log_bundle.log_data.version()));
+      ilog("first block= ${first}",("first", log_bundle.log_data.first_block_num()));
+      ilog("last block= ${last}",("last", log_bundle.log_data.last_block_num()));
+
       ilog("blocks.log and blocks.index agree on number of blocks");
 
       if (interval == 0) {
-         interval = std::max((log_bundle.log_index.num_blocks() + 7u) >> 3, 1u);
+         interval = std::max((log_bundle.log_index.num_blocks() + 7) >> 3, 1U);
       }
       uint32_t expected_block_num = log_bundle.log_data.first_block_num();
 
@@ -1543,18 +1595,18 @@ namespace eosio { namespace chain {
       }
    }
 
-   std::pair<fc::path, fc::path> blocklog_files(const fc::path& dir, uint32_t start_block_num, uint32_t num_blocks) {
+   std::pair<std::filesystem::path, std::filesystem::path> blocklog_files(const std::filesystem::path& dir, uint32_t start_block_num, uint32_t num_blocks) {
       const int bufsize = 64;
       char      buf[bufsize];
       snprintf(buf, bufsize, "blocks-%u-%u.log", start_block_num, start_block_num + num_blocks - 1);
-      fc::path new_block_filename = dir / buf;
-      fc::path new_index_filename(new_block_filename);
+      std::filesystem::path new_block_filename = dir / buf;
+      std::filesystem::path new_index_filename(new_block_filename);
       new_index_filename.replace_extension(".index");
       return std::make_pair(new_block_filename, new_index_filename);
    }
 
    // static
-   void block_log::extract_block_range(const fc::path& block_dir, const fc::path& dest_dir,
+   void block_log::extract_block_range(const std::filesystem::path& block_dir, const std::filesystem::path& dest_dir,
                                        block_num_type start_block_num, block_num_type last_block_num) {
 
 
@@ -1564,8 +1616,8 @@ namespace eosio { namespace chain {
                  "The first available block is block ${first_block}.",
                  ("first_block", log_bundle.log_data.first_block_num()));
 
-      if (!fc::exists(dest_dir))
-         fc::create_directories(dest_dir);
+      if (!std::filesystem::exists(dest_dir))
+         std::filesystem::create_directories(dest_dir);
 
       uint32_t num_blocks = last_block_num - start_block_num + 1;
 
@@ -1575,14 +1627,14 @@ namespace eosio { namespace chain {
    }
 
    // static
-   void block_log::split_blocklog(const fc::path& block_dir, const fc::path& dest_dir, uint32_t stride) {
+   void block_log::split_blocklog(const std::filesystem::path& block_dir, const std::filesystem::path& dest_dir, uint32_t stride) {
 
       block_log_bundle log_bundle(block_dir);
       const uint32_t   first_block_num = log_bundle.log_data.first_block_num();
       const uint32_t   last_block_num  = log_bundle.log_data.last_block_num();
 
-      if (!fc::exists(dest_dir))
-         fc::create_directories(dest_dir);
+      if (!std::filesystem::exists(dest_dir))
+         std::filesystem::create_directories(dest_dir);
 
       for (uint32_t i = (first_block_num - 1) / stride; i < (last_block_num + stride - 1) / stride; ++i) {
          uint32_t start_block_num = std::max(i * stride + 1, first_block_num);
@@ -1594,16 +1646,16 @@ namespace eosio { namespace chain {
       }
    }
 
-   inline bfs::path operator+(const bfs::path& left, const bfs::path& right) { return bfs::path(left) += right; }
+   inline std::filesystem::path operator+(const std::filesystem::path& left, const std::filesystem::path& right) { return std::filesystem::path(left) += right; }
 
-   void move_blocklog_files(const bfs::path& src_dir, const fc::path& dest_dir, uint32_t start_block,
+   void move_blocklog_files(const std::filesystem::path& src_dir, const std::filesystem::path& dest_dir, uint32_t start_block,
                             uint32_t end_block) {
       auto [log_filename, index_filename] = blocklog_files(dest_dir, start_block, end_block - start_block + 1);
-      bfs::rename(src_dir / "blocks.log", log_filename);
-      bfs::rename(src_dir / "blocks.index", index_filename);
+      std::filesystem::rename(src_dir / "blocks.log", log_filename);
+      std::filesystem::rename(src_dir / "blocks.index", index_filename);
    }
 
-   uint32_t get_blocklog_version(const bfs::path& blocklog_file) {
+   uint32_t get_blocklog_version(const std::filesystem::path& blocklog_file) {
       uint32_t  version;
       fc::cfile f;
       f.set_file_path(blocklog_file.generic_string());
@@ -1613,7 +1665,7 @@ namespace eosio { namespace chain {
    }
 
    // static
-   void block_log::merge_blocklogs(const fc::path& blocks_dir, const fc::path& dest_dir) {
+   void block_log::merge_blocklogs(const std::filesystem::path& blocks_dir, const std::filesystem::path& dest_dir) {
       block_log_catalog catalog;
 
       catalog.open("", blocks_dir, "", "blocks");
@@ -1622,20 +1674,20 @@ namespace eosio { namespace chain {
          return;
       }
 
-      if (!fc::exists(dest_dir))
-         fc::create_directories(dest_dir);
+      if (!std::filesystem::exists(dest_dir))
+         std::filesystem::create_directories(dest_dir);
 
-      fc::temp_directory temp_dir;
-      bfs::path          temp_path   = temp_dir.path();
-      uint32_t           start_block = 0, end_block = 0;
+      fc::temp_directory    temp_dir;
+      std::filesystem::path temp_path   = temp_dir.path();
+      uint32_t              start_block = 0, end_block = 0;
 
-      bfs::path                 temp_block_log   = temp_path / "blocks.log";
-      bfs::path                 temp_block_index = temp_path / "blocks.index";
+      std::filesystem::path     temp_block_log   = temp_path / "blocks.log";
+      std::filesystem::path     temp_block_index = temp_path / "blocks.index";
       fc::datastream<fc::cfile> file;
       file.set_file_path(temp_block_log);
 
       for (auto const& [first_block_num, val] : catalog.collection) {
-         if (bfs::exists(temp_block_log)) {
+         if (std::filesystem::exists(temp_block_log)) {
             if (first_block_num == end_block + 1) {
                block_log_data log_data;
                log_data.open(val.filename_base + ".log");
@@ -1654,13 +1706,13 @@ namespace eosio { namespace chain {
             } else
                wlog("${file}.log cannot be merged with previous block log file because of the discontinuity of blocks, "
                     "skip merging.",
-                    ("file", val.filename_base.generic_string()));
+                    ("file", val.filename_base));
             // there is a version or block number gap between the stride files
             move_blocklog_files(temp_path, dest_dir, start_block, end_block);
          }
 
-         bfs::copy(val.filename_base + ".log", temp_block_log);
-         bfs::copy(val.filename_base + ".index", temp_block_index);
+         std::filesystem::copy(val.filename_base + ".log", temp_block_log);
+         std::filesystem::copy(val.filename_base + ".index", temp_block_index);
          start_block = first_block_num;
          end_block   = val.last_block_num;
       }
@@ -1668,7 +1720,7 @@ namespace eosio { namespace chain {
       if (file.is_open())
          file.close();
 
-      if (bfs::exists(temp_block_log)) {
+      if (std::filesystem::exists(temp_block_log)) {
          move_blocklog_files(temp_path, dest_dir, start_block, end_block);
       }
    }

libraries/chain/controller.cpp

@@ -21,6 +21,7 @@
 #include <eosio/chain/protocol_feature_manager.hpp>
 #include <eosio/chain/authorization_manager.hpp>
 #include <eosio/chain/resource_limits.hpp>
+#include <eosio/chain/subjective_billing.hpp>
 #include <eosio/chain/chain_snapshot.hpp>
 #include <eosio/chain/thread_utils.hpp>
 #include <eosio/chain/platform_timer.hpp>
@@ -69,7 +70,7 @@ class maybe_session {
       maybe_session() = default;
 
       maybe_session( maybe_session&& other)
-      :_session(move(other._session))
+         :_session(std::move(other._session))
       {
       }
 
@@ -96,7 +97,7 @@ class maybe_session {
 
       maybe_session& operator = ( maybe_session&& mv ) {
          if (mv._session) {
-            _session.emplace(move(*mv._session));
+            _session.emplace(std::move(*mv._session));
             mv._session.reset();
          } else {
             _session.reset();
@@ -150,7 +151,7 @@ struct pending_state {
                   block_timestamp_type when,
                   uint16_t num_prev_blocks_to_confirm,
                   const vector<digest_type>& new_protocol_feature_activations )
-   :_db_session( move(s) )
+   :_db_session( std::move(s) )
    ,_block_stage( building_block( prev, when, num_prev_blocks_to_confirm, new_protocol_feature_activations ) )
    {}
 
@@ -210,6 +211,12 @@ struct pending_state {
 };
 
 struct controller_impl {
+   enum class app_window_type {
+      write, // Only main thread is running; read-only threads are not running.
+             // All read-write and read-only tasks are sequentially executed.
+      read   // Main thread and read-only threads are running read-ony tasks in parallel.
+             // Read-write tasks are not being executed.
+   };
 
    // LLVM sets the new handler, we need to reset this to throw a bad_alloc exception so we can possibly exit cleanly
    // and not just abort.
@@ -226,6 +233,7 @@ struct controller_impl {
    block_state_ptr                 head;
    fork_database                   fork_db;
    resource_limits_manager         resource_limits;
+   subjective_billing              subjective_bill;
    authorization_manager           authorization;
    protocol_feature_manager        protocol_features;
    controller::config              conf;
@@ -246,11 +254,10 @@ struct controller_impl {
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
    thread_local static vm::wasm_allocator wasm_alloc; // a copy for main thread and each read-only thread
 #endif
-   // Ideally wasmif should be thread_local which must be a static.
-   // Unittests can create multiple controller objects (testers) at the same time,
-   // which overwrites the same static wasmif, is used for eosvmoc too.
    wasm_interface  wasmif;  // used by main thread and all threads for EOSVMOC
-   thread_local static std::unique_ptr<wasm_interface> wasmif_thread_local; // a copy for each read-only thread, used by eos-vm and eos-vm-jit
+   std::mutex threaded_wasmifs_mtx;
+   std::unordered_map<std::thread::id, std::unique_ptr<wasm_interface>> threaded_wasmifs; // one for each read-only thread, used by eos-vm and eos-vm-jit
+   app_window_type app_window = app_window_type::write;
 
    typedef pair<scope_name,action_name>                   handler_key;
    map< account_name, map<handler_key, apply_handler> >   apply_handlers;
@@ -265,9 +272,7 @@ struct controller_impl {
          prev = fork_db.root();
       }
 
-      if ( read_mode == db_read_mode::HEAD ) {
-         EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
-      }
+      EOS_ASSERT( head->block, block_validate_exception, "attempting to pop a block that was sparsely loaded from a snapshot");
 
       head = prev;
 
@@ -337,7 +342,12 @@ struct controller_impl {
       set_activation_handler<builtin_protocol_feature_t::crypto_primitives>();
 
       self.irreversible_block.connect([this](const block_state_ptr& bsp) {
-         get_wasm_interface().current_lib(bsp->block_num);
+         // producer_plugin has already asserted irreversible_block signal is
+         // called in write window
+         wasmif.current_lib(bsp->block_num);
+         for (auto& w: threaded_wasmifs) {
+            w.second->current_lib(bsp->block_num);
+         }
       });
 
 
@@ -392,11 +402,13 @@ struct controller_impl {
       }
    }
 
-   void dmlog_applied_transaction(const transaction_trace_ptr& t) {
+   void dmlog_applied_transaction(const transaction_trace_ptr& t, const signed_transaction* trx = nullptr) {
       // dmlog_applied_transaction is called by push_scheduled_transaction
       // where transient transactions are not possible, and by push_transaction
       // only when the transaction is not transient
       if (auto dm_logger = get_deep_mind_logger(false)) {
+         if (trx && is_onblock(*t))
+            dm_logger->on_onblock(*trx);
          dm_logger->on_applied_transaction(self.head_block_num() + 1, t);
       }
    }
@@ -415,7 +427,8 @@ struct controller_impl {
          EOS_ASSERT( root_id == log_head_id, fork_database_exception, "fork database root does not match block log head" );
       } else {
          EOS_ASSERT( fork_db.root()->block_num == lib_num, fork_database_exception,
-                     "empty block log expects the first appended block to build off a block that is not the fork database root. root block number: ${block_num}, lib: ${lib_num}", ("block_num", fork_db.root()->block_num) ("lib_num", lib_num) );
+                     "The first block ${lib_num} when starting with an empty block log should be the block after fork database root ${bn}.",
+                     ("lib_num", lib_num)("bn", fork_db.root()->block_num) );
       }
 
       const auto fork_head = fork_db_head();
@@ -497,9 +510,10 @@ struct controller_impl {
 
    void replay(std::function<bool()> check_shutdown) {
       auto blog_head = blog.head();
-      if( !blog_head && !fork_db.root() ) {
+      if( !fork_db.root() ) {
          fork_db.reset( *head );
-         return;
+         if (!blog_head)
+            return;
       }
 
       replaying = true;
@@ -1159,7 +1173,7 @@ struct controller_impl {
          etrx.ref_block_num = 0;
          etrx.ref_block_prefix = 0;
       } else {
-         etrx.expiration = self.pending_block_time() + fc::microseconds(999'999); // Round up to nearest second to avoid appearing expired
+         etrx.expiration = time_point_sec{self.pending_block_time() + fc::microseconds(999'999)}; // Round up to nearest second to avoid appearing expired
          etrx.set_reference_block( self.head_block_id() );
       }
 
@@ -1616,7 +1630,7 @@ struct controller_impl {
                        emit(self.accepted_transaction, trx);
                    }
 
-                   dmlog_applied_transaction(trace);
+                   dmlog_applied_transaction(trace, &trn);
                    emit(self.applied_transaction, std::tie(trace, trx->packed_trx()));
                 }
             }
@@ -1624,7 +1638,7 @@ struct controller_impl {
             if ( trx->is_transient() ) {
                // remove trx from pending block by not canceling 'restore'
                trx_context.undo(); // this will happen automatically in destructor, but make it more explicit
-            } else if ( pending->_block_status == controller::block_status::ephemeral ) {
+            } else if ( read_mode != db_read_mode::SPECULATIVE && pending->_block_status == controller::block_status::ephemeral ) {
                // An ephemeral block will never become a full block, but on a producer node the trxs should be saved
                // in the un-applied transaction queue for execution during block production. For a non-producer node
                // save the trxs in the un-applied transaction queue for use during block validation to skip signature
@@ -2420,7 +2434,7 @@ struct controller_impl {
       for( const auto& a : trxs )
          trx_digests.emplace_back( a.digest() );
 
-      return merkle( move(trx_digests) );
+      return merkle( std::move(trx_digests) );
    }
 
    void update_producers_authority() {
@@ -2475,7 +2489,7 @@ struct controller_impl {
       auto now = self.is_building_block() ? self.pending_block_time() : self.head_block_time();
       const auto total = dedupe_index.size();
       uint32_t num_removed = 0;
-      while( (!dedupe_index.empty()) && ( now > fc::time_point(dedupe_index.begin()->expiration) ) ) {
+      while( (!dedupe_index.empty()) && ( now > dedupe_index.begin()->expiration.to_time_point() ) ) {
          transaction_idx.remove(*dedupe_index.begin());
          ++num_removed;
          if( deadline <= fc::time_point::now() ) {
@@ -2650,15 +2664,10 @@ struct controller_impl {
          trx.ref_block_num = 0;
          trx.ref_block_prefix = 0;
       } else {
-         trx.expiration = self.pending_block_time() + fc::microseconds(999'999); // Round up to nearest second to avoid appearing expired
+         trx.expiration = time_point_sec{self.pending_block_time() + fc::microseconds(999'999)}; // Round up to nearest second to avoid appearing expired
          trx.set_reference_block( self.head_block_id() );
       }
 
-      // onblock transaction cannot be transient
-      if (auto dm_logger = get_deep_mind_logger(false)) {
-         dm_logger->on_onblock(trx);
-      }
-
       return trx;
    }
 
@@ -2680,28 +2689,51 @@ struct controller_impl {
    // only called from non-main threads (read-only trx execution threads)
    // when producer_plugin starts them
    void init_thread_local_data() {
-      EOS_ASSERT( !is_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
+      EOS_ASSERT( !is_on_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if ( is_eos_vm_oc_enabled() )
          // EOSVMOC needs further initialization of its thread local data
          wasmif.init_thread_local_data();
       else
 #endif
+      {
+         std::lock_guard g(threaded_wasmifs_mtx);
          // Non-EOSVMOC needs a wasmif per thread
-         wasmif_thread_local = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+         threaded_wasmifs[std::this_thread::get_id()]  = std::make_unique<wasm_interface>( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty());
+      }
    }
 
-   bool is_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+   bool is_on_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+
+   void set_to_write_window() {
+      app_window = app_window_type::write;
+   }
+   void set_to_read_window() {
+      app_window = app_window_type::read;
+   }
+   bool is_write_window() const {
+      return app_window == app_window_type::write;
+   }
 
    wasm_interface& get_wasm_interface() {
-      if ( is_main_thread()
+      if ( is_on_main_thread()
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
           || is_eos_vm_oc_enabled()
 #endif
          )
          return wasmif;
       else
-         return *wasmif_thread_local;
+         return *threaded_wasmifs[std::this_thread::get_id()];
+   }
+
+   void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+      // The caller of this function apply_eosio_setcode has already asserted that
+      // the transaction is not a read-only trx, which implies we are
+      // in write window. Safe to call threaded_wasmifs's code_block_num_last_used
+      wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      for (auto& w: threaded_wasmifs) {
+         w.second->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+      }
    }
 
    block_state_ptr fork_db_head() const;
@@ -2711,7 +2743,6 @@ thread_local platform_timer controller_impl::timer;
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
 thread_local eosio::vm::wasm_allocator controller_impl::wasm_alloc;
 #endif
-thread_local std::unique_ptr<wasm_interface> controller_impl::wasmif_thread_local;
 
 const resource_limits_manager&   controller::get_resource_limits_manager()const
 {
@@ -2736,6 +2767,15 @@ const protocol_feature_manager& controller::get_protocol_feature_manager()const
    return my->protocol_features;
 }
 
+const subjective_billing& controller::get_subjective_billing()const {
+   return my->subjective_bill;
+}
+
+subjective_billing& controller::get_mutable_subjective_billing() {
+   return my->subjective_bill;
+}
+
+
 uint32_t controller::get_max_nonprivileged_inline_action_size()const
 {
    return my->conf.max_nonprivileged_inline_action_size;
@@ -3084,7 +3124,7 @@ block_id_type controller::fork_db_head_block_id()const {
    return my->fork_db_head()->id;
 }
 
-time_point controller::pending_block_time()const {
+block_timestamp_type controller::pending_block_timestamp()const {
    EOS_ASSERT( my->pending, block_validate_exception, "no pending block" );
 
    if( std::holds_alternative<completed_block>(my->pending->_block_stage) )
@@ -3093,6 +3133,10 @@ time_point controller::pending_block_time()const {
    return my->pending->get_pending_block_header_state().timestamp;
 }
 
+time_point controller::pending_block_time()const {
+   return pending_block_timestamp();
+}
+
 uint32_t controller::pending_block_num()const {
    EOS_ASSERT( my->pending, block_validate_exception, "no pending block" );
 
@@ -3436,12 +3480,12 @@ uint32_t controller::configured_subjective_signature_length_limit()const {
 void controller::validate_expiration( const transaction& trx )const { try {
    const auto& chain_configuration = get_global_properties().configuration;
 
-   EOS_ASSERT( time_point(trx.expiration) >= pending_block_time(),
+   EOS_ASSERT( trx.expiration.to_time_point() >= pending_block_time(),
                expired_tx_exception,
                "transaction has expired, "
                "expiration is ${trx.expiration} and pending block time is ${pending_block_time}",
                ("trx.expiration",trx.expiration)("pending_block_time",pending_block_time()));
-   EOS_ASSERT( time_point(trx.expiration) <= pending_block_time() + fc::seconds(chain_configuration.max_transaction_lifetime),
+   EOS_ASSERT( trx.expiration.to_time_point() <= pending_block_time() + fc::seconds(chain_configuration.max_transaction_lifetime),
                tx_exp_too_far_exception,
                "Transaction expiration is too far in the future relative to the reference time of ${reference_time}, "
                "expiration is ${trx.expiration} and the maximum transaction lifetime is ${max_til_exp} seconds",
@@ -3686,6 +3730,20 @@ void controller::init_thread_local_data() {
    my->init_thread_local_data();
 }
 
+void controller::set_to_write_window() {
+   my->set_to_write_window();
+}
+void controller::set_to_read_window() {
+   my->set_to_read_window();
+}
+bool controller::is_write_window() const {
+   return my->is_write_window();
+}
+
+void controller::code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+   return my->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+}
+
 /// Protocol feature activation handlers:
 
 template<>

libraries/chain/eosio_contract.cpp

@@ -160,7 +160,7 @@ void apply_eosio_setcode(apply_context& context) {
       old_size  = (int64_t)old_code_entry.code.size() * config::setcode_ram_bytes_multiplier;
       if( old_code_entry.code_ref_count == 1 ) {
          db.remove(old_code_entry);
-         context.control.get_wasm_interface().code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
+         context.control.code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
       } else {
          db.modify(old_code_entry, [](code_object& o) {
             --o.code_ref_count;

libraries/chain/fork_database.cpp

@@ -61,7 +61,7 @@ namespace eosio { namespace chain {
    }
 
    struct fork_database_impl {
-      explicit fork_database_impl( const fc::path& data_dir )
+      explicit fork_database_impl( const std::filesystem::path& data_dir )
       :datadir(data_dir)
       {}
 
@@ -69,7 +69,7 @@ namespace eosio { namespace chain {
       fork_multi_index_type index;
       block_state_ptr       root; // Only uses the block_header_state portion
       block_state_ptr       head;
-      fc::path              datadir;
+      std::filesystem::path datadir;
 
       void open_impl( const std::function<void( block_timestamp_type,
                                                 const flat_set<digest_type>&,
@@ -97,7 +97,7 @@ namespace eosio { namespace chain {
    };
 
 
-   fork_database::fork_database( const fc::path& data_dir )
+   fork_database::fork_database( const std::filesystem::path& data_dir )
    :my( new fork_database_impl( data_dir ) )
    {}
 
@@ -114,11 +114,11 @@ namespace eosio { namespace chain {
                                                                  const flat_set<digest_type>&,
                                                                  const vector<digest_type>& )>& validator )
    {
-      if (!fc::is_directory(datadir))
-         fc::create_directories(datadir);
+      if (!std::filesystem::is_directory(datadir))
+         std::filesystem::create_directories(datadir);
 
       auto fork_db_dat = datadir / config::forkdb_filename;
-      if( fc::exists( fork_db_dat ) ) {
+      if( std::filesystem::exists( fork_db_dat ) ) {
          try {
             string content;
             fc::read_file_contents( fork_db_dat, content );
@@ -130,7 +130,7 @@ namespace eosio { namespace chain {
             fc::raw::unpack( ds, totem );
             EOS_ASSERT( totem == fork_database::magic_number, fork_database_exception,
                         "Fork database file '${filename}' has unexpected magic number: ${actual_totem}. Expected ${expected_totem}",
-                        ("filename", fork_db_dat.generic_string())
+                        ("filename", fork_db_dat)
                         ("actual_totem", totem)
                         ("expected_totem", fork_database::magic_number)
             );
@@ -142,7 +142,7 @@ namespace eosio { namespace chain {
                         fork_database_exception,
                        "Unsupported version of fork database file '${filename}'. "
                        "Fork database version is ${version} while code supports version(s) [${min},${max}]",
-                       ("filename", fork_db_dat.generic_string())
+                       ("filename", fork_db_dat)
                        ("version", version)
                        ("min", fork_database::min_supported_version)
                        ("max", fork_database::max_supported_version)
@@ -169,22 +169,22 @@ namespace eosio { namespace chain {
                head = get_block_impl( head_id );
                EOS_ASSERT( head, fork_database_exception,
                            "could not find head while reconstructing fork database from file; '${filename}' is likely corrupted",
-                           ("filename", fork_db_dat.generic_string()) );
+                           ("filename", fork_db_dat) );
             }
 
             auto candidate = index.get<by_lib_block_num>().begin();
             if( candidate == index.get<by_lib_block_num>().end() || !(*candidate)->is_valid() ) {
                EOS_ASSERT( head->id == root->id, fork_database_exception,
                            "head not set to root despite no better option available; '${filename}' is likely corrupted",
-                           ("filename", fork_db_dat.generic_string()) );
+                           ("filename", fork_db_dat) );
             } else {
                EOS_ASSERT( !first_preferred( **candidate, *head ), fork_database_exception,
                            "head not set to best available option available; '${filename}' is likely corrupted",
-                           ("filename", fork_db_dat.generic_string()) );
+                           ("filename", fork_db_dat) );
             }
          } FC_CAPTURE_AND_RETHROW( (fork_db_dat) )
 
-         fc::remove( fork_db_dat );
+         std::filesystem::remove( fork_db_dat );
       }
    }
 
@@ -199,7 +199,7 @@ namespace eosio { namespace chain {
       if( !root ) {
          if( index.size() > 0 ) {
             elog( "fork_database is in a bad state when closing; not writing out '${filename}'",
-                  ("filename", fork_db_dat.generic_string()) );
+                  ("filename", fork_db_dat) );
          }
          return;
       }
@@ -250,7 +250,7 @@ namespace eosio { namespace chain {
          fc::raw::pack( out, head->id );
       } else {
          elog( "head not set in fork database; '${filename}' will be corrupted",
-               ("filename", fork_db_dat.generic_string()) );
+               ("filename", fork_db_dat) );
       }
 
       index.clear();

libraries/chain/include/eosio/chain/abi_serializer.hpp

@@ -6,8 +6,9 @@
 #include <utility>
 #include <fc/variant_object.hpp>
 #include <fc/scoped_exit.hpp>
+#include <fc/time.hpp>
 
-namespace eosio { namespace chain {
+namespace eosio::chain {
 
 using std::map;
 using std::string;
@@ -23,6 +24,7 @@ namespace impl {
    struct abi_traverse_context_with_path;
    struct binary_to_variant_context;
    struct variant_to_binary_context;
+   struct action_data_to_variant_context;
 }
 
 /**
@@ -48,7 +50,6 @@ struct abi_serializer {
    bool      is_szarray(const std::string_view& type)const;
    bool      is_optional(const std::string_view& type)const;
    bool      is_type( const std::string_view& type, const yield_function_t& yield )const;
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
    bool      is_type(const std::string_view& type, const fc::microseconds& max_serialization_time)const;
    bool      is_builtin_type(const std::string_view& type)const;
    bool      is_integer(const std::string_view& type) const;
@@ -67,33 +68,29 @@ struct abi_serializer {
    std::optional<string>  get_error_message( uint64_t error_code )const;
 
    fc::variant binary_to_variant( const std::string_view& type, const bytes& binary, const yield_function_t& yield, bool short_path = false )const;
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   fc::variant binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const;
+   fc::variant binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const;
    fc::variant binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const yield_function_t& yield, bool short_path = false )const;
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   fc::variant binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const;
+   fc::variant binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const;
 
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   bytes       variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path = false )const;
+   bytes       variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const;
    bytes       variant_to_binary( const std::string_view& type, const fc::variant& var, const yield_function_t& yield, bool short_path = false )const;
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   void        variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const fc::microseconds& max_serialization_time, bool short_path = false )const;
+   void        variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const;
    void        variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const yield_function_t& yield, bool short_path = false )const;
 
    template<typename T, typename Resolver>
-   static void to_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield );
+   static void to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield );
    template<typename T, typename Resolver>
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   static void to_variant( const T& o, fc::variant& vo, Resolver resolver, const fc::microseconds& max_serialization_time );
+   static void to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time );
 
    template<typename T, typename Resolver>
-   static void to_log_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield );
+   static void to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield );
+   template<typename T, typename Resolver>
+   static void to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time );
 
    template<typename T, typename Resolver>
-   static void from_variant( const fc::variant& v, T& o, Resolver resolver, const yield_function_t& yield );
+   static void from_variant( const fc::variant& v, T& o, const Resolver& resolver, const yield_function_t& yield );
    template<typename T, typename Resolver>
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   static void from_variant( const fc::variant& v, T& o, Resolver resolver, const fc::microseconds& max_serialization_time );
+   static void from_variant( const fc::variant& v, T& o, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time );
 
    template<typename Vec>
    static bool is_empty_abi(const Vec& abi_vec)
@@ -120,14 +117,10 @@ struct abi_serializer {
    static constexpr size_t max_recursion_depth = 32; // arbitrary depth to prevent infinite recursion
 
    // create standard yield function that checks for max_serialization_time and max_recursion_depth.
-   // now() deadline caputered at time of this call
+   // restricts serialization time from creation of yield function until serialization is complete.
+   // now() deadline captured at time of this call
    static yield_function_t create_yield_function(const fc::microseconds& max_serialization_time) {
-      fc::time_point deadline = fc::time_point::now();
-      if( max_serialization_time > fc::microseconds::maximum() - deadline.time_since_epoch() ) {
-         deadline = fc::time_point::maximum();
-      } else {
-         deadline += max_serialization_time;
-      }
+      fc::time_point deadline = fc::time_point::now().safe_add(max_serialization_time);
       return [max_serialization_time, deadline](size_t recursion_depth) {
          EOS_ASSERT( recursion_depth < max_recursion_depth, abi_recursion_depth_exception,
                      "recursive definition, max_recursion_depth ${r} ", ("r", max_recursion_depth) );
@@ -137,6 +130,13 @@ struct abi_serializer {
       };
    }
 
+   static yield_function_t create_depth_yield_function() {
+      return [](size_t recursion_depth) {
+         EOS_ASSERT( recursion_depth < max_recursion_depth, abi_recursion_depth_exception,
+                     "recursive definition, max_recursion_depth ${r} ", ("r", max_recursion_depth) );
+      };
+   }
+
 private:
 
    map<type_name, type_name, std::less<>>     typedefs;
@@ -172,8 +172,9 @@ private:
 namespace impl {
    const static size_t hex_log_max_size = 64;
    struct abi_traverse_context {
-      explicit abi_traverse_context( abi_serializer::yield_function_t yield )
+      abi_traverse_context( abi_serializer::yield_function_t yield, fc::microseconds max_action_data_serialization )
       : yield(std::move( yield ))
+      , max_action_serialization_time(max_action_data_serialization)
       {
       }
 
@@ -187,7 +188,9 @@ namespace impl {
 
    protected:
       abi_serializer::yield_function_t  yield;
-      size_t                            recursion_depth = 0;
+      // if set then restricts each individual action data serialization
+      fc::microseconds                  max_action_serialization_time;
+      size_t                            recursion_depth = 1;
       bool                              log = false;
    };
 
@@ -226,8 +229,8 @@ namespace impl {
    using path_item = std::variant<empty_path_item, array_index_path_item, field_path_item, variant_path_item>;
 
    struct abi_traverse_context_with_path : public abi_traverse_context {
-      abi_traverse_context_with_path( const abi_serializer& abis, abi_serializer::yield_function_t yield, const std::string_view& type )
-      : abi_traverse_context( std::move( yield ) ), abis(abis)
+      abi_traverse_context_with_path( const abi_serializer& abis, abi_serializer::yield_function_t yield, fc::microseconds max_action_data_serialization_time, const std::string_view& type )
+      : abi_traverse_context( std::move( yield ), max_action_data_serialization_time ), abis(abis)
       {
          set_path_root(type);
       }
@@ -263,6 +266,22 @@ namespace impl {
       using abi_traverse_context_with_path::abi_traverse_context_with_path;
    };
 
+   struct action_data_to_variant_context : public binary_to_variant_context {
+      action_data_to_variant_context( const abi_serializer& abis, const abi_traverse_context& ctx, const std::string_view& type )
+            : binary_to_variant_context(abis, ctx, type)
+      {
+         short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place
+         if (max_action_serialization_time.count() > 0) {
+            fc::time_point deadline = fc::time_point::now().safe_add(max_action_serialization_time);
+            yield = [deadline, y=yield, max=max_action_serialization_time](size_t depth) {
+               y(depth); // call provided yield that might include an overall time limit or not
+               EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception,
+                           "serialization action data time limit ${t}us exceeded", ("t", max) );
+            };
+         }
+      }
+   };
+
    struct variant_to_binary_context : public abi_traverse_context_with_path {
       using abi_traverse_context_with_path::abi_traverse_context_with_path;
 
@@ -335,7 +354,7 @@ namespace impl {
        * and can be degraded to the normal ::to_variant(...) processing
        */
       template<typename M, typename Resolver, not_require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const M& v, Resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver&, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          mvo(name,v);
@@ -346,14 +365,14 @@ namespace impl {
        * for these types we create new ABI aware visitors
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const M& v, Resolver resolver, abi_traverse_context& ctx );
+      static void add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver& resolver, abi_traverse_context& ctx );
 
       /**
        * template which overloads add for vectors of types which contain ABI information in their trees
        * for these members we call ::add in order to trigger further processing
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const vector<M>& v, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const vector<M>& v, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          vector<fc::variant> array;
@@ -372,7 +391,7 @@ namespace impl {
       * for these members we call ::add in order to trigger further processing
       */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const deque<M>& v, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const deque<M>& v, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          deque<fc::variant> array;
@@ -389,7 +408,7 @@ namespace impl {
        * for these members we call ::add in order to trigger further processing
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const std::shared_ptr<M>& v, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const std::shared_ptr<M>& v, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          if( !v ) return;
@@ -402,10 +421,10 @@ namespace impl {
       struct add_static_variant
       {
          mutable_variant_object& obj_mvo;
-         Resolver& resolver;
+         const Resolver& resolver;
          abi_traverse_context& ctx;
 
-         add_static_variant( mutable_variant_object& o, Resolver& r, abi_traverse_context& ctx )
+         add_static_variant( mutable_variant_object& o, const Resolver& r, abi_traverse_context& ctx )
                :obj_mvo(o), resolver(r), ctx(ctx) {}
 
          typedef void result_type;
@@ -416,7 +435,7 @@ namespace impl {
       };
 
       template<typename Resolver, typename... Args>
-      static void add( mutable_variant_object &mvo, const char* name, const std::variant<Args...>& v, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const std::variant<Args...>& v, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          mutable_variant_object obj_mvo;
@@ -426,7 +445,7 @@ namespace impl {
       }
 
       template<typename Resolver>
-      static bool add_special_logging( mutable_variant_object& mvo, const char* name, const action& act, Resolver& resolver, abi_traverse_context& ctx ) {
+      static bool add_special_logging( mutable_variant_object& mvo, const char* name, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) {
          if( !ctx.is_logging() ) return false;
 
          try {
@@ -455,7 +474,7 @@ namespace impl {
        * @return
        */
       template<typename Resolver>
-      static void add( mutable_variant_object &out, const char* name, const action& act, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &out, const char* name, const action& act, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<action>::total_member_count == 4);
          auto h = ctx.enter_scope();
@@ -485,16 +504,16 @@ namespace impl {
          };
 
          try {
-            auto abi = resolver(act.account);
-            if (abi) {
-               auto type = abi->get_action_type(act.name);
+            auto abi_optional = resolver(act.account);
+            if (abi_optional) {
+               const abi_serializer& abi = *abi_optional;
+               auto type = abi.get_action_type(act.name);
                if (!type.empty()) {
                   try {
-                     binary_to_variant_context _ctx(*abi, ctx, type);
-                     _ctx.short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place
-                     mvo( "data", abi->_binary_to_variant( type, act.data, _ctx ));
+                     action_data_to_variant_context _ctx(abi, ctx, type);
+                     mvo( "data", abi._binary_to_variant( type, act.data, _ctx ));
                   } catch(...) {
-                     // any failure to serialize data, then leave as not serailzed
+                     // any failure to serialize data, then leave as not serialized
                      set_hex_data(mvo, "data", act.data);
                   }
                } else {
@@ -520,7 +539,7 @@ namespace impl {
        * @return
        */
       template<typename Resolver>
-      static void add( mutable_variant_object& out, const char* name, const action_trace& act_trace, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object& out, const char* name, const action_trace& act_trace, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<action_trace>::total_member_count == 17);
          auto h = ctx.enter_scope();
@@ -546,13 +565,13 @@ namespace impl {
          mvo("return_value_hex_data", act_trace.return_value);
          auto act = act_trace.act;
          try {
-            auto abi = resolver(act.account);
-            if (abi) {
-               auto type = abi->get_action_result_type(act.name);
+            auto abi_optional = resolver(act.account);
+            if (abi_optional) {
+               const abi_serializer& abi = *abi_optional;
+               auto type = abi.get_action_result_type(act.name);
                if (!type.empty()) {
-                  binary_to_variant_context _ctx(*abi, ctx, type);
-                  _ctx.short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place
-                  mvo( "return_value_data", abi->_binary_to_variant( type, act_trace.return_value, _ctx ));
+                  action_data_to_variant_context _ctx(abi, ctx, type);
+                  mvo( "return_value_data", abi._binary_to_variant( type, act_trace.return_value, _ctx ));
                }
             }
          } catch(...) {}
@@ -569,7 +588,7 @@ namespace impl {
        * @return
        */
       template<typename Resolver>
-      static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<packed_transaction>::total_member_count == 4);
          auto h = ctx.enter_scope();
@@ -593,7 +612,7 @@ namespace impl {
        * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of trx.transaction_extensions
        */
       template<typename Resolver>
-      static void add( mutable_variant_object &out, const char* name, const transaction& trx, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &out, const char* name, const transaction& trx, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<transaction>::total_member_count == 9);
          auto h = ctx.enter_scope();
@@ -624,7 +643,7 @@ namespace impl {
        * block.header_extensions and block.block_extensions
        */
       template<typename Resolver>
-      static void add( mutable_variant_object &out, const char* name, const signed_block& block, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &out, const char* name, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<signed_block>::total_member_count == 12);
          auto h = ctx.enter_scope();
@@ -648,7 +667,7 @@ namespace impl {
             for (auto feature : new_protocol_features) {
                mutable_variant_object feature_mvo;
                add(feature_mvo, "feature_digest", feature, resolver, ctx);
-               pf_array.push_back(feature_mvo);
+               pf_array.push_back(std::move(feature_mvo));
             }
             mvo("new_protocol_features", pf_array);
          }
@@ -678,13 +697,13 @@ namespace impl {
     * this will degrade to the common fc::to_variant as soon as the type no longer contains
     * ABI related info
     *
-    * @tparam Reslover - callable with the signature (const name& code_account) -> std::optional<abi_def>
+    * @tparam Resolver - callable with the signature (const name& code_account) -> std::optional<abi_def>
     */
    template<typename T, typename Resolver>
    class abi_to_variant_visitor
    {
       public:
-         abi_to_variant_visitor( mutable_variant_object& _mvo, const T& _val, Resolver _resolver, abi_traverse_context& _ctx )
+         abi_to_variant_visitor( mutable_variant_object& _mvo, const T& _val, const Resolver& _resolver, abi_traverse_context& _ctx )
          :_vo(_mvo)
          ,_val(_val)
          ,_resolver(_resolver)
@@ -707,7 +726,7 @@ namespace impl {
       private:
          mutable_variant_object& _vo;
          const T& _val;
-         Resolver _resolver;
+         const Resolver& _resolver;
          abi_traverse_context& _ctx;
    };
 
@@ -717,7 +736,7 @@ namespace impl {
        * and can be degraded to the normal ::from_variant(...) processing
        */
       template<typename M, typename Resolver, not_require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, M& o, Resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, M& o, const Resolver&, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          from_variant(v, o);
@@ -728,14 +747,14 @@ namespace impl {
        * for these types we create new ABI aware visitors
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, M& o, Resolver resolver, abi_traverse_context& ctx );
+      static void extract( const fc::variant& v, M& o, const Resolver& resolver, abi_traverse_context& ctx );
 
       /**
        * template which overloads extract for vectors of types which contain ABI information in their trees
        * for these members we call ::extract in order to trigger further processing
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, vector<M>& o, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, vector<M>& o, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variants& array = v.get_array();
@@ -753,7 +772,7 @@ namespace impl {
   * for these members we call ::extract in order to trigger further processing
   */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, deque<M>& o, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, deque<M>& o, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variants& array = v.get_array();
@@ -771,7 +790,7 @@ namespace impl {
        * for these members we call ::extract in order to trigger further processing
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, std::shared_ptr<M>& o, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, std::shared_ptr<M>& o, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variant_object& vo = v.get_object();
@@ -786,7 +805,7 @@ namespace impl {
        * exploded and processed explicitly
        */
       template<typename Resolver>
-      static void extract( const fc::variant& v, action& act, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, action& act, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variant_object& vo = v.get_object();
@@ -806,13 +825,14 @@ namespace impl {
                from_variant(data, act.data);
                valid_empty_data = act.data.empty();
             } else if ( data.is_object() ) {
-               auto abi = resolver(act.account);
-               if (abi) {
-                  auto type = abi->get_action_type(act.name);
+               auto abi_optional = resolver(act.account);
+               if (abi_optional) {
+                  const abi_serializer& abi = *abi_optional;
+                  auto type = abi.get_action_type(act.name);
                   if (!type.empty()) {
-                     variant_to_binary_context _ctx(*abi, ctx, type);
+                     variant_to_binary_context _ctx(abi, ctx, type);
                      _ctx.short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place
-                     act.data = std::move( abi->_variant_to_binary( type, data, _ctx ));
+                     act.data = abi._variant_to_binary( type, data, _ctx );
                      valid_empty_data = act.data.empty();
                   }
                }
@@ -833,7 +853,7 @@ namespace impl {
       }
 
       template<typename Resolver>
-      static void extract( const fc::variant& v, packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variant_object& vo = v.get_object();
@@ -890,7 +910,7 @@ namespace impl {
    class abi_from_variant_visitor : public reflector_init_visitor<T>
    {
       public:
-         abi_from_variant_visitor( const variant_object& _vo, T& v, Resolver _resolver, abi_traverse_context& _ctx )
+         abi_from_variant_visitor( const variant_object& _vo, T& v, const Resolver& _resolver, abi_traverse_context& _ctx )
          : reflector_init_visitor<T>(v)
          ,_vo(_vo)
          ,_resolver(_resolver)
@@ -914,12 +934,12 @@ namespace impl {
 
       private:
          const variant_object& _vo;
-         Resolver _resolver;
+         const Resolver& _resolver;
          abi_traverse_context& _ctx;
    };
 
    template<typename M, typename Resolver, require_abi_t<M>>
-   void abi_to_variant::add( mutable_variant_object &mvo, const char* name, const M& v, Resolver resolver, abi_traverse_context& ctx )
+   void abi_to_variant::add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver& resolver, abi_traverse_context& ctx )
    {
       auto h = ctx.enter_scope();
       mutable_variant_object member_mvo;
@@ -928,7 +948,7 @@ namespace impl {
    }
 
    template<typename M, typename Resolver, require_abi_t<M>>
-   void abi_from_variant::extract( const fc::variant& v, M& o, Resolver resolver, abi_traverse_context& ctx )
+   void abi_from_variant::extract( const fc::variant& v, M& o, const Resolver& resolver, abi_traverse_context& ctx )
    {
       auto h = ctx.enter_scope();
       const variant_object& vo = v.get_object();
@@ -937,38 +957,158 @@ namespace impl {
 } /// namespace eosio::chain::impl
 
 template<typename T, typename Resolver>
-void abi_serializer::to_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield ) try {
+void abi_serializer::to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield ) try {
    mutable_variant_object mvo;
-   impl::abi_traverse_context ctx( yield );
+   impl::abi_traverse_context ctx( yield, fc::microseconds{} );
    impl::abi_to_variant::add(mvo, "_", o, resolver, ctx);
    vo = std::move(mvo["_"]);
 } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: ${type}", ("type", boost::core::demangle( typeid(o).name() ) ))
 
 template<typename T, typename Resolver>
-void abi_serializer::to_variant( const T& o, fc::variant& vo, Resolver resolver, const fc::microseconds& max_serialization_time ) {
-   to_variant( o, vo, resolver, create_yield_function(max_serialization_time) );
-}
+void abi_serializer::to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try {
+   mutable_variant_object mvo;
+   impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time );
+   impl::abi_to_variant::add(mvo, "_", o, resolver, ctx);
+   vo = std::move(mvo["_"]);
+} FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: ${type}", ("type", boost::core::demangle( typeid(o).name() ) ))
 
 template<typename T, typename Resolver>
-void abi_serializer::to_log_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield ) try {
+void abi_serializer::to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield ) try {
     mutable_variant_object mvo;
-    impl::abi_traverse_context ctx( yield );
+    impl::abi_traverse_context ctx( yield, fc::microseconds{} );
     ctx.logging();
     impl::abi_to_variant::add(mvo, "_", o, resolver, ctx);
     vo = std::move(mvo["_"]);
 } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: ${type}", ("type", boost::core::demangle( typeid(o).name() ) ))
 
+template<typename T, typename Resolver>
+void abi_serializer::to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try {
+   mutable_variant_object mvo;
+   impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time );
+   ctx.logging();
+   impl::abi_to_variant::add(mvo, "_", o, resolver, ctx);
+   vo = std::move(mvo["_"]);
+} FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: ${type}", ("type", boost::core::demangle( typeid(o).name() ) ))
 
 template<typename T, typename Resolver>
-void abi_serializer::from_variant( const fc::variant& v, T& o, Resolver resolver, const yield_function_t& yield ) try {
-   impl::abi_traverse_context ctx( yield );
+void abi_serializer::from_variant( const fc::variant& v, T& o, const Resolver& resolver, const yield_function_t& yield ) try {
+   impl::abi_traverse_context ctx( yield, fc::microseconds{} );
    impl::abi_from_variant::extract(v, o, resolver, ctx);
 } FC_RETHROW_EXCEPTIONS(error, "Failed to deserialize variant", ("variant",v))
 
 template<typename T, typename Resolver>
-void abi_serializer::from_variant( const fc::variant& v, T& o, Resolver resolver, const fc::microseconds& max_serialization_time ) {
-   from_variant( v, o, resolver, create_yield_function(max_serialization_time) );
-}
+void abi_serializer::from_variant( const fc::variant& v, T& o, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try {
+   impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time );
+   impl::abi_from_variant::extract(v, o, resolver, ctx);
+} FC_RETHROW_EXCEPTIONS(error, "Failed to deserialize variant", ("variant",v))
 
+using abi_serializer_cache_t = std::unordered_map<account_name, std::optional<abi_serializer>>;
+using resolver_fn_t = std::function<std::optional<abi_serializer>(const account_name& name)>;
+   
+class abi_resolver {
+public:
+   explicit abi_resolver(abi_serializer_cache_t&& abi_serializers) :
+      abi_serializers(std::move(abi_serializers))
+   {}
 
-} } // eosio::chain
+   std::optional<std::reference_wrapper<const abi_serializer>> operator()(const account_name& account) const {
+      auto it = abi_serializers.find(account);
+      if (it != abi_serializers.end() && it->second)
+         return std::reference_wrapper<const abi_serializer>(*it->second);
+      return {};
+   };
+
+private:
+   abi_serializer_cache_t abi_serializers;
+};
+
+class abi_serializer_cache_builder {
+public:
+   explicit abi_serializer_cache_builder(resolver_fn_t resolver) :
+      resolver_(std::move(resolver))
+   {
+   }
+
+   abi_serializer_cache_builder(const abi_serializer_cache_builder&) = delete;
+
+   abi_serializer_cache_builder&& add_serializers(const chain::signed_block_ptr& block) && {
+      for( const auto& receipt: block->transactions ) {
+         if( std::holds_alternative<chain::packed_transaction>( receipt.trx ) ) {
+            const auto& pt = std::get<chain::packed_transaction>( receipt.trx );
+            const auto& t = pt.get_transaction();
+            for( const auto& a: t.actions )
+               add_to_cache( a );
+            for( const auto& a: t.context_free_actions )
+               add_to_cache( a );
+         }
+      }
+      return std::move(*this);
+   }
+
+   abi_serializer_cache_builder&& add_serializers(const transaction_trace_ptr& trace_ptr) && {
+      for( const auto& trace: trace_ptr->action_traces ) {
+         add_to_cache(trace.act);
+      }
+      return std::move(*this);
+   }
+
+   abi_serializer_cache_t&& get() && {
+      return std::move(abi_serializers);
+   }
+
+private:
+   void add_to_cache(const chain::action& a) {
+      auto it = abi_serializers.find( a.account );
+      if( it == abi_serializers.end() ) {
+         try {
+            abi_serializers.emplace_hint( it, a.account, resolver_( a.account ) );
+         } catch( ... ) {
+            // keep behavior of not throwing on invalid abi, will result in hex data
+         }
+      }
+   }
+
+   resolver_fn_t resolver_;
+   abi_serializer_cache_t abi_serializers;
+};
+
+/*
+ * This is equivalent to a resolver, except that everytime the abi_serializer for an account 
+ * is retrieved, it is stored in an unordered_map, so we won't waste time retrieving it again.
+ * This is handy when parsing packed_transactions received in a fc::variant.
+ */
+class caching_resolver {
+public:
+   explicit caching_resolver(resolver_fn_t resolver) :
+      resolver_(std::move(resolver))
+   {
+   }
+
+   // make it non-copiable (we should only move it for performance reasons)
+   caching_resolver(const caching_resolver&) = delete;
+   caching_resolver& operator=(const caching_resolver&) = delete;
+
+   std::optional<std::reference_wrapper<const abi_serializer>> operator()(const account_name& account) const {
+      auto it = abi_serializers.find(account);
+      if (it != abi_serializers.end()) {
+         if (it->second)
+            return *it->second;
+         return {};
+      }
+      auto serializer = resolver_(account);
+      auto& dest = abi_serializers[account]; // add entry regardless
+      if (serializer) {
+         // we got a serializer, so move it into the cache
+         dest = abi_serializer_cache_t::mapped_type{std::move(*serializer)};
+         return *dest; // and return a reference to it
+      }
+      return {}; 
+   };
+
+private:
+   const resolver_fn_t resolver_;
+   mutable abi_serializer_cache_t abi_serializers;
+};
+      
+
+} // eosio::chain

libraries/chain/include/eosio/chain/authority.hpp

@@ -188,7 +188,7 @@ struct authority {
    }
 
    authority( uint32_t t, vector<key_weight> k, vector<permission_level_weight> p = {}, vector<wait_weight> w = {} )
-   :threshold(t),keys(move(k)),accounts(move(p)),waits(move(w)){}
+   :threshold(t),keys(std::move(k)),accounts(std::move(p)),waits(std::move(w)){}
    authority(){}
 
    uint32_t                          threshold = 0;

libraries/chain/include/eosio/chain/block_log.hpp

@@ -42,7 +42,7 @@ namespace eosio { namespace chain {
 
    class block_log {
       public:
-         explicit block_log(const fc::path& data_dir, const block_log_config& config = block_log_config{});
+         explicit block_log(const std::filesystem::path& data_dir, const block_log_config& config = block_log_config{});
          block_log(block_log&& other) noexcept;
          ~block_log();
 
@@ -80,13 +80,21 @@ namespace eosio { namespace chain {
           * All static methods expected to be called on quiescent block log
           */
 
-         static fc::path repair_log( const fc::path& data_dir, uint32_t truncate_at_block = 0, const char* reversible_block_dir_name="" );
+         static std::filesystem::path repair_log( const std::filesystem::path& data_dir, uint32_t truncate_at_block = 0, const char* reversible_block_dir_name="" );
 
-         static std::optional<genesis_state> extract_genesis_state( const fc::path& data_dir );
+         using chain_context = std::variant<genesis_state, chain_id_type>;
+         static std::optional<chain_context> extract_chain_context(const std::filesystem::path& data_dir,
+                                                                   const std::filesystem::path& retained_dir);
 
-         static chain_id_type extract_chain_id( const fc::path& data_dir );
+         static std::optional<genesis_state>
+         extract_genesis_state(const std::filesystem::path& data_dir,
+                               const std::filesystem::path& retained_dir = std::filesystem::path{});
 
-         static void construct_index(const fc::path& block_file_name, const fc::path& index_file_name);
+         static std::optional<chain_id_type>
+         extract_chain_id(const std::filesystem::path& data_dir,
+                          const std::filesystem::path& retained_dir = std::filesystem::path{});
+
+         static void construct_index(const std::filesystem::path& block_file_name, const std::filesystem::path& index_file_name);
 
          static bool contains_genesis_state(uint32_t version, uint32_t first_block_num);
 
@@ -94,12 +102,12 @@ namespace eosio { namespace chain {
 
          static bool is_supported_version(uint32_t version);
 
-         static bool is_pruned_log(const fc::path& data_dir);
+         static bool is_pruned_log(const std::filesystem::path& data_dir);
 
-         static void extract_block_range(const fc::path& block_dir, const fc::path&output_dir, block_num_type start, block_num_type end);
+         static void extract_block_range(const std::filesystem::path& block_dir, const std::filesystem::path&output_dir, block_num_type start, block_num_type end);
 
-         static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block);
-         static int  trim_blocklog_end(const fc::path& block_dir, uint32_t n);
+         static bool trim_blocklog_front(const std::filesystem::path& block_dir, const std::filesystem::path& temp_dir, uint32_t truncate_at_block);
+         static int  trim_blocklog_end(const std::filesystem::path& block_dir, uint32_t n);
 
          // used for unit test to generate older version blocklog
          static void set_initial_version(uint32_t);
@@ -109,10 +117,10 @@ namespace eosio { namespace chain {
          /**
           * @param n Only test 1 block out of every n blocks. If n is 0, the interval is adjusted so that at most 8 blocks are tested.
           */
-         static void smoke_test(const fc::path& block_dir, uint32_t n);
+         static void smoke_test(const std::filesystem::path& block_dir, uint32_t n);
 
-         static void split_blocklog(const fc::path& block_dir, const fc::path& dest_dir, uint32_t stride);
-         static void merge_blocklogs(const fc::path& block_dir, const fc::path& dest_dir);
+         static void split_blocklog(const std::filesystem::path& block_dir, const std::filesystem::path& dest_dir, uint32_t stride);
+         static void merge_blocklogs(const std::filesystem::path& block_dir, const std::filesystem::path& dest_dir);
    private:
          std::unique_ptr<detail::block_log_impl> my;
    };

libraries/chain/include/eosio/chain/block_log_config.hpp

@@ -1,20 +1,18 @@
 #pragma once
-#include <boost/filesystem/path.hpp>
 #include <variant>
 
 namespace eosio { namespace chain {
 
-   namespace bfs = boost::filesystem;
 
    struct basic_blocklog_config {};
 
    struct empty_blocklog_config {};
 
    struct partitioned_blocklog_config {
-      bfs::path retained_dir;
-      bfs::path archive_dir;
-      uint32_t  stride                  = UINT32_MAX;
-      uint32_t  max_retained_files      = UINT32_MAX;
+      std::filesystem::path retained_dir;
+      std::filesystem::path archive_dir;
+      uint32_t              stride             = UINT32_MAX;
+      uint32_t              max_retained_files = UINT32_MAX;
    };
 
    struct prune_blocklog_config {

libraries/chain/include/eosio/chain/config.hpp

@@ -76,7 +76,7 @@ const static uint32_t   default_max_inline_action_size               = 512 * 102
 const static uint16_t   default_max_inline_action_depth              = 4;
 const static uint16_t   default_max_auth_depth                       = 6;
 const static uint32_t   default_sig_cpu_bill_pct                     = 50 * percent_1; // billable percentage of signature recovery
-const static uint32_t   default_block_cpu_effort_pct                 = 80 * percent_1; // percentage of block time used for producing block
+const static uint32_t   default_block_cpu_effort_pct                 = 90 * percent_1; // percentage of block time used for producing block
 const static uint16_t   default_controller_thread_pool_size          = 2;
 const static uint32_t   default_max_variable_signature_length        = 16384u;
 const static uint32_t   default_max_nonprivileged_inline_action_size = 4 * 1024; // 4 KB

libraries/chain/include/eosio/chain/controller.hpp

@@ -39,6 +39,7 @@ namespace eosio { namespace chain {
    class permission_object;
    class account_object;
    class deep_mind_handler;
+   class subjective_billing;
    using resource_limits::resource_limits_manager;
    using apply_handler = std::function<void(apply_context&)>;
    using forked_branch_callback = std::function<void(const branch_type&)>;
@@ -49,7 +50,8 @@ namespace eosio { namespace chain {
 
    enum class db_read_mode {
       HEAD,
-      IRREVERSIBLE
+      IRREVERSIBLE,
+      SPECULATIVE
    };
 
    enum class validation_mode {
@@ -196,6 +198,8 @@ namespace eosio { namespace chain {
          const authorization_manager&          get_authorization_manager()const;
          authorization_manager&                get_mutable_authorization_manager();
          const protocol_feature_manager&       get_protocol_feature_manager()const;
+         const subjective_billing&             get_subjective_billing()const;
+         subjective_billing&                   get_mutable_subjective_billing();
          uint32_t                              get_max_nonprivileged_inline_action_size()const;
 
          const flat_set<account_name>&   get_actor_whitelist() const;
@@ -223,6 +227,7 @@ namespace eosio { namespace chain {
          block_id_type        fork_db_head_block_id()const;
 
          time_point                     pending_block_time()const;
+         block_timestamp_type           pending_block_timestamp()const;
          account_name                   pending_block_producer()const;
          const block_signing_authority& pending_block_signing_authority()const;
          std::optional<block_id_type>   pending_producer_block_id()const;
@@ -343,26 +348,6 @@ namespace eosio { namespace chain {
          const apply_handler* find_apply_handler( account_name contract, scope_name scope, action_name act )const;
          wasm_interface& get_wasm_interface();
 
-
-         std::optional<abi_serializer> get_abi_serializer( account_name n, const abi_serializer::yield_function_t& yield )const {
-            if( n.good() ) {
-               try {
-                  const auto& a = get_account( n );
-                  if( abi_def abi; abi_serializer::to_abi( a.abi, abi ))
-                     return abi_serializer( std::move(abi), yield );
-               } FC_CAPTURE_AND_LOG((n))
-            }
-            return std::optional<abi_serializer>();
-         }
-
-         template<typename T>
-         fc::variant to_variant_with_abi( const T& obj, const abi_serializer::yield_function_t& yield )const {
-            fc::variant pretty_output;
-            abi_serializer::to_variant( obj, pretty_output,
-                                        [&]( account_name n ){ return get_abi_serializer( n, yield ); }, yield );
-            return pretty_output;
-         }
-
       static chain_id_type extract_chain_id(snapshot_reader& snapshot);
 
       static std::optional<chain_id_type> extract_chain_id_from_db( const path& state_dir );
@@ -373,6 +358,10 @@ namespace eosio { namespace chain {
       void set_db_read_only_mode();
       void unset_db_read_only_mode();
       void init_thread_local_data();
+      void set_to_write_window();
+      void set_to_read_window();
+      bool is_write_window() const;
+      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
 
       private:
          friend class apply_context;

libraries/chain/include/eosio/chain/database_utils.hpp

@@ -170,7 +170,7 @@ namespace fc {
 
    inline
    void from_variant( const variant& v, eosio::chain::shared_string& s ) {
-      string _s;
+      std::string _s;
       from_variant(v, _s);
       s = eosio::chain::shared_string(_s.begin(), _s.end(), s.get_allocator());
    }
@@ -182,7 +182,7 @@ namespace fc {
 
    inline
    void from_variant( const variant& v, eosio::chain::shared_blob& b ) {
-      string _s = base64_decode(v.as_string());
+      std::string _s = base64_decode(v.as_string());
       b = eosio::chain::shared_blob(_s.begin(), _s.end(), b.get_allocator());
    }
 

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -87,6 +87,26 @@
       NEXT(e.dynamic_copy_exception());\
    }
 
+/**
+ * Capture all exceptions and return return_type which is constructible from a fc::exception_ptr
+ */
+#define CATCH_AND_RETURN(return_type)\
+   catch ( const fc::exception& err ) {\
+      return return_type(err.dynamic_copy_exception());\
+   } catch ( const std::exception& e ) {\
+      fc::exception fce( \
+         FC_LOG_MESSAGE( warn, "rethrow ${what}: ", ("what",e.what())),\
+         fc::std_exception_code,\
+         BOOST_CORE_TYPEID(e).name(),\
+         e.what() ) ;\
+      return return_type(fce.dynamic_copy_exception());\
+   } catch( ... ) {\
+      fc::unhandled_exception e(\
+         FC_LOG_MESSAGE(warn, "rethrow"),\
+         std::current_exception());\
+      return return_type(e.dynamic_copy_exception());\
+   }
+
 #define EOS_RECODE_EXC( cause_type, effect_type ) \
    catch( const cause_type& e ) \
    { throw( effect_type( e.what(), e.get_log() ) ); }
@@ -361,6 +381,10 @@ namespace eosio { namespace chain {
                                     3080007, "Transaction exceeded the current greylisted account network usage limit" )
       FC_DECLARE_DERIVED_EXCEPTION( greylist_cpu_usage_exceeded, resource_exhausted_exception,
                                     3080008, "Transaction exceeded the current greylisted account CPU usage limit" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_temporary_failure, resource_exhausted_exception,
+                                    3080009, "Read-only transaction eos-vm-oc compile temporary failure" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_permanent_failure, resource_exhausted_exception,
+                                    3080010, "Read-only transaction eos-vm-oc compile permanent failure" )
 
       FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception,
                                     3081001, "Transaction reached the deadline set due to leeway on account CPU limits" )
@@ -579,6 +603,8 @@ namespace eosio { namespace chain {
                                     3170014, "Snapshot request not found" )
       FC_DECLARE_DERIVED_EXCEPTION( invalid_snapshot_request,  producer_exception,
                                     3170015, "Invalid snapshot request" )
+      FC_DECLARE_DERIVED_EXCEPTION( snapshot_execution_exception,  producer_exception,
+                                    3170016, "Snapshot execution exception" )
 
    FC_DECLARE_DERIVED_EXCEPTION( reversible_blocks_exception,           chain_exception,
                                  3180000, "Reversible Blocks exception" )

libraries/chain/include/eosio/chain/fork_database.hpp

@@ -22,7 +22,7 @@ namespace eosio { namespace chain {
    class fork_database {
       public:
 
-         explicit fork_database( const fc::path& data_dir );
+         explicit fork_database( const std::filesystem::path& data_dir );
          ~fork_database();
 
          void open( const std::function<void( block_timestamp_type,

libraries/chain/include/eosio/chain/log_catalog.hpp

@@ -1,6 +1,5 @@
 #pragma once
 #include <boost/container/flat_map.hpp>
-#include <boost/filesystem/path.hpp>
 #include <boost/iostreams/device/mapped_file.hpp>
 #include <fc/io/cfile.hpp>
 #include <fc/io/datastream.hpp>
@@ -9,19 +8,19 @@
 namespace eosio {
 namespace chain {
 
-namespace bfs = boost::filesystem;
 
 template <typename Lambda>
-void for_each_file_in_dir_matches(const bfs::path& dir, std::string pattern, Lambda&& lambda) {
+void for_each_file_in_dir_matches(const std::filesystem::path& dir, std::string pattern, Lambda&& lambda) {
    const std::regex        my_filter(pattern);
    std::smatch             what;
-   bfs::directory_iterator end_itr; // Default ctor yields past-the-end
-   for (bfs::directory_iterator p(dir); p != end_itr; ++p) {
+   std::filesystem::directory_iterator end_itr; // Default ctor yields past-the-end
+   for (std::filesystem::directory_iterator p(dir); p != end_itr; ++p) {
       // Skip if not a file
-      if (!bfs::is_regular_file(p->status()))
+      if (!std::filesystem::is_regular_file(p->status()))
          continue;
+      auto name = p->path().filename().string();
       // skip if it does not match the pattern
-      if (!std::regex_match(p->path().filename().string(), what, my_filter))
+      if (!std::regex_match(name, what, my_filter))
          continue;
       lambda(p->path());
    }
@@ -29,7 +28,7 @@ void for_each_file_in_dir_matches(const bfs::path& dir, std::string pattern, Lam
 
 struct null_verifier {
    template <typename LogData>
-   void verify(const LogData&, const bfs::path&) {}
+   void verify(const LogData&, const std::filesystem::path&) {}
 };
 
 template <typename LogData, typename LogIndex, typename LogVerifier = null_verifier>
@@ -37,21 +36,21 @@ struct log_catalog {
    using block_num_t = uint32_t;
 
    struct mapped_type {
-      block_num_t last_block_num;
-      bfs::path   filename_base;
+      block_num_t           last_block_num;
+      std::filesystem::path filename_base;
    };
    using collection_t              = boost::container::flat_map<block_num_t, mapped_type>;
    using size_type                 = typename collection_t::size_type;
    static constexpr size_type npos = std::numeric_limits<size_type>::max();
 
-   bfs::path    retained_dir;
-   bfs::path    archive_dir;
-   size_type    max_retained_files = std::numeric_limits<size_type>::max();
-   collection_t collection;
-   size_type    active_index = npos;
-   LogData      log_data;
-   LogIndex     log_index;
-   LogVerifier  verifier;
+   std::filesystem::path retained_dir;
+   std::filesystem::path archive_dir;
+   size_type             max_retained_files = std::numeric_limits<size_type>::max();
+   collection_t          collection;
+   size_type             active_index = npos;
+   LogData               log_data;
+   LogIndex              log_index;
+   LogVerifier           verifier;
 
    bool empty() const { return collection.empty(); }
 
@@ -67,17 +66,17 @@ struct log_catalog {
       return collection.rbegin()->second.last_block_num;
    }
 
-   static bfs::path make_absolute_dir(const bfs::path& base_dir, bfs::path new_dir) {
+   static std::filesystem::path make_absolute_dir(const std::filesystem::path& base_dir, std::filesystem::path new_dir) {
       if (new_dir.is_relative())
          new_dir = base_dir / new_dir;
 
-      if (!bfs::is_directory(new_dir))
-         bfs::create_directories(new_dir);
+      if (!std::filesystem::is_directory(new_dir))
+         std::filesystem::create_directories(new_dir);
 
       return new_dir;
    }
 
-   void open(const bfs::path& log_dir, const bfs::path& retained_path, const bfs::path& archive_path, const char* name,
+   void open(const std::filesystem::path& log_dir, const std::filesystem::path& retained_path, const std::filesystem::path& archive_path, const char* name,
              const char* suffix_pattern = R"(-\d+-\d+\.log)") {
 
       retained_dir = make_absolute_dir(log_dir, retained_path.empty() ? log_dir : retained_path);
@@ -85,7 +84,7 @@ struct log_catalog {
          archive_dir = make_absolute_dir(log_dir, archive_path);
       }
 
-      for_each_file_in_dir_matches(retained_dir, std::string(name) + suffix_pattern, [this](bfs::path path) {
+      for_each_file_in_dir_matches(retained_dir, std::string(name) + suffix_pattern, [this](std::filesystem::path path) {
          auto log_path               = path;
          auto index_path             = path.replace_extension("index");
          auto path_without_extension = log_path.parent_path() / log_path.stem().string();
@@ -117,11 +116,11 @@ struct log_catalog {
       });
    }
 
-   bool index_matches_data(const bfs::path& index_path, LogData& log) const {
-      if (!bfs::exists(index_path))
+   bool index_matches_data(const std::filesystem::path& index_path, LogData& log) const {
+      if (!std::filesystem::exists(index_path))
          return false;
 
-      auto num_blocks_in_index = bfs::file_size(index_path) / sizeof(uint64_t);
+      auto num_blocks_in_index = std::filesystem::file_size(index_path) / sizeof(uint64_t);
       if (num_blocks_in_index != log.num_blocks())
          return false;
 
@@ -187,17 +186,17 @@ struct log_catalog {
       return {};
    }
 
-   static void rename_if_not_exists(bfs::path old_name, bfs::path new_name) {
-      if (!bfs::exists(new_name)) {
-         bfs::rename(old_name, new_name);
+   static void rename_if_not_exists(std::filesystem::path old_name, std::filesystem::path new_name) {
+      if (!std::filesystem::exists(new_name)) {
+         std::filesystem::rename(old_name, new_name);
       } else {
-         bfs::remove(old_name);
+         std::filesystem::remove(old_name);
          wlog("${new_name} already exists, just removing ${old_name}",
               ("old_name", old_name.string())("new_name", new_name.string()));
       }
    }
 
-   static void rename_bundle(bfs::path orig_path, bfs::path new_path) {
+   static void rename_bundle(std::filesystem::path orig_path, std::filesystem::path new_path) {
       rename_if_not_exists(orig_path.replace_extension(".log"), new_path.replace_extension(".log"));
       rename_if_not_exists(orig_path.replace_extension(".index"), new_path.replace_extension(".index"));
    }
@@ -209,12 +208,12 @@ struct log_catalog {
    /// invalidated and the mapping between the log data their block range would be wrong. This function is only used
    /// during the splitting of block log. Using this function for other purpose should make sure if the monotonically
    /// increasing block num guarantee can be met.
-   void add(uint32_t start_block_num, uint32_t end_block_num, const bfs::path& dir, const char* name) {
+   void add(uint32_t start_block_num, uint32_t end_block_num, const std::filesystem::path& dir, const char* name) {
 
       const int bufsize = 64;
       char      buf[bufsize];
       snprintf(buf, bufsize, "%s-%u-%u", name, start_block_num, end_block_num);
-      bfs::path new_path = retained_dir / buf;
+      std::filesystem::path new_path = retained_dir / buf;
       rename_bundle(dir / name, new_path);
       size_type items_to_erase = 0;
       collection.emplace(start_block_num, mapped_type{end_block_num, new_path});
@@ -226,8 +225,8 @@ struct log_catalog {
             auto orig_name = it->second.filename_base;
             if (archive_dir.empty()) {
                // delete the old files when no backup dir is specified
-               bfs::remove(orig_name.replace_extension("log"));
-               bfs::remove(orig_name.replace_extension("index"));
+               std::filesystem::remove(orig_name.replace_extension("log"));
+               std::filesystem::remove(orig_name.replace_extension("index"));
             } else {
                // move the the archive dir
                rename_bundle(orig_name, archive_dir / orig_name.filename());
@@ -246,14 +245,14 @@ struct log_catalog {
    /// from the catalog.
    ///
    /// \return if nonzero, it's the starting block number for the log/index bundle being renamed.
-   uint32_t truncate(uint32_t block_num, bfs::path new_name) {
+   uint32_t truncate(uint32_t block_num, std::filesystem::path new_name) {
       if (collection.empty())
          return 0;
 
       auto remove_files = [](typename collection_t::const_reference v) {
          auto name = v.second.filename_base;
-         bfs::remove(name.replace_extension("log"));
-         bfs::remove(name.replace_extension("index"));
+         std::filesystem::remove(name.replace_extension("log"));
+         std::filesystem::remove(name.replace_extension("index"));
       };
 
       active_index = npos;
@@ -266,8 +265,8 @@ struct log_catalog {
       } else {
          auto truncate_it = --it;
          auto name        = truncate_it->second.filename_base;
-         bfs::rename(name.replace_extension("log"), new_name.replace_extension("log"));
-         bfs::rename(name.replace_extension("index"), new_name.replace_extension("index"));
+         std::filesystem::rename(name.replace_extension("log"), new_name.replace_extension("log"));
+         std::filesystem::rename(name.replace_extension("index"), new_name.replace_extension("index"));
          std::for_each(truncate_it + 1, collection.end(), remove_files);
          auto result = truncate_it->first;
          collection.erase(truncate_it, collection.end());

libraries/chain/include/eosio/chain/log_index.hpp

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <boost/filesystem/path.hpp>
 #include <fc/io/cfile.hpp>
 
 namespace eosio {
@@ -14,11 +13,11 @@ class log_index {
    std::size_t num_blocks_ = 0;
  public:
    log_index() = default;
-   log_index(const boost::filesystem::path& path) {
+   log_index(const std::filesystem::path& path) {
       open(path);
    }
 
-   void open(const boost::filesystem::path& path) {
+   void open(const std::filesystem::path& path) {
       if (file_.is_open())
          file_.close();
       file_.set_file_path(path);
@@ -26,13 +25,13 @@ class log_index {
       file_.seek_end(0);
       num_blocks_ = file_.tellp()/ sizeof(uint64_t);
       EOS_ASSERT(file_.tellp() % sizeof(uint64_t) == 0, Exception,
-                 "The size of ${file} is not a multiple of sizeof(uint64_t)", ("file", path.generic_string()));
+                 "The size of ${file} is not a multiple of sizeof(uint64_t)", ("file", path));
    }
 
    bool is_open() const { return file_.is_open(); }
 
    uint64_t back() { return nth_block_position(num_blocks()-1); }
-   unsigned num_blocks() const { return num_blocks_; }
+   uint32_t num_blocks() const { return num_blocks_; }
    uint64_t nth_block_position(uint32_t n) {
       file_.seek(n*sizeof(uint64_t));
       uint64_t r;

libraries/chain/include/eosio/chain/pending_snapshot.hpp

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <eosio/chain/controller.hpp>
+#include <eosio/chain/types.hpp>
+
+#include <string>
+
+namespace eosio::chain {
+
+namespace fs = std::filesystem;
+
+template<typename T>
+class pending_snapshot {
+public:
+   using next_t = eosio::chain::next_function<T>;
+
+   pending_snapshot(const chain::block_id_type& block_id, const next_t& next, std::string pending_path, std::string final_path)
+       : block_id(block_id), next(next), pending_path(std::move(pending_path)), final_path(std::move(final_path)) {}
+
+   uint32_t get_height() const {
+      return chain::block_header::num_from_id(block_id);
+   }
+
+   static fs::path get_final_path(const chain::block_id_type& block_id, const fs::path& snapshots_dir) {
+      return snapshots_dir / fc::format_string("snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id));
+   }
+
+   static fs::path get_pending_path(const chain::block_id_type& block_id, const fs::path& snapshots_dir) {
+      return snapshots_dir / fc::format_string(".pending-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id));
+   }
+
+   static fs::path get_temp_path(const chain::block_id_type& block_id, const fs::path& snapshots_dir) {
+      return snapshots_dir / fc::format_string(".incomplete-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id));
+   }
+
+   T finalize(const chain::controller& chain) const {
+      auto block_ptr = chain.fetch_block_by_id(block_id);
+      auto in_chain = (bool) block_ptr;
+      std::error_code ec;
+
+      if(!in_chain) {
+         fs::remove(fs::path(pending_path), ec);
+         EOS_THROW(chain::snapshot_finalization_exception,
+                   "Snapshotted block was forked out of the chain.  ID: ${block_id}",
+                   ("block_id", block_id));
+      }
+
+      fs::rename(fs::path(pending_path), fs::path(final_path), ec);
+      EOS_ASSERT(!ec, chain::snapshot_finalization_exception,
+                 "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}",
+                 ("bn", get_height())("ec", ec.value())("message", ec.message()));
+
+      return {block_id, block_ptr->block_num(), block_ptr->timestamp, chain::chain_snapshot_header::current_version, final_path};
+   }
+
+   chain::block_id_type block_id;
+   next_t next;
+   std::string pending_path;
+   std::string final_path;
+};
+}// namespace eosio::chain

libraries/chain/include/eosio/chain/protocol_feature_manager.hpp

@@ -396,8 +396,8 @@ private:
    std::function<deep_mind_handler*(bool is_trx_transient)> _get_deep_mind_logger;
 };
 
-std::optional<builtin_protocol_feature> read_builtin_protocol_feature( const fc::path& p  );
-protocol_feature_set initialize_protocol_features( const fc::path& p, bool populate_missing_builtins = true );
+std::optional<builtin_protocol_feature> read_builtin_protocol_feature( const std::filesystem::path& p  );
+protocol_feature_set initialize_protocol_features( const std::filesystem::path& p, bool populate_missing_builtins = true );
 
 } } // namespace eosio::chain
 

libraries/chain/include/eosio/chain/snapshot.hpp

@@ -376,7 +376,7 @@ namespace eosio { namespace chain {
 
    class istream_json_snapshot_reader : public snapshot_reader {
       public:
-         explicit istream_json_snapshot_reader(const fc::path& p);
+         explicit istream_json_snapshot_reader(const std::filesystem::path& p);
          ~istream_json_snapshot_reader();
 
          void validate() const override;

libraries/chain/include/eosio/chain/snapshot_scheduler.hpp

@@ -0,0 +1,223 @@
+#pragma once
+
+#include <eosio/chain/pending_snapshot.hpp>
+
+#include <eosio/chain/block_state.hpp>
+#include <eosio/chain/config.hpp>
+#include <eosio/chain/exceptions.hpp>
+#include <eosio/chain/resource_limits.hpp>
+#include <eosio/chain/resource_limits_private.hpp>
+#include <eosio/chain/transaction.hpp>
+#include <eosio/chain/types.hpp>
+
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/random_access_index.hpp>
+#include <boost/multi_index_container.hpp>
+
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/property_tree/json_parser.hpp>
+#include <boost/property_tree/ptree.hpp>
+
+#include <limits>
+
+namespace eosio::chain {
+
+namespace bmi = boost::multi_index;
+namespace fs = std::filesystem;
+
+class snapshot_scheduler {
+public:
+   struct snapshot_information {
+      chain::block_id_type head_block_id;
+      uint32_t head_block_num;
+      fc::time_point head_block_time;
+      uint32_t version;
+      std::string snapshot_name;
+   };
+
+   struct snapshot_request_information {
+      uint32_t block_spacing = 0;
+      uint32_t start_block_num = 0;
+      uint32_t end_block_num = std::numeric_limits<uint32_t>::max();
+      std::string snapshot_description = "";
+   };
+
+   // this struct used to hold request params in api call
+   // it is differentiate between 0 and empty values
+   struct snapshot_request_params {
+      std::optional<uint32_t> block_spacing;
+      std::optional<uint32_t> start_block_num;
+      std::optional<uint32_t> end_block_num;
+      std::optional<std::string> snapshot_description;
+   };
+
+   struct snapshot_request_id_information {
+      uint32_t snapshot_request_id = 0;
+   };
+
+   struct snapshot_schedule_result : public snapshot_request_id_information, public snapshot_request_information {
+   };
+
+   struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information {
+      std::vector<snapshot_information> pending_snapshots;
+   };
+
+   struct get_snapshot_requests_result {
+      std::vector<snapshot_schedule_information> snapshot_requests;
+   };
+
+   template<typename T>
+   using next_function = eosio::chain::next_function<T>;
+
+   struct by_height;
+
+   using pending_snapshot_index = bmi::multi_index_container<
+         pending_snapshot<snapshot_information>,
+         indexed_by<
+               bmi::hashed_unique<tag<by_id>, BOOST_MULTI_INDEX_MEMBER(pending_snapshot<snapshot_information>, block_id_type, block_id)>,
+               bmi::ordered_non_unique<tag<by_height>, BOOST_MULTI_INDEX_CONST_MEM_FUN(pending_snapshot<snapshot_information>, uint32_t, get_height)>>>;
+
+   class snapshot_db_json {
+   public:
+      snapshot_db_json() = default;
+      ~snapshot_db_json() = default;
+
+      void set_path(std::filesystem::path path) {
+         db_path = std::move(path);
+      }
+
+      std::filesystem::path get_json_path() const {
+         return db_path / "snapshot-schedule.json";
+      }
+
+      const snapshot_db_json& operator>>(std::vector<snapshot_schedule_information>& sr) {
+         boost::property_tree::ptree root;
+
+         try {
+            std::ifstream file(get_json_path().string());
+            file.exceptions(std::istream::failbit | std::istream::eofbit);
+            boost::property_tree::read_json(file, root);
+
+            // parse ptree
+            for(boost::property_tree::ptree::value_type& req: root.get_child("snapshot_requests")) {
+               snapshot_schedule_information ssi;
+               ssi.snapshot_request_id = req.second.get<uint32_t>("snapshot_request_id");
+               ssi.snapshot_description = req.second.get<std::string>("snapshot_description");
+               ssi.block_spacing = req.second.get<uint32_t>("block_spacing");
+               ssi.start_block_num = req.second.get<uint32_t>("start_block_num");
+               ssi.end_block_num = req.second.get<uint32_t>("end_block_num");
+               sr.push_back(ssi);
+            }
+         } catch(std::ifstream::failure& e) {
+            elog("unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}",
+                 ("jsonpath", get_json_path().string())("details", e.what()));
+         }
+
+         return *this;
+      }
+
+      const snapshot_db_json& operator<<(const std::vector<snapshot_schedule_information>& sr) const {
+         boost::property_tree::ptree root;
+         boost::property_tree::ptree node_srs;
+
+         for(const auto& key: sr) {
+            boost::property_tree::ptree node;
+            node.put("snapshot_request_id", key.snapshot_request_id);
+            node.put("snapshot_description", key.snapshot_description);
+            node.put("block_spacing", key.block_spacing);
+            node.put("start_block_num", key.start_block_num);
+            node.put("end_block_num", key.end_block_num);
+            node_srs.push_back(std::make_pair("", node));
+         }
+
+         root.push_back(std::make_pair("snapshot_requests", node_srs));
+
+         try {
+            std::ofstream file(get_json_path().string());
+            file.exceptions(std::istream::failbit | std::istream::eofbit);
+            boost::property_tree::write_json(file, root);
+         } catch(std::ofstream::failure& e) {
+            elog("unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}",
+                 ("jsonpath", get_json_path().string())("details", e.what()));
+         }
+
+         return *this;
+      }
+
+   private:
+      fs::path db_path;
+   };
+
+private:
+   struct by_snapshot_id;
+   struct by_snapshot_value;
+   struct as_vector;
+
+   using snapshot_requests = bmi::multi_index_container<
+         snapshot_schedule_information,
+         indexed_by<
+               bmi::hashed_unique<tag<by_snapshot_id>, BOOST_MULTI_INDEX_MEMBER(snapshot_request_id_information, uint32_t, snapshot_request_id)>,
+               bmi::random_access<tag<as_vector>>,
+               bmi::ordered_unique<tag<by_snapshot_value>,
+                                   composite_key<snapshot_schedule_information,
+                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, block_spacing),
+                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, start_block_num),
+                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, end_block_num)>>>>;
+   snapshot_requests _snapshot_requests;
+   snapshot_db_json _snapshot_db;
+   pending_snapshot_index _pending_snapshot_index;
+
+   uint32_t _snapshot_id = 0;
+   uint32_t _inflight_sid = 0;
+
+   // path to write the snapshots to
+   fs::path _snapshots_dir;
+
+   void x_serialize() {
+      auto& vec = _snapshot_requests.get<as_vector>();
+      std::vector<snapshot_schedule_information> sr(vec.begin(), vec.end());
+      _snapshot_db << sr;
+   };
+
+public:
+   snapshot_scheduler() = default;
+
+   // snapshot scheduler listener
+   void on_start_block(uint32_t height, chain::controller& chain);
+
+   // to promote pending snapshots
+   void on_irreversible_block(const signed_block_ptr& lib, const chain::controller& chain);
+
+   // snapshot scheduler handlers
+   snapshot_schedule_result schedule_snapshot(const snapshot_request_information& sri);
+   snapshot_schedule_result unschedule_snapshot(uint32_t sri);
+   get_snapshot_requests_result get_snapshot_requests();
+
+   // initialize with storage
+   void set_db_path(fs::path db_path);
+
+   // set snapshot path
+   void set_snapshots_path(fs::path sn_path);
+
+   // add pending snapshot info to inflight snapshot request
+   void add_pending_snapshot_info(const snapshot_information& si);
+
+   // execute snapshot
+   void execute_snapshot(uint32_t srid, chain::controller& chain);
+
+   // former producer_plugin snapshot fn
+   void create_snapshot(next_function<snapshot_information> next, chain::controller& chain, std::function<void(void)> predicate);
+};
+
+
+}// namespace eosio::chain
+
+FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_information, (head_block_id) (head_block_num) (head_block_time) (version) (snapshot_name))
+FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_information, (block_spacing) (start_block_num) (end_block_num) (snapshot_description))
+FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_params, (block_spacing) (start_block_num) (end_block_num) (snapshot_description))
+FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_id_information, (snapshot_request_id))
+FC_REFLECT(eosio::chain::snapshot_scheduler::get_snapshot_requests_result, (snapshot_requests))
+FC_REFLECT_DERIVED(eosio::chain::snapshot_scheduler::snapshot_schedule_information, (eosio::chain::snapshot_scheduler::snapshot_request_id_information)(eosio::chain::snapshot_scheduler::snapshot_request_information), (pending_snapshots))
+FC_REFLECT_DERIVED(eosio::chain::snapshot_scheduler::snapshot_schedule_result, (eosio::chain::snapshot_scheduler::snapshot_request_id_information)(eosio::chain::snapshot_scheduler::snapshot_request_information), )

libraries/chain/include/eosio/chain/subjective_billing.hpp

@@ -8,12 +8,14 @@
 #include <eosio/chain/resource_limits_private.hpp>
 #include <eosio/chain/config.hpp>
 
+#include <fc/time.hpp>
+
 #include <boost/multi_index_container.hpp>
 #include <boost/multi_index/member.hpp>
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
 
-namespace eosio {
+namespace eosio::chain {
 
 class subjective_billing {
 private:
@@ -92,7 +94,7 @@ private:
 
 public: // public for tests
    static constexpr uint32_t subjective_time_interval_ms = 5'000;
-   size_t get_account_cache_size() {return _account_subjective_bill_cache.size();}
+   size_t get_account_cache_size() const {return _account_subjective_bill_cache.size();}
    void remove_subjective_billing( const chain::transaction_id_type& trx_id, uint32_t time_ordinal ) {
       auto& idx = _trx_cache_index.get<by_id>();
       auto itr = idx.find( trx_id );
@@ -107,7 +109,7 @@ public:
    void disable_account( chain::account_name a ) { _disabled_accounts.emplace( a ); }
    bool is_account_disabled(const chain::account_name& a ) const { return _disabled || _disabled_accounts.count( a ); }
 
-   void subjective_bill( const chain::transaction_id_type& id, const fc::time_point& expire,
+   void subjective_bill( const chain::transaction_id_type& id, fc::time_point_sec expire,
                          const chain::account_name& first_auth, const fc::microseconds& elapsed )
    {
       if( !_disabled && !_disabled_accounts.count( first_auth ) ) {
@@ -116,7 +118,7 @@ public:
                trx_cache_entry{id,
                                first_auth,
                                bill,
-                               expire} );
+                               expire.to_time_point()} );
          if( p.second ) {
             _account_subjective_bill_cache[first_auth].pending_cpu_us += bill;
          }
@@ -149,15 +151,15 @@ public:
       }
    }
 
-   void abort_block() {
-   }
-
    void on_block( fc::logger& log, const chain::block_state_ptr& bsp, const fc::time_point& now ) {
       if( bsp == nullptr || _disabled ) return;
       const auto time_ordinal = time_ordinal_for(now);
       const auto orig_count = _account_subjective_bill_cache.size();
       remove_subjective_billing( bsp, time_ordinal );
-      fc_dlog( log, "Subjective billed accounts ${n} removed ${r}", ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
+      if (orig_count > 0) {
+         fc_dlog( log, "Subjective billed accounts ${n} removed ${r}",
+                  ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
+      }
    }
 
    template <typename Yield>
@@ -197,4 +199,4 @@ public:
    }
 };
 
-} //eosio
+} //eosio::chain

libraries/chain/include/eosio/chain/symbol.hpp

@@ -1,12 +1,12 @@
 #pragma once
 #include <fc/exception/exception.hpp>
+#include <eosio/chain/exceptions.hpp>
 #include <eosio/chain/types.hpp>
 #include <eosio/chain/core_symbol.hpp>
 #include <string>
 #include <functional>
 
-namespace eosio {
-   namespace chain {
+namespace eosio::chain {
 
       /**
          class symbol represents a token and contains precision and name.
@@ -65,20 +65,7 @@ namespace eosio {
             explicit symbol(uint64_t v = CORE_SYMBOL): m_value(v) {
                EOS_ASSERT(valid(), symbol_type_exception, "invalid symbol: ${name}", ("name",name()));
             }
-            static symbol from_string(const string& from)
-            {
-               try {
-                  string s = fc::trim(from);
-                  EOS_ASSERT(!s.empty(), symbol_type_exception, "creating symbol from empty string");
-                  auto comma_pos = s.find(',');
-                  EOS_ASSERT(comma_pos != string::npos, symbol_type_exception, "missing comma in symbol");
-                  auto prec_part = s.substr(0, comma_pos);
-                  uint8_t p = fc::to_int64(prec_part);
-                  string name_part = s.substr(comma_pos + 1);
-                  EOS_ASSERT( p <= max_precision, symbol_type_exception, "precision ${p} should be <= 18", ("p", p));
-                  return symbol(string_to_symbol(p, name_part.c_str()));
-               } FC_CAPTURE_LOG_AND_RETHROW((from))
-            }
+            static symbol from_string(const string& from);
             uint64_t value() const { return m_value; }
             bool valid() const
             {
@@ -184,8 +171,7 @@ namespace eosio {
       {
          return std::tie(lhs.sym, lhs.contract) > std::tie(rhs.sym, rhs.contract);
       }
-   } // namespace chain
-} // namespace eosio
+} // namespace eosio::chain
 
 namespace fc {
    inline void to_variant(const eosio::chain::symbol& var, fc::variant& vo) { vo = var.to_string(); }

libraries/chain/include/eosio/chain/thread_utils.hpp

@@ -37,9 +37,13 @@ namespace eosio { namespace chain {
 
       /// Spawn threads, can be re-started after stop().
       /// Assumes start()/stop() called from the same thread or externally protected.
+      /// Blocks until all threads are created and completed their init function, or an exception is thrown
+      ///  during thread startup or an init function. Exceptions thrown during these stages are rethrown from start()
+      ///  but some threads might still have been started. Calling stop() after such a failure is safe.
       /// @param num_threads is number of threads spawned
       /// @param on_except is the function to call if io_context throws an exception, is called from thread pool thread.
-      ///                  if an empty function then logs and rethrows exception on thread which will terminate.
+      ///                  if an empty function then logs and rethrows exception on thread which will terminate. Not called
+      ///                  for exceptions during the init function (such exceptions are rethrown from start())
       /// @param init is an optional function to call at startup to initialize any data.
       /// @throw assert_exception if already started and not stopped.
       void start( size_t num_threads, on_except_t on_except, init_t init = {} ) {
@@ -47,9 +51,25 @@ namespace eosio { namespace chain {
          _ioc_work.emplace( boost::asio::make_work_guard( _ioc ) );
          _ioc.restart();
          _thread_pool.reserve( num_threads );
-         for( size_t i = 0; i < num_threads; ++i ) {
-            _thread_pool.emplace_back( std::thread( &named_thread_pool::run_thread, this, i, on_except, init )  );
+
+         std::promise<void> start_complete;
+         std::atomic<uint32_t> threads_remaining = num_threads;
+         std::exception_ptr pending_exception;
+         std::mutex pending_exception_mutex;
+
+         try {
+            for( size_t i = 0; i < num_threads; ++i ) {
+               _thread_pool.emplace_back( std::thread( &named_thread_pool::run_thread, this, i, on_except, init, std::ref(start_complete),
+                                                       std::ref(threads_remaining), std::ref(pending_exception), std::ref(pending_exception_mutex) ) );
+            }
          }
+         catch( ... ) {
+            /// only an exception from std::thread's ctor should end up here. shut down all threads to ensure no
+            ///  potential access to the promise, atomic, etc above performed after throwing out of start
+            stop();
+            throw;
+         }
+         start_complete.get_future().get();
       }
 
       /// destroy work guard, stop io_context, join thread_pool
@@ -63,16 +83,42 @@ namespace eosio { namespace chain {
       }
 
    private:
-      void run_thread( size_t i, const on_except_t& on_except, const init_t& init ) {
-         std::string tn = boost::core::demangle(typeid(this).name());
-         auto offset = tn.rfind("::");
-         if (offset != std::string::npos)
-            tn.erase(0, offset+2);
-         tn = tn.substr(0, tn.find('>')) + "-" + std::to_string( i );
+      void run_thread( size_t i, const on_except_t& on_except, const init_t& init, std::promise<void>& start_complete,
+                       std::atomic<uint32_t>& threads_remaining, std::exception_ptr& pending_exception, std::mutex& pending_exception_mutex ) {
+
+         std::string tn;
+
+         auto decrement_remaining = [&]() {
+            if( !--threads_remaining ) {
+               if( pending_exception )
+                  start_complete.set_exception( pending_exception );
+               else
+                  start_complete.set_value();
+            }
+         };
+
+         try {
+            try {
+               tn = boost::core::demangle(typeid(this).name());
+               auto offset = tn.rfind("::");
+               if (offset != std::string::npos)
+                  tn.erase(0, offset+2);
+               tn = tn.substr(0, tn.find('>')) + "-" + std::to_string( i );
+               fc::set_os_thread_name( tn );
+               if ( init )
+                  init();
+            } FC_LOG_AND_RETHROW()
+         }
+         catch( ... ) {
+            std::lock_guard<std::mutex> l( pending_exception_mutex );
+            pending_exception = std::current_exception();
+            decrement_remaining();
+            return;
+         }
+
+         decrement_remaining();
+
          try {
-            fc::set_os_thread_name( tn );
-            if ( init )
-               init();
             _ioc.run();
          } catch( const fc::exception& e ) {
             if( on_except ) {

libraries/chain/include/eosio/chain/types.hpp

@@ -8,7 +8,6 @@
 #include <fc/io/varint.hpp>
 #include <fc/io/enum_type.hpp>
 #include <fc/crypto/sha224.hpp>
-#include <fc/safe.hpp>
 #include <fc/container/flat.hpp>
 #include <fc/string.hpp>
 #include <fc/io/raw.hpp>
@@ -43,7 +42,7 @@
 
 #define _V(n, v)  fc::mutable_variant_object(n, v)
 
-namespace eosio { namespace chain {
+namespace eosio::chain {
    using                               std::map;
    using                               std::vector;
    using                               std::unordered_map;
@@ -61,7 +60,7 @@ namespace eosio { namespace chain {
    using                               std::to_string;
    using                               std::all_of;
 
-   using                               fc::path;
+   using                               std::filesystem::path;
    using                               fc::variant_object;
    using                               fc::variant;
    using                               fc::enum_type;
@@ -69,7 +68,6 @@ namespace eosio { namespace chain {
    using                               fc::signed_int;
    using                               fc::time_point_sec;
    using                               fc::time_point;
-   using                               fc::safe;
    using                               fc::flat_map;
    using                               fc::flat_multimap;
    using                               fc::flat_set;
@@ -78,15 +76,12 @@ namespace eosio { namespace chain {
    using public_key_type  = fc::crypto::public_key;
    using private_key_type = fc::crypto::private_key;
    using signature_type   = fc::crypto::signature;
-#if BOOST_VERSION >= 107100
-      // configurable boost deque performs much better than std::deque in our use cases
-      using block_1024_option_t = boost::container::deque_options< boost::container::block_size<1024u> >::type;
-      template<typename T>
-      using deque = boost::container::deque< T, void, block_1024_option_t >;
-#else
-      template<typename T>
-    using deque = std::deque<T>;
-#endif
+   
+   // configurable boost deque (for boost >= 1.71) performs much better than std::deque in our use cases
+   using block_1024_option_t = boost::container::deque_options< boost::container::block_size<1024u> >::type;
+   template<typename T>
+   using deque = boost::container::deque< T, void, block_1024_option_t >;
+   
    struct void_t{};
 
    using chainbase::allocator;
@@ -375,7 +370,7 @@ namespace eosio { namespace chain {
    }
 
    template<typename E, typename F>
-   static inline auto has_field( F flags, E field )
+   static constexpr auto has_field( F flags, E field )
    -> std::enable_if_t< std::is_integral<F>::value && std::is_unsigned<F>::value &&
                         std::is_enum<E>::value && std::is_same< F, std::underlying_type_t<E> >::value, bool>
    {
@@ -383,7 +378,7 @@ namespace eosio { namespace chain {
    }
 
    template<typename E, typename F>
-   static inline auto set_field( F flags, E field, bool value = true )
+   static constexpr auto set_field( F flags, E field, bool value = true )
    -> std::enable_if_t< std::is_integral<F>::value && std::is_unsigned<F>::value &&
                         std::is_enum<E>::value && std::is_same< F, std::underlying_type_t<E> >::value, F >
    {
@@ -396,7 +391,24 @@ namespace eosio { namespace chain {
    template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
    template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 
-} }  // eosio::chain
+   // next_function is a function passed to an API (like send_transaction) and which is called at the end of
+   // the API processing on the main thread. The type T is a description of the API result that can be
+   // serialized as output.
+   // The function accepts a variant which can contain an exception_ptr (if an exception occured while
+   // processing the API) or the result T.
+   // The third option is a function which can be executed in a multithreaded context (likely on the
+   // http_plugin thread pool) and which completes the API processing and returns the result T.
+   // -------------------------------------------------------------------------------------------------------
+   template<typename T>
+   using t_or_exception = std::variant<T, fc::exception_ptr>;
+
+   template<typename T>
+   using next_function_variant = std::variant<fc::exception_ptr, T, std::function<t_or_exception<T>()>>;
+
+   template<typename T>
+   using next_function = std::function<void(const next_function_variant<T>&)>;
+
+}  // eosio::chain
 
 namespace chainbase {
    // chainbase::shared_cow_string

libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp

@@ -27,7 +27,7 @@ enum class trx_enum_type {
    incoming_p2p = 4 // incoming_end() needs to be updated if this changes
 };
 
-using next_func_t = std::function<void(const std::variant<fc::exception_ptr, transaction_trace_ptr>&)>;
+using next_func_t = next_function<transaction_trace_ptr>;
 
 struct unapplied_transaction {
    const transaction_metadata_ptr trx_meta;
@@ -37,11 +37,6 @@ struct unapplied_transaction {
 
    const transaction_id_type& id()const { return trx_meta->id(); }
    fc::time_point_sec expiration()const { return trx_meta->packed_trx()->expiration(); }
-
-   unapplied_transaction(const unapplied_transaction&) = delete;
-   unapplied_transaction() = delete;
-   unapplied_transaction& operator=(const unapplied_transaction&) = delete;
-   unapplied_transaction(unapplied_transaction&&) = default;
 };
 
 /**
@@ -99,7 +94,7 @@ public:
       auto& persisted_by_expiry = queue.get<by_expiry>();
       while( !persisted_by_expiry.empty() ) {
          const auto& itr = persisted_by_expiry.begin();
-         if( itr->expiration() > pending_block_time ) {
+         if( itr->expiration().to_time_point() > pending_block_time ) {
             break;
          }
          if( yield() ) {

libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp

@@ -168,121 +168,121 @@ namespace eosio { namespace chain { namespace wasm_injections {
    constexpr const char* inject_which_op( uint16_t opcode ) {
       switch ( opcode ) {
          case wasm_ops::f32_add_code:
-            return u8"_eosio_f32_add";
+            return "_eosio_f32_add";
          case wasm_ops::f32_sub_code:
-            return u8"_eosio_f32_sub";
+            return "_eosio_f32_sub";
          case wasm_ops::f32_mul_code:
-            return u8"_eosio_f32_mul";
+            return "_eosio_f32_mul";
          case wasm_ops::f32_div_code:
-            return u8"_eosio_f32_div";
+            return "_eosio_f32_div";
          case wasm_ops::f32_min_code:
-            return u8"_eosio_f32_min";
+            return "_eosio_f32_min";
          case wasm_ops::f32_max_code:
-            return u8"_eosio_f32_max";
+            return "_eosio_f32_max";
          case wasm_ops::f32_copysign_code:
-            return u8"_eosio_f32_copysign";
+            return "_eosio_f32_copysign";
          case wasm_ops::f32_abs_code:
-            return u8"_eosio_f32_abs";
+            return "_eosio_f32_abs";
          case wasm_ops::f32_neg_code:
-            return u8"_eosio_f32_neg";
+            return "_eosio_f32_neg";
          case wasm_ops::f32_sqrt_code:
-            return u8"_eosio_f32_sqrt";
+            return "_eosio_f32_sqrt";
          case wasm_ops::f32_ceil_code:
-            return u8"_eosio_f32_ceil";
+            return "_eosio_f32_ceil";
          case wasm_ops::f32_floor_code:
-            return u8"_eosio_f32_floor";
+            return "_eosio_f32_floor";
          case wasm_ops::f32_trunc_code:
-            return u8"_eosio_f32_trunc";
+            return "_eosio_f32_trunc";
          case wasm_ops::f32_nearest_code:
-            return u8"_eosio_f32_nearest";
+            return "_eosio_f32_nearest";
          case wasm_ops::f32_eq_code:
-            return u8"_eosio_f32_eq";
+            return "_eosio_f32_eq";
          case wasm_ops::f32_ne_code:
-            return u8"_eosio_f32_ne";
+            return "_eosio_f32_ne";
          case wasm_ops::f32_lt_code:
-            return u8"_eosio_f32_lt";
+            return "_eosio_f32_lt";
          case wasm_ops::f32_le_code:
-            return u8"_eosio_f32_le";
+            return "_eosio_f32_le";
          case wasm_ops::f32_gt_code:
-            return u8"_eosio_f32_gt";
+            return "_eosio_f32_gt";
          case wasm_ops::f32_ge_code:
-            return u8"_eosio_f32_ge";
+            return "_eosio_f32_ge";
          case wasm_ops::f64_add_code:
-            return u8"_eosio_f64_add";
+            return "_eosio_f64_add";
          case wasm_ops::f64_sub_code:
-            return u8"_eosio_f64_sub";
+            return "_eosio_f64_sub";
          case wasm_ops::f64_mul_code:
-            return u8"_eosio_f64_mul";
+            return "_eosio_f64_mul";
          case wasm_ops::f64_div_code:
-            return u8"_eosio_f64_div";
+            return "_eosio_f64_div";
          case wasm_ops::f64_min_code:
-            return u8"_eosio_f64_min";
+            return "_eosio_f64_min";
          case wasm_ops::f64_max_code:
-            return u8"_eosio_f64_max";
+            return "_eosio_f64_max";
          case wasm_ops::f64_copysign_code:
-            return u8"_eosio_f64_copysign";
+            return "_eosio_f64_copysign";
          case wasm_ops::f64_abs_code:
-            return u8"_eosio_f64_abs";
+            return "_eosio_f64_abs";
          case wasm_ops::f64_neg_code:
-            return u8"_eosio_f64_neg";
+            return "_eosio_f64_neg";
          case wasm_ops::f64_sqrt_code:
-            return u8"_eosio_f64_sqrt";
+            return "_eosio_f64_sqrt";
          case wasm_ops::f64_ceil_code:
-            return u8"_eosio_f64_ceil";
+            return "_eosio_f64_ceil";
          case wasm_ops::f64_floor_code:
-            return u8"_eosio_f64_floor";
+            return "_eosio_f64_floor";
          case wasm_ops::f64_trunc_code:
-            return u8"_eosio_f64_trunc";
+            return "_eosio_f64_trunc";
          case wasm_ops::f64_nearest_code:
-            return u8"_eosio_f64_nearest";
+            return "_eosio_f64_nearest";
          case wasm_ops::f64_eq_code:
-            return u8"_eosio_f64_eq";
+            return "_eosio_f64_eq";
          case wasm_ops::f64_ne_code:
-            return u8"_eosio_f64_ne";
+            return "_eosio_f64_ne";
          case wasm_ops::f64_lt_code:
-            return u8"_eosio_f64_lt";
+            return "_eosio_f64_lt";
          case wasm_ops::f64_le_code:
-            return u8"_eosio_f64_le";
+            return "_eosio_f64_le";
          case wasm_ops::f64_gt_code:
-            return u8"_eosio_f64_gt";
+            return "_eosio_f64_gt";
          case wasm_ops::f64_ge_code:
-            return u8"_eosio_f64_ge";
+            return "_eosio_f64_ge";
          case wasm_ops::f64_promote_f32_code:
-            return u8"_eosio_f32_promote";
+            return "_eosio_f32_promote";
          case wasm_ops::f32_demote_f64_code:
-            return u8"_eosio_f64_demote";
+            return "_eosio_f64_demote";
          case wasm_ops::i32_trunc_u_f32_code:
-            return u8"_eosio_f32_trunc_i32u";
+            return "_eosio_f32_trunc_i32u";
          case wasm_ops::i32_trunc_s_f32_code:
-            return u8"_eosio_f32_trunc_i32s";
+            return "_eosio_f32_trunc_i32s";
          case wasm_ops::i32_trunc_u_f64_code:
-            return u8"_eosio_f64_trunc_i32u";
+            return "_eosio_f64_trunc_i32u";
          case wasm_ops::i32_trunc_s_f64_code:
-            return u8"_eosio_f64_trunc_i32s";
+            return "_eosio_f64_trunc_i32s";
          case wasm_ops::i64_trunc_u_f32_code:
-            return u8"_eosio_f32_trunc_i64u";
+            return "_eosio_f32_trunc_i64u";
          case wasm_ops::i64_trunc_s_f32_code:
-            return u8"_eosio_f32_trunc_i64s";
+            return "_eosio_f32_trunc_i64s";
          case wasm_ops::i64_trunc_u_f64_code:
-            return u8"_eosio_f64_trunc_i64u";
+            return "_eosio_f64_trunc_i64u";
          case wasm_ops::i64_trunc_s_f64_code:
-            return u8"_eosio_f64_trunc_i64s";
+            return "_eosio_f64_trunc_i64s";
          case wasm_ops::f32_convert_s_i32_code:
-            return u8"_eosio_i32_to_f32";
+            return "_eosio_i32_to_f32";
          case wasm_ops::f32_convert_u_i32_code:
-            return u8"_eosio_ui32_to_f32";
+            return "_eosio_ui32_to_f32";
          case wasm_ops::f32_convert_s_i64_code:
-            return u8"_eosio_i64_f32";
+            return "_eosio_i64_f32";
          case wasm_ops::f32_convert_u_i64_code:
-            return u8"_eosio_ui64_to_f32";
+            return "_eosio_ui64_to_f32";
          case wasm_ops::f64_convert_s_i32_code:
-            return u8"_eosio_i32_to_f64";
+            return "_eosio_i32_to_f64";
          case wasm_ops::f64_convert_u_i32_code:
-            return u8"_eosio_ui32_to_f64";
+            return "_eosio_ui32_to_f64";
          case wasm_ops::f64_convert_s_i64_code:
-            return u8"_eosio_i64_to_f64";
+            return "_eosio_i64_to_f64";
          case wasm_ops::f64_convert_u_i64_code:
-            return u8"_eosio_ui64_to_f64";
+            return "_eosio_ui64_to_f64";
 
          default:
             FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, unknown opcode in injection ${op}", ("op", opcode));
@@ -483,7 +483,7 @@ namespace eosio { namespace chain { namespace wasm_injections {
       static void init() {}
       static void accept( wasm_ops::instr* inst, wasm_ops::visitor_arg& arg ) {
          int32_t idx;
-         injector_utils::add_import<ResultType::f64, ValueType::f32>( *(arg.module), u8"_eosio_f32_promote", idx );
+         injector_utils::add_import<ResultType::f64, ValueType::f32>( *(arg.module), "_eosio_f32_promote", idx );
          wasm_ops::op_types<>::call_t f32promote;
          f32promote.field = idx;
          f32promote.pack(arg.new_code);
@@ -496,7 +496,7 @@ namespace eosio { namespace chain { namespace wasm_injections {
       static void init() {}
       static void accept( wasm_ops::instr* inst, wasm_ops::visitor_arg& arg ) {
          int32_t idx;
-         injector_utils::add_import<ResultType::f32, ValueType::f64>( *(arg.module), u8"_eosio_f64_demote", idx );
+         injector_utils::add_import<ResultType::f32, ValueType::f64>( *(arg.module), "_eosio_f64_demote", idx );
          wasm_ops::op_types<>::call_t f32promote;
          f32promote.field = idx;
          f32promote.pack(arg.new_code);

libraries/chain/include/eosio/chain/wasm_interface.hpp

@@ -4,8 +4,6 @@
 #include <eosio/chain/whitelisted_intrinsics.hpp>
 #include <eosio/chain/exceptions.hpp>
 #include <functional>
-#include "Runtime/Linker.h"
-#include "Runtime/Runtime.h"
 
 namespace eosio { namespace chain {
 
@@ -42,7 +40,7 @@ namespace eosio { namespace chain {
              }
          }
 
-         wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile);
+         wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile);
          ~wasm_interface();
 
          // initialize exec per thread
@@ -63,9 +61,6 @@ namespace eosio { namespace chain {
          //Calls apply or error on a given code
          void apply(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, apply_context& context);
 
-         //Immediately exits currently running wasm. UB is called when no wasm running
-         void exit();
-
          //Returns true if the code is cached
          bool is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const;
 

libraries/chain/include/eosio/chain/wasm_interface_private.hpp

@@ -15,7 +15,6 @@
 #include <fc/scoped_exit.hpp>
 
 #include "IR/Module.h"
-#include "Runtime/Intrinsics.h"
 #include "Platform/Platform.h"
 #include "WAST/WAST.h"
 #include "IR/Validate.h"
@@ -26,7 +25,6 @@
 using namespace fc;
 using namespace eosio::chain::webassembly;
 using namespace IR;
-using namespace Runtime;
 
 using boost::multi_index_container;
 
@@ -37,7 +35,6 @@ namespace eosio { namespace chain {
    struct wasm_interface_impl {
       struct wasm_cache_entry {
          digest_type                                          code_hash;
-         uint32_t                                             first_block_num_used;
          uint32_t                                             last_block_num_used;
          std::unique_ptr<wasm_instantiated_module_interface>  module;
          uint8_t                                              vm_type = 0;
@@ -49,7 +46,7 @@ namespace eosio { namespace chain {
 
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       struct eosvmoc_tier {
-         eosvmoc_tier(const boost::filesystem::path& d, const eosvmoc::config& c, const chainbase::database& db)
+         eosvmoc_tier(const std::filesystem::path& d, const eosvmoc::config& c, const chainbase::database& db)
           : cc(d, c, db) {
              // construct exec for the main thread
              init_thread_local_data();
@@ -68,7 +65,7 @@ namespace eosio { namespace chain {
       };
 #endif
 
-      wasm_interface_impl(wasm_interface::vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile) : db(d), wasm_runtime_time(vm) {
+      wasm_interface_impl(wasm_interface::vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile) : db(d), wasm_runtime_time(vm) {
 #ifdef EOSIO_EOS_VM_RUNTIME_ENABLED
          if(vm == wasm_interface::vm_type::eos_vm)
             runtime_interface = std::make_unique<webassembly::eos_vm_runtime::eos_vm_runtime<eosio::vm::interpreter>>();
@@ -109,24 +106,6 @@ namespace eosio { namespace chain {
          return it != wasm_instantiation_cache.end();
       }
 
-      std::vector<uint8_t> parse_initial_memory(const Module& module) {
-         std::vector<uint8_t> mem_image;
-
-         for(const DataSegment& data_segment : module.dataSegments) {
-            EOS_ASSERT(data_segment.baseOffset.type == InitializerExpression::Type::i32_const, wasm_exception, "");
-            EOS_ASSERT(module.memories.defs.size(), wasm_exception, "");
-            const U32 base_offset = data_segment.baseOffset.i32;
-            const Uptr memory_size = (module.memories.defs[0].type.size.min << IR::numBytesPerPageLog2);
-            if(base_offset >= memory_size || base_offset + data_segment.data.size() > memory_size)
-               FC_THROW_EXCEPTION(wasm_execution_error, "WASM data segment outside of valid memory range");
-            if(base_offset + data_segment.data.size() > mem_image.size())
-               mem_image.resize(base_offset + data_segment.data.size(), 0x00);
-            memcpy(mem_image.data() + base_offset, data_segment.data.data(), data_segment.data.size());
-         }
-
-         return mem_image;
-      }
-
       void code_block_num_last_used(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, const uint32_t& block_num) {
          wasm_cache_index::iterator it = wasm_instantiation_cache.find(boost::make_tuple(code_hash, vm_type, vm_version));
          if(it != wasm_instantiation_cache.end())
@@ -157,7 +136,6 @@ namespace eosio { namespace chain {
 
             it = wasm_instantiation_cache.emplace( wasm_interface_impl::wasm_cache_entry{
                                                       .code_hash = code_hash,
-                                                      .first_block_num_used = codeobject->first_block_used,
                                                       .last_block_num_used = UINT32_MAX,
                                                       .module = nullptr,
                                                       .vm_type = vm_type,
@@ -173,24 +151,8 @@ namespace eosio { namespace chain {
                trx_context.resume_billing_timer();
             });
             trx_context.pause_billing_timer();
-            IR::Module module;
-            std::vector<U8> bytes = {
-                (const U8*)codeobject->code.data(),
-                (const U8*)codeobject->code.data() + codeobject->code.size()};
-            try {
-               Serialization::MemoryInputStream stream((const U8*)bytes.data(),
-                                                       bytes.size());
-               WASM::scoped_skip_checks no_check;
-               WASM::serialize(stream, module);
-               module.userSections.clear();
-            } catch (const Serialization::FatalSerializationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            } catch (const IR::ValidationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            }
-
             wasm_instantiation_cache.modify(it, [&](auto& c) {
-               c.module = runtime_interface->instantiate_module((const char*)bytes.data(), bytes.size(), parse_initial_memory(module), code_hash, vm_type, vm_version);
+               c.module = runtime_interface->instantiate_module(codeobject->code.data(), codeobject->code.size(), code_hash, vm_type, vm_version);
             });
          }
          return it->module;
@@ -209,7 +171,6 @@ namespace eosio { namespace chain {
                   member<wasm_cache_entry, uint8_t,     &wasm_cache_entry::vm_version>
                >
             >,
-            ordered_non_unique<tag<by_first_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::first_block_num_used>>,
             ordered_non_unique<tag<by_last_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::last_block_num_used>>
          >
       > wasm_cache_index;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp

@@ -19,7 +19,6 @@
 namespace eosio { namespace chain { namespace webassembly { namespace eosvmoc {
 
 using namespace IR;
-using namespace Runtime;
 using namespace fc;
 
 using namespace eosio::chain::eosvmoc;
@@ -28,12 +27,11 @@ class eosvmoc_instantiated_module;
 
 class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface {
    public:
-      eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
+      eosvmoc_runtime(const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~eosvmoc_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
       void init_thread_local_data() override;
 
       friend eosvmoc_instantiated_module;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -31,21 +31,27 @@ using namespace boost::multi_index;
 using namespace boost::asio;
 
 namespace bip = boost::interprocess;
-namespace bfs = boost::filesystem;
 
 using allocator_t = bip::rbtree_best_fit<bip::null_mutex_family, bip::offset_ptr<void>, alignof(std::max_align_t)>;
 
 struct config;
 
+
 class code_cache_base {
    public:
-      code_cache_base(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
+      code_cache_base(const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~code_cache_base();
 
       const int& fd() const { return _cache_fd; }
 
       void free_code(const digest_type& code_id, const uint8_t& vm_version);
 
+      // get_descriptor_for_code failure reasons
+      enum class get_cd_failure {
+         temporary, // oc compile not done yet, users like read-only trxs can retry
+         permanent  // oc will not start, users should not retry
+      };
+
    protected:
       struct by_hash;
 
@@ -65,8 +71,8 @@ class code_cache_base {
 
       const chainbase::database& _db;
 
-      bfs::path _cache_file_path;
-      int _cache_fd;
+      std::filesystem::path _cache_file_path;
+      int                   _cache_fd;
 
       io_context _ctx;
       local::datagram_protocol::socket _compile_monitor_write_socket{_ctx};
@@ -85,20 +91,17 @@ class code_cache_base {
 
       template <typename T>
       void serialize_cache_index(fc::datastream<T>& ds);
-
-      std::thread::id _main_thread_id;
-      bool is_main_thread() const;
 };
 
 class code_cache_async : public code_cache_base {
    public:
-      code_cache_async(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
+      code_cache_async(const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~code_cache_async();
 
       //If code is in cache: returns pointer & bumps to front of MRU list
       //If code is not in cache, and not blacklisted, and not currently compiling: return nullptr and kick off compile
       //otherwise: return nullptr
-      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window, get_cd_failure& failure);
 
    private:
       std::thread _monitor_reply_thread;
@@ -115,7 +118,7 @@ class code_cache_sync : public code_cache_base {
       ~code_cache_sync();
 
       //Can still fail and return nullptr if, for example, there is an expected instantiation failure
-      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window);
 };
 
 }}}

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/config.hpp

@@ -5,7 +5,6 @@
 #include <vector>
 #include <string>
 
-#include <boost/filesystem/path.hpp>
 #include <fc/reflect/reflect.hpp>
 
 namespace eosio { namespace chain { namespace eosvmoc {

libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp

@@ -43,15 +43,11 @@ template<typename Backend>
 class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
-
    private:
       // todo: managing this will get more complicated with sync calls;
-      //       immediately_exit_currently_running_module() should probably
-      //       move from wasm_runtime_interface to wasm_instantiated_module_interface.
       eos_vm_backend_t<Backend>* _bkend = nullptr;  // non owning pointer to allow for immediate exit
 
    template<typename Impl>
@@ -61,10 +57,8 @@ class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
 class eos_vm_profile_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_profile_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
-
-      void immediately_exit_currently_running_module() override;
 };
 
 }}}}// eosio::chain::webassembly::eos_vm_runtime

libraries/chain/include/eosio/chain/webassembly/runtime_interface.hpp

@@ -20,12 +20,9 @@ class wasm_instantiated_module_interface {
 
 class wasm_runtime_interface {
    public:
-      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                                      const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) = 0;
 
-      //immediately exit the currently running wasm_instantiated_module_interface. Yep, this assumes only one can possibly run at a time.
-      virtual void immediately_exit_currently_running_module() = 0;
-
       virtual ~wasm_runtime_interface();
 
       // eosvmoc_runtime needs this

libraries/chain/platform_timer_kqueue.cpp

@@ -11,9 +11,18 @@
 
 #include <sys/event.h>
 
+// this makes kqueue calls compatible with FreeBSD and this timer can be safely used
+#if defined(__FreeBSD__)
+#define EV_SET64(kev, ident, filter, flags, fflags, data, udata, ext0, ext1) EV_SET(kev, ident, filter, flags, fflags, data, reinterpret_cast<void*>(udata))
+#define kevent64(kq, changelist, nchanges, eventlist, nevents, flags, timeout) kevent(kq, changelist, nchanges, eventlist, nevents, timeout)
+#define kevent64_s kevent
+#define KEVENT_FLAG_IMMEDIATE 0
+#define NOTE_CRITICAL 0
+#endif
+
 namespace eosio { namespace chain {
 
-//a kqueue & thread is shared for all platform_timer_macos instances
+// a kqueue & thread is shared for all platform_timer_macos instances
 static std::mutex timer_ref_mutex;
 static unsigned next_timerid;
 static unsigned refcount;

libraries/chain/protocol_feature_manager.cpp

@@ -780,34 +780,34 @@ Enables new `get_block_num` intrinsic which returns the current block number.
       }
    }
 
-   std::optional<builtin_protocol_feature> read_builtin_protocol_feature( const fc::path& p  ) {
+   std::optional<builtin_protocol_feature> read_builtin_protocol_feature( const std::filesystem::path& p  ) {
       try {
          return fc::json::from_file<builtin_protocol_feature>( p );
       } catch( const fc::exception& e ) {
          wlog( "problem encountered while reading '${path}':\n${details}",
-               ("path", p.generic_string())("details",e.to_detail_string()) );
+               ("path", p)("details",e.to_detail_string()) );
       } catch( ... ) {
          dlog( "unknown problem encountered while reading '${path}'",
-               ("path", p.generic_string()) );
+               ("path", p) );
       }
       return {};
    }
 
-   protocol_feature_set initialize_protocol_features( const fc::path& p, bool populate_missing_builtins ) {
-      using boost::filesystem::directory_iterator;
+   protocol_feature_set initialize_protocol_features( const std::filesystem::path& p, bool populate_missing_builtins ) {
+      using std::filesystem::directory_iterator;
 
       protocol_feature_set pfs;
 
       bool directory_exists = true;
 
-      if( fc::exists( p ) ) {
-         EOS_ASSERT( fc::is_directory( p ), plugin_exception,
+      if( std::filesystem::exists( p ) ) {
+         EOS_ASSERT( std::filesystem::is_directory( p ), plugin_exception,
                      "Path to protocol-features is not a directory: ${path}",
-                     ("path", p.generic_string())
+                     ("path", p)
          );
       } else {
          if( populate_missing_builtins )
-            boost::filesystem::create_directories( p );
+            std::filesystem::create_directories( p );
          else
             directory_exists = false;
       }
@@ -849,7 +849,7 @@ Enables new `get_block_num` intrinsic which returns the current block number.
          }
       };
 
-      map<builtin_protocol_feature_t, fc::path>  found_builtin_protocol_features;
+      map<builtin_protocol_feature_t, std::filesystem::path>  found_builtin_protocol_features;
       map<digest_type, std::pair<builtin_protocol_feature, bool> > builtin_protocol_features_to_add;
       // The bool in the pair is set to true if the builtin protocol feature has already been visited to add
       map< builtin_protocol_feature_t, std::optional<digest_type> > visited_builtins;
@@ -858,7 +858,7 @@ Enables new `get_block_num` intrinsic which returns the current block number.
       if( directory_exists ) {
          for( directory_iterator enditr, itr{p}; itr != enditr; ++itr ) {
             auto file_path = itr->path();
-            if( !fc::is_regular_file( file_path ) || file_path.extension().generic_string().compare( ".json" ) != 0 )
+            if( !std::filesystem::is_regular_file( file_path ) || file_path.extension().generic_string().compare( ".json" ) != 0 )
                continue;
 
             auto f = read_builtin_protocol_feature( file_path );
@@ -870,8 +870,8 @@ Enables new `get_block_num` intrinsic which returns the current block number.
             EOS_ASSERT( res.second, plugin_exception,
                         "Builtin protocol feature '${codename}' was already included from a previous_file",
                         ("codename", builtin_protocol_feature_codename(f->get_codename()))
-                              ("current_file", file_path.generic_string())
-                              ("previous_file", res.first->second.generic_string())
+                              ("current_file", file_path)
+                              ("previous_file", res.first->second)
             );
 
             const auto feature_digest = f->digest();
@@ -916,23 +916,23 @@ Enables new `get_block_num` intrinsic which returns the current block number.
 
          auto file_path = p / filename;
 
-         EOS_ASSERT( !fc::exists( file_path ), plugin_exception,
+         EOS_ASSERT( !std::filesystem::exists( file_path ), plugin_exception,
                      "Could not save builtin protocol feature with codename '${codename}' because a file at the following path already exists: ${path}",
                      ("codename", builtin_protocol_feature_codename( f.get_codename() ))
-                           ("path", file_path.generic_string())
+                           ("path", file_path)
          );
 
          if( fc::json::save_to_file( f, file_path ) ) {
             ilog( "Saved default specification for builtin protocol feature '${codename}' (with digest of '${digest}') to: ${path}",
                   ("codename", builtin_protocol_feature_codename(f.get_codename()))
                         ("digest", feature_digest)
-                        ("path", file_path.generic_string())
+                        ("path", file_path)
             );
          } else {
             elog( "Error occurred while writing default specification for builtin protocol feature '${codename}' (with digest of '${digest}') to: ${path}",
                   ("codename", builtin_protocol_feature_codename(f.get_codename()))
                         ("digest", feature_digest)
-                        ("path", file_path.generic_string())
+                        ("path", file_path)
             );
          }
       };

libraries/chain/snapshot.cpp

@@ -355,7 +355,7 @@ struct istream_json_snapshot_reader_impl {
 
 istream_json_snapshot_reader::~istream_json_snapshot_reader() = default;
 
-istream_json_snapshot_reader::istream_json_snapshot_reader(const fc::path& p)
+istream_json_snapshot_reader::istream_json_snapshot_reader(const std::filesystem::path& p)
    : impl{new istream_json_snapshot_reader_impl{0, 0, {}, {}}}
 {
    FILE* fp = fopen(p.string().c_str(), "rb");

libraries/chain/snapshot_scheduler.cpp

@@ -0,0 +1,231 @@
+#include <eosio/chain/controller.hpp>
+#include <eosio/chain/exceptions.hpp>
+#include <eosio/chain/pending_snapshot.hpp>
+#include <eosio/chain/snapshot_scheduler.hpp>
+#include <fc/scoped_exit.hpp>
+
+namespace eosio::chain {
+
+// snapshot_scheduler_listener
+void snapshot_scheduler::on_start_block(uint32_t height, chain::controller& chain) {
+   bool snapshot_executed = false;
+
+   auto execute_snapshot_with_log = [this, height, &snapshot_executed, &chain](const auto& req) {
+      // one snapshot per height
+      if(!snapshot_executed) {
+         dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}",
+              ("start_block_num", req.start_block_num)("end_block_num", req.end_block_num)("block_spacing", req.block_spacing)("height", height));
+
+         execute_snapshot(req.snapshot_request_id, chain);
+         snapshot_executed = true;
+      }
+   };
+
+   std::vector<uint32_t> unschedule_snapshot_request_ids;
+   for(const auto& req: _snapshot_requests.get<0>()) {
+      // -1 since its called from start block
+      bool recurring_snapshot  = req.block_spacing && (height >= req.start_block_num + 1) && (!((height - req.start_block_num - 1) % req.block_spacing));
+      bool onetime_snapshot    = (!req.block_spacing) && (height == req.start_block_num + 1);
+      
+      bool marked_for_deletion = ((!req.block_spacing) && (height >= req.start_block_num + 1)) || // if one time snapshot executed or scheduled for the past, it should be gone
+                                 (height > 0 && ((height-1) >= req.end_block_num));               // any snapshot can expire by end block num (end_block_num can be max value)
+
+      if(recurring_snapshot || onetime_snapshot) {
+         execute_snapshot_with_log(req);
+      }
+
+      // cleanup - remove expired (or invalid) request
+      if(marked_for_deletion) {
+         unschedule_snapshot_request_ids.push_back(req.snapshot_request_id);
+      }
+   }
+
+   for(const auto& i: unschedule_snapshot_request_ids) {
+      unschedule_snapshot(i);
+   }
+}
+
+void snapshot_scheduler::on_irreversible_block(const signed_block_ptr& lib, const chain::controller& chain) {
+   auto& snapshots_by_height = _pending_snapshot_index.get<by_height>();
+   uint32_t lib_height = lib->block_num();
+
+   while(!snapshots_by_height.empty() && snapshots_by_height.begin()->get_height() <= lib_height) {
+      const auto& pending = snapshots_by_height.begin();
+      auto next = pending->next;
+
+      try {
+         next(pending->finalize(chain));
+      }
+      CATCH_AND_CALL(next);
+
+      snapshots_by_height.erase(snapshots_by_height.begin());
+   }
+}
+
+snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::schedule_snapshot(const snapshot_request_information& sri) {
+   auto& snapshot_by_value = _snapshot_requests.get<by_snapshot_value>();
+   auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
+   EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request");
+   EOS_ASSERT(sri.start_block_num <= sri.end_block_num, chain::invalid_snapshot_request, "End block number should be greater or equal to start block number");
+   EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, chain::invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
+
+   _snapshot_requests.emplace(snapshot_schedule_information{{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}, {}});
+   x_serialize();
+
+   // returning snapshot_schedule_result
+   return snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}};
+}
+
+snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::unschedule_snapshot(uint32_t sri) {
+   auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+   auto existing = snapshot_by_id.find(sri);
+   EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found");
+
+   snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}};
+   _snapshot_requests.erase(existing);
+   x_serialize();
+
+   // returning snapshot_schedule_result
+   return result;
+}
+
+snapshot_scheduler::get_snapshot_requests_result snapshot_scheduler::get_snapshot_requests() {
+   get_snapshot_requests_result result;
+   auto& asvector = _snapshot_requests.get<as_vector>();
+   result.snapshot_requests.reserve(asvector.size());
+   result.snapshot_requests.insert(result.snapshot_requests.begin(), asvector.begin(), asvector.end());
+   return result;
+}
+
+void snapshot_scheduler::set_db_path(fs::path db_path) {
+   _snapshot_db.set_path(std::move(db_path));
+   // init from db
+   if(std::filesystem::exists(_snapshot_db.get_json_path())) {
+      std::vector<snapshot_schedule_information> sr;
+      _snapshot_db >> sr;
+      // if db read succeeded, clear/load
+      _snapshot_requests.get<by_snapshot_id>().clear();
+      _snapshot_requests.insert(sr.begin(), sr.end());
+   }
+}
+
+void snapshot_scheduler::set_snapshots_path(fs::path sn_path) {
+   _snapshots_dir = std::move(sn_path);
+}
+
+void snapshot_scheduler::add_pending_snapshot_info(const snapshot_information& si) {
+   auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+   auto snapshot_req = snapshot_by_id.find(_inflight_sid);
+   if(snapshot_req != snapshot_by_id.end()) {
+      _snapshot_requests.modify(snapshot_req, [&si](auto& p) {
+         p.pending_snapshots.emplace_back(si);
+      });
+   }
+}
+
+void snapshot_scheduler::execute_snapshot(uint32_t srid, chain::controller& chain) {
+   _inflight_sid = srid;
+   auto next = [srid, this](const chain::next_function_variant<snapshot_information>& result) {
+      if(std::holds_alternative<fc::exception_ptr>(result)) {
+         try {
+            std::get<fc::exception_ptr>(result)->dynamic_rethrow_exception();
+         } catch(const fc::exception& e) {
+            EOS_THROW(snapshot_execution_exception,
+                     "Snapshot creation error: ${details}",
+                     ("details", e.to_detail_string()));
+         } catch(const std::exception& e) {
+            EOS_THROW(snapshot_execution_exception,
+                     "Snapshot creation error: ${details}",
+                     ("details", e.what()));
+         }
+      } else {
+         // success, snapshot finalized
+         auto snapshot_info = std::get<snapshot_information>(result);
+         auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+         auto snapshot_req = snapshot_by_id.find(srid);
+
+         if(snapshot_req != snapshot_by_id.end()) {
+            _snapshot_requests.modify(snapshot_req, [&](auto& p) {
+               auto& pending = p.pending_snapshots;
+               auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const snapshot_information& s) { return s.head_block_num <= snapshot_info.head_block_num; });
+               pending.erase(it, pending.end());
+            });
+         }
+      }
+   };
+   create_snapshot(next, chain, {});
+}
+
+void snapshot_scheduler::create_snapshot(next_function<snapshot_information> next, chain::controller& chain, std::function<void(void)> predicate) {
+   auto head_id = chain.head_block_id();
+   const auto head_block_num = chain.head_block_num();
+   const auto head_block_time = chain.head_block_time();
+   const auto& snapshot_path = pending_snapshot<snapshot_information>::get_final_path(head_id, _snapshots_dir);
+   const auto& temp_path = pending_snapshot<snapshot_information>::get_temp_path(head_id, _snapshots_dir);
+
+   // maintain legacy exception if the snapshot exists
+   if(fs::is_regular_file(snapshot_path)) {
+      auto ex = snapshot_exists_exception(FC_LOG_MESSAGE(error, "snapshot named ${name} already exists", ("name", _snapshots_dir)));
+      next(ex.dynamic_copy_exception());
+      return;
+   }
+
+   auto write_snapshot = [&](const fs::path& p) -> void {
+      if(predicate) predicate();
+      fs::create_directory(p.parent_path());
+      auto snap_out = std::ofstream(p.generic_string(), (std::ios::out | std::ios::binary));
+      auto writer = std::make_shared<ostream_snapshot_writer>(snap_out);
+      chain.write_snapshot(writer);
+      writer->finalize();
+      snap_out.flush();
+      snap_out.close();
+   };
+
+   // If in irreversible mode, create snapshot and return path to snapshot immediately.
+   if(chain.get_read_mode() == db_read_mode::IRREVERSIBLE) {
+      try {
+         write_snapshot(temp_path);
+         std::error_code ec;
+         fs::rename(temp_path, snapshot_path, ec);
+         EOS_ASSERT(!ec, snapshot_finalization_exception,
+                    "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}",
+                    ("bn", head_block_num)("ec", ec.value())("message", ec.message()));
+
+         next(snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, snapshot_path.generic_string()});
+      }
+      CATCH_AND_CALL(next);
+      return;
+   }
+
+   // Otherwise, the result will be returned when the snapshot becomes irreversible.
+
+   // determine if this snapshot is already in-flight
+   auto& pending_by_id = _pending_snapshot_index.get<by_id>();
+   auto existing = pending_by_id.find(head_id);
+   if(existing != pending_by_id.end()) {
+      // if a snapshot at this block is already pending, attach this requests handler to it
+      pending_by_id.modify(existing, [&next](auto& entry) {
+         entry.next = [prev = entry.next, next](const next_function_variant<snapshot_information>& res) {
+            prev(res);
+            next(res);
+         };
+      });
+   } else {
+      const auto& pending_path = pending_snapshot<snapshot_information>::get_pending_path(head_id, _snapshots_dir);
+
+      try {
+         write_snapshot(temp_path);// create a new pending snapshot
+
+         std::error_code ec;
+         fs::rename(temp_path, pending_path, ec);
+         EOS_ASSERT(!ec, snapshot_finalization_exception,
+                    "Unable to promote temp snapshot to pending for block number ${bn}: [code: ${ec}] ${message}",
+                    ("bn", head_block_num)("ec", ec.value())("message", ec.message()));
+         _pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string());
+         add_pending_snapshot_info(snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, pending_path.generic_string()});
+      }
+      CATCH_AND_CALL(next);
+   }
+}
+
+}// namespace eosio::chain

libraries/chain/symbol.cpp

@@ -0,0 +1,21 @@
+#include <eosio/chain/symbol.hpp>
+#include <boost/algorithm/string.hpp>
+
+namespace eosio::chain {
+   
+   symbol symbol::from_string(const string& from)
+{
+   try {
+      string s = boost::algorithm::trim_copy(from);
+      EOS_ASSERT(!s.empty(), symbol_type_exception, "creating symbol from empty string");
+      auto comma_pos = s.find(',');
+      EOS_ASSERT(comma_pos != string::npos, symbol_type_exception, "missing comma in symbol");
+      auto prec_part = s.substr(0, comma_pos);
+      uint8_t p = fc::to_int64(prec_part);
+      string name_part = s.substr(comma_pos + 1);
+      EOS_ASSERT( p <= max_precision, symbol_type_exception, "precision ${p} should be <= 18", ("p", p));
+      return symbol(string_to_symbol(p, name_part.c_str()));
+   } FC_CAPTURE_LOG_AND_RETHROW((from));
+}
+   
+} // namespace eosio::chain

libraries/chain/transaction_context.cpp

@@ -469,7 +469,7 @@ namespace eosio { namespace chain {
                      "not enough time left in block to complete executing transaction ${billing_timer}us",
                      ("now", now)("deadline", _deadline)("start", start)("billing_timer", now - pseudo_start) );
       } else if( deadline_exception_code == tx_cpu_usage_exceeded::code_value ) {
-         std::string assert_msg = "transaction was executing for too long ${billing_timer}us";
+         std::string assert_msg = "transaction ${id} was executing for too long ${billing_timer}us";
          if (subjective_cpu_bill_us > 0) {
             assert_msg += " with a subjective cpu of (${subjective} us)";
          }
@@ -477,10 +477,10 @@ namespace eosio { namespace chain {
          assert_msg += get_tx_cpu_usage_exceeded_reason_msg(limit);
          if (cpu_limit_due_to_greylist) {
             assert_msg = "greylisted " + assert_msg;
-            EOS_THROW( greylist_cpu_usage_exceeded, assert_msg,
+            EOS_THROW( greylist_cpu_usage_exceeded, assert_msg, ("id", packed_trx.id())
                      ("billing_timer", now - pseudo_start)("subjective", subjective_cpu_bill_us)("limit", limit) );
          } else {
-            EOS_THROW( tx_cpu_usage_exceeded, assert_msg,
+            EOS_THROW( tx_cpu_usage_exceeded, assert_msg, ("id", packed_trx.id())
                      ("billing_timer", now - pseudo_start)("subjective", subjective_cpu_bill_us)("limit", limit) );
          }
       } else if( deadline_exception_code == leeway_deadline_exception::code_value ) {

libraries/chain/wasm_interface.cpp

@@ -32,7 +32,7 @@
 
 namespace eosio { namespace chain {
 
-   wasm_interface::wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile)
+   wasm_interface::wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile)
      : my( new wasm_interface_impl(vm, eosvmoc_tierup, d, data_dir, eosvmoc_config, profile) ), vm( vm ) {}
 
    wasm_interface::~wasm_interface() {}
@@ -92,8 +92,9 @@ namespace eosio { namespace chain {
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if(my->eosvmoc) {
          const chain::eosvmoc::code_descriptor* cd = nullptr;
+         chain::eosvmoc::code_cache_base::get_cd_failure failure = chain::eosvmoc::code_cache_base::get_cd_failure::temporary;
          try {
-            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version);
+            cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version, context.control.is_write_window(), failure);
          }
          catch(...) {
             //swallow errors here, if EOS VM OC has gone in to the weeds we shouldn't bail: continue to try and run baseline
@@ -107,15 +108,18 @@ namespace eosio { namespace chain {
             my->eosvmoc->exec->execute(*cd, my->eosvmoc->mem, context);
             return;
          }
+         else if (context.trx_context.is_read_only()) {
+            if (failure == chain::eosvmoc::code_cache_base::get_cd_failure::temporary) {
+               EOS_ASSERT(false, ro_trx_vm_oc_compile_temporary_failure, "get_descriptor_for_code failed with temporary failure");
+            } else {
+               EOS_ASSERT(false, ro_trx_vm_oc_compile_permanent_failure, "get_descriptor_for_code failed with permanent failure");
+            }
+         }
       }
 #endif
       my->get_instantiated_module(code_hash, vm_type, vm_version, context.trx_context)->apply(context);
    }
 
-   void wasm_interface::exit() {
-      my->runtime_interface->immediately_exit_currently_running_module();
-   }
-
    bool wasm_interface::is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const {
       return my->is_code_cached(code_hash, vm_type, vm_version);
    }

libraries/chain/webassembly/cf_system.cpp

@@ -45,7 +45,8 @@ namespace eosio { namespace chain { namespace webassembly {
       }
    }
 
+   //be aware that EOS VM OC handles eosio_exit internally and this function will not be called by OC
    void interface::eosio_exit( int32_t code ) const {
-      context.control.get_wasm_interface().exit();
+      throw wasm_exit{};
    }
 }}} // ns eosio::chain::webassembly

libraries/chain/webassembly/crypto.cpp

@@ -118,14 +118,18 @@ namespace eosio { namespace chain { namespace webassembly {
 
    int32_t interface::alt_bn128_add(span<const char> op1, span<const char> op2, span<char> result ) const {
       if (op1.size() != 64 ||  op2.size() != 64 ||  result.size() < 64 ||
-         bn256::g1_add({(const uint8_t*)op1.data(), 64}, {(const uint8_t*)op2.data(), 64}, { (uint8_t*)result.data(), 64}) == -1)
+         bn256::g1_add(std::span<const uint8_t, 64>{(const uint8_t*)op1.data(), 64},
+                       std::span<const uint8_t, 64>{(const uint8_t*)op2.data(), 64},
+                       std::span<uint8_t, 64>{ (uint8_t*)result.data(), 64}) == -1)
          return return_code::failure;
       return return_code::success;
    }
 
    int32_t interface::alt_bn128_mul(span<const char> g1_point, span<const char> scalar, span<char> result) const {
       if (g1_point.size() != 64 ||  scalar.size() != 32 ||  result.size() < 64 ||
-         bn256::g1_scalar_mul({(const uint8_t*)g1_point.data(), 64}, {(const uint8_t*)scalar.data(), 32}, { (uint8_t*)result.data(), 64}) == -1)
+         bn256::g1_scalar_mul(std::span<const uint8_t, 64>{(const uint8_t*)g1_point.data(), 64},
+                              std::span<const uint8_t, 32>{(const uint8_t*)scalar.data(), 32},
+                              std::span<uint8_t, 64>{ (uint8_t*)result.data(), 64}) == -1)
          return return_code::failure;
       return return_code::success;
    }

libraries/chain/webassembly/runtimes/eos-vm-oc.cpp

@@ -28,7 +28,7 @@ class eosvmoc_instantiated_module : public wasm_instantiated_module_interface {
       bool is_main_thread() { return _main_thread_id == std::this_thread::get_id(); };
 
       void apply(apply_context& context) override {
-         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version);
+         const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version, context.control.is_write_window());
          EOS_ASSERT(cd, wasm_execution_error, "EOS VM OC instantiation failed");
 
          if ( is_main_thread() )
@@ -43,21 +43,18 @@ class eosvmoc_instantiated_module : public wasm_instantiated_module_interface {
       std::thread::id                _main_thread_id;
 };
 
-eosvmoc_runtime::eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db)
+eosvmoc_runtime::eosvmoc_runtime(const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db)
    : cc(data_dir, eosvmoc_config, db), exec(cc), mem(wasm_constraints::maximum_linear_memory/wasm_constraints::wasm_page_size) {
 }
 
 eosvmoc_runtime::~eosvmoc_runtime() {
 }
 
-std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+std::unique_ptr<wasm_instantiated_module_interface> eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                         const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) {
    return std::make_unique<eosvmoc_instantiated_module>(code_hash, vm_type, *this);
 }
 
-//never called. EOS VM OC overrides eosio_exit to its own implementation
-void eosvmoc_runtime::immediately_exit_currently_running_module() {}
-
 void eosvmoc_runtime::init_thread_local_data() {
    exec_thread_local = std::make_unique<eosvmoc::executor>(cc);
 }

libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp

@@ -20,7 +20,6 @@
 #include "LLVMJIT.h"
 
 using namespace IR;
-using namespace Runtime;
 
 namespace eosio { namespace chain { namespace eosvmoc {
 
@@ -39,7 +38,7 @@ static constexpr size_t descriptor_ptr_from_file_start = header_offset + offseto
 
 static_assert(sizeof(code_cache_header) <= header_size, "code_cache_header too big");
 
-code_cache_async::code_cache_async(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
+code_cache_async::code_cache_async(const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
    code_cache_base(data_dir, eosvmoc_config, db),
    _result_queue(eosvmoc_config.threads * 2),
    _threads(eosvmoc_config.threads)
@@ -107,10 +106,11 @@ std::tuple<size_t, size_t> code_cache_async::consume_compile_thread_queue() {
 }
 
 
-const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version) {
+const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window, get_cd_failure& failure) {
    //if there are any outstanding compiles, process the result queue now
-   //do this only on main thread (which is in single threaded write window)
-   if(is_main_thread() && _outstanding_compiles_and_poison.size()) {
+   //When app is in write window, all tasks are running sequentially and read-only threads
+   //are not running. Safe to update cache entries.
+   if(is_write_window && _outstanding_compiles_and_poison.size()) {
       auto [count_processed, bytes_remaining] = consume_compile_thread_queue();
 
       if(count_processed)
@@ -136,37 +136,48 @@ const code_descriptor* const code_cache_async::get_descriptor_for_code(const dig
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_main_thread())
+      if (is_write_window)
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_main_thread()) // on read-only thread
+   if(!is_write_window) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
+   }
 
    const code_tuple ct = code_tuple{code_id, vm_version};
 
-   if(_blacklist.find(ct) != _blacklist.end())
+   if(_blacklist.find(ct) != _blacklist.end()) {
+      failure = get_cd_failure::permanent; // Compile will not start
       return nullptr;
+   }
    if(auto it = _outstanding_compiles_and_poison.find(ct); it != _outstanding_compiles_and_poison.end()) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       it->second = false;
       return nullptr;
    }
-   if(_queued_compiles.find(ct) != _queued_compiles.end())
+   if(_queued_compiles.find(ct) != _queued_compiles.end()) {
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
+   }
 
    if(_outstanding_compiles_and_poison.size() >= _threads) {
       _queued_compiles.emplace(ct);
+      failure = get_cd_failure::temporary; // Compile might not be done yet
       return nullptr;
    }
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
-   if(!codeobject) //should be impossible right?
+   if(!codeobject) { //should be impossible right?
+      failure = get_cd_failure::permanent; // Compile will not start
       return nullptr;
+   }
 
    _outstanding_compiles_and_poison.emplace(ct, false);
    std::vector<wrapped_fd> fds_to_pass;
    fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code));
    write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ ct }, fds_to_pass);
+   failure = get_cd_failure::temporary; // Compile might not be done yet
    return nullptr;
 }
 
@@ -179,15 +190,15 @@ code_cache_sync::~code_cache_sync() {
       elog("unexpected response from EOS VM OC compile monitor during shutdown");
 }
 
-const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version) {
+const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window) {
    //check for entry in cache
    code_cache_index::index<by_hash>::type::iterator it = _cache_index.get<by_hash>().find(boost::make_tuple(code_id, vm_version));
    if(it != _cache_index.get<by_hash>().end()) {
-      if (is_main_thread())
+      if (is_write_window)
          _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it));
       return &*it;
    }
-   if(!is_main_thread())
+   if(!is_write_window)
       return nullptr;
 
    const code_object* const codeobject = _db.find<code_object,by_code_hash>(boost::make_tuple(code_id, 0, vm_version));
@@ -210,20 +221,19 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const
    return &*_cache_index.push_front(std::move(std::get<code_descriptor>(result.result))).first;
 }
 
-code_cache_base::code_cache_base(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
+code_cache_base::code_cache_base(const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
    _db(db),
-   _cache_file_path(data_dir/"code_cache.bin"),
-   _main_thread_id(std::this_thread::get_id())
+   _cache_file_path(data_dir/"code_cache.bin")
 {
    static_assert(sizeof(allocator_t) <= header_offset, "header offset intersects with allocator");
 
-   bfs::create_directories(data_dir);
+   std::filesystem::create_directories(data_dir);
 
-   if(!bfs::exists(_cache_file_path)) {
+   if(!std::filesystem::exists(_cache_file_path)) {
       EOS_ASSERT(eosvmoc_config.cache_size >= allocator_t::get_min_size(total_header_size), database_exception, "configured code cache size is too small");
       std::ofstream ofs(_cache_file_path.generic_string(), std::ofstream::trunc);
       EOS_ASSERT(ofs.good(), database_exception, "unable to create EOS VM Optimized Compiler code cache");
-      bfs::resize_file(_cache_file_path, eosvmoc_config.cache_size);
+      std::filesystem::resize_file(_cache_file_path, eosvmoc_config.cache_size);
       bip::file_mapping creation_mapping(_cache_file_path.generic_string().c_str(), bip::read_write);
       bip::mapped_region creation_region(creation_mapping, bip::read_write);
       new (creation_region.get_address()) allocator_t(eosvmoc_config.cache_size, total_header_size);
@@ -244,9 +254,9 @@ code_cache_base::code_cache_base(const boost::filesystem::path data_dir, const e
 
    set_on_disk_region_dirty(true);
 
-   auto existing_file_size = bfs::file_size(_cache_file_path);
+   auto existing_file_size = std::filesystem::file_size(_cache_file_path);
    if(eosvmoc_config.cache_size > existing_file_size) {
-      bfs::resize_file(_cache_file_path, eosvmoc_config.cache_size);
+      std::filesystem::resize_file(_cache_file_path, eosvmoc_config.cache_size);
 
       bip::file_mapping resize_mapping(_cache_file_path.generic_string().c_str(), bip::read_write);
       bip::mapped_region resize_region(resize_mapping, bip::read_write);
@@ -390,8 +400,4 @@ void code_cache_base::check_eviction_threshold(size_t free_bytes) {
    if(free_bytes < _free_bytes_eviction_threshold)
       run_eviction_round();
 }
-
-bool code_cache_base::is_main_thread() const {
-   return _main_thread_id == std::this_thread::get_id();
-}
 }}}

libraries/chain/webassembly/runtimes/eos-vm-oc/executor.cpp

@@ -15,6 +15,8 @@
 
 #include <boost/hana/equal.hpp>
 
+#include "IR/Types.h"
+
 #include <asm/prctl.h>
 #include <sys/prctl.h>
 #include <sys/syscall.h>

libraries/chain/webassembly/runtimes/eos-vm.cpp

@@ -234,12 +234,7 @@ template<typename Impl>
 eos_vm_runtime<Impl>::eos_vm_runtime() {}
 
 template<typename Impl>
-void eos_vm_runtime<Impl>::immediately_exit_currently_running_module() {
-   throw wasm_exit{};
-}
-
-template<typename Impl>
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_runtime<Impl>::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                              const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eos_vm_backend_t<Impl>;
@@ -261,11 +256,7 @@ template class eos_vm_runtime<eosio::vm::jit>;
 
 eos_vm_profile_runtime::eos_vm_profile_runtime() {}
 
-void eos_vm_profile_runtime::immediately_exit_currently_running_module() {
-   throw wasm_exit{};
-}
-
-std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+std::unique_ptr<wasm_instantiated_module_interface> eos_vm_profile_runtime::instantiate_module(const char* code_bytes, size_t code_size,
                                                                                                const digest_type&, const uint8_t&, const uint8_t&) {
 
    using backend_t = eosio::vm::backend<eos_vm_host_functions_t, eosio::vm::jit_profile, webassembly::eos_vm_runtime::apply_options, vm::profile_instr_map>;

libraries/custom_appbase/include/eosio/chain/application.hpp

@@ -1,87 +1,121 @@
 #pragma once
 
 #include <appbase/application_base.hpp>
-#include <appbase/execution_priority_queue.hpp>
+#include <eosio/chain/exec_pri_queue.hpp>
+#include <chrono>
+#include <mutex>
 
 /*
- * Custmomize appbase to support two-queue exector.
+ * Customize appbase to support two-queue executor.
 */
 namespace appbase { 
 
 enum class exec_window {
-   read,  // the window during which operations from read_only_trx_safe queue
-          // can be executed in app thread in parallel with read-only transactions
-          // in read-only transaction excuting threads.
-   write, // the window during which operations from both general and
-          // read_only_trx_safe queues can be executed in app thread,
-          // while read-only transactions are not executed in read-only
-          // transaction excuting threads.
+   read,  // the window during which operations from read_only queue
+          // can be executed in parallel in the read-only thread pool
+          // as well as in the app thread.
+   write, // the window during which operations from both read_write and
+          // parallel queues can be executed in app thread,
+          // while read-only operations are not executed in read-only
+          // thread pool. The read-only thread pool is not active; only
+          // the main app thread is active.
 };
 
 enum class exec_queue {
-   read_only_trx_safe, // the queue storing operations which are safe to execute
-                       // on app thread in parallel with read-only transactions
-                       // in read-only transaction excuting threads.
-   general             // the queue storing operations which can be only executed
-                       // on the app thread while read-only transactions are
-                       // not being executed in read-only threads
+   read_only,          // the queue storing tasks which are safe to execute
+                       // in parallel with other read-only tasks in the read-only
+                       // thread pool as well as on the main app thread.
+                       // Multi-thread safe as long as nothing is executed from the read_write queue.
+   read_write          // the queue storing tasks which can be only executed
+                       // on the app thread while read-only tasks are
+                       // not being executed in read-only threads. Single threaded.
 };
 
 class two_queue_executor {
 public:
 
+   // Trade off on returning to appbase exec() loop as the overhead of poll/run can be measurable for small running tasks.
+   // This adds to the total time that the main thread can be busy when a high priority task is waiting.
+   static constexpr uint16_t minimum_runtime_ms = 3;
+
    template <typename Func>
    auto post( int priority, exec_queue q, Func&& func ) {
-      if ( q == exec_queue::general )
-         return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      if ( q == exec_queue::read_write )
+         return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
       else
-         return boost::asio::post(io_serv_, read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+         return boost::asio::post( io_serv_, read_only_queue_.wrap( priority, --order_, std::forward<Func>( func)));
    }
 
    // Legacy and deprecated. To be removed after cleaning up its uses in base appbase
    template <typename Func>
    auto post( int priority, Func&& func ) {
-      // safer to use general queue for unknown type of operation since operations
-      // from general queue are not executed in parallel with read-only transactions
-      return boost::asio::post(io_serv_, general_queue_.wrap(priority, --order_, std::forward<Func>(func)));
+      // safer to use read_write queue for unknown type of operation since operations
+      // from read_write queue are not executed in parallel with read-only operations
+      return boost::asio::post(io_serv_, read_write_queue_.wrap(priority, --order_, std::forward<Func>(func)));
    }
 
    boost::asio::io_service& get_io_service() { return io_serv_; }
 
    bool execute_highest() {
-      if ( exec_window_ == exec_window::write ) {
-         if( !general_queue_.empty() && ( read_only_trx_safe_queue_.empty() || *read_only_trx_safe_queue_.top() < *general_queue_.top()) )  {
-            // general_queue_'s top function's priority greater than read_only_trx_safe_queue_'s top function's, or general_queue_ empty
-            general_queue_.execute_highest();
-         } else if( !read_only_trx_safe_queue_.empty() ) {
-            read_only_trx_safe_queue_.execute_highest();
+      // execute for at least minimum runtime
+      const auto end = std::chrono::high_resolution_clock::now() + std::chrono::milliseconds(minimum_runtime_ms);
+
+      bool more = false;
+      while (true) {
+         if ( exec_window_ == exec_window::write ) {
+            // During write window only main thread is accessing anything in two_queue_executor, no locking required
+            if( !read_write_queue_.empty() && (read_only_queue_.empty() || *read_only_queue_.top() < *read_write_queue_.top()) )  {
+               // read_write_queue_'s top function's priority greater than read_only_queue_'s top function's, or read_only_queue_ empty
+               read_write_queue_.execute_highest();
+            } else if( !read_only_queue_.empty() ) {
+               read_only_queue_.execute_highest();
+            }
+            more = !read_only_queue_.empty() || !read_write_queue_.empty();
+         } else {
+            // When in read window, multiple threads including main app thread are accessing two_queue_executor, locking required
+            more = read_only_queue_.execute_highest_locked(false);
          }
-         return !read_only_trx_safe_queue_.empty() || !general_queue_.empty();
-      } else {
-         return read_only_trx_safe_queue_.execute_highest();
+         if (!more || std::chrono::high_resolution_clock::now() > end)
+            break;
       }
+      return more;
+   }
+
+   bool execute_highest_read_only() {
+      // execute for at least minimum runtime
+      const auto end = std::chrono::high_resolution_clock::now() + std::chrono::milliseconds(minimum_runtime_ms);
+
+      bool more = false;
+      while (true) {
+         more = read_only_queue_.execute_highest_locked( true );
+         if (!more || std::chrono::high_resolution_clock::now() > end)
+            break;
+      }
+      return more;
    }
 
    template <typename Function>
-   boost::asio::executor_binder<Function, appbase::execution_priority_queue::executor>
+   boost::asio::executor_binder<Function, appbase::exec_pri_queue::executor>
    wrap(int priority, exec_queue q, Function&& func ) {
-      if ( q == exec_queue::general )
-         return general_queue_.wrap(priority, --order_, std::forward<Function>(func));
+      if ( q == exec_queue::read_write )
+         return read_write_queue_.wrap(priority, --order_, std::forward<Function>(func));
       else
-         return read_only_trx_safe_queue_.wrap(priority, --order_, std::forward<Function>(func));
+         return read_only_queue_.wrap( priority, --order_, std::forward<Function>( func));
    }
      
    void clear() {
-      read_only_trx_safe_queue_.clear();
-      general_queue_.clear();
+      read_only_queue_.clear();
+      read_write_queue_.clear();
    }
 
-   void set_to_read_window() {
+   void set_to_read_window(uint32_t num_threads, std::function<bool()> should_exit) {
       exec_window_ = exec_window::read;
+      read_only_queue_.enable_locking(num_threads, std::move(should_exit));
    }
 
    void set_to_write_window() {
       exec_window_ = exec_window::write;
+      read_only_queue_.disable_locking();
    }
 
    bool is_read_window() const {
@@ -92,15 +126,15 @@ public:
       return exec_window_ == exec_window::write;
    }
 
-   auto& read_only_trx_safe_queue() { return read_only_trx_safe_queue_; }
+   auto& read_only_queue() { return read_only_queue_; }
 
-   auto& general_queue() { return general_queue_; }
+   auto& read_write_queue() { return read_write_queue_; }
 
    // members are ordered taking into account that the last one is destructed first
 private:
    boost::asio::io_service            io_serv_;
-   appbase::execution_priority_queue  read_only_trx_safe_queue_;
-   appbase::execution_priority_queue  general_queue_;
+   appbase::exec_pri_queue            read_only_queue_;
+   appbase::exec_pri_queue            read_write_queue_;
    std::atomic<std::size_t>           order_ { std::numeric_limits<size_t>::max() }; // to maintain FIFO ordering in both queues within priority
    exec_window                        exec_window_ { exec_window::write };
 };

libraries/custom_appbase/include/eosio/chain/exec_pri_queue.hpp

@@ -0,0 +1,233 @@
+#pragma once
+#include <boost/asio.hpp>
+
+#include <condition_variable>
+#include <mutex>
+#include <queue>
+
+namespace appbase {
+// adapted from: https://www.boost.org/doc/libs/1_69_0/doc/html/boost_asio/example/cpp11/invocation/prioritised_handlers.cpp
+
+// Locking has to be coordinated by caller, use with care.
+class exec_pri_queue : public boost::asio::execution_context
+{
+public:
+
+   void enable_locking(uint32_t num_threads, std::function<bool()> should_exit) {
+      assert(num_threads > 0 && num_waiting_ == 0);
+      lock_enabled_ = true;
+      max_waiting_ = num_threads;
+      should_exit_ = std::move(should_exit);
+      exiting_blocking_ = false;
+   }
+
+   void disable_locking() {
+      lock_enabled_ = false;
+      should_exit_ = [](){ assert(false); return true; }; // should not be called when locking is disabled
+   }
+
+   // called from appbase::application_base::exec poll_one() or run_one()
+   template <typename Function>
+   void add(int priority, size_t order, Function function)
+   {
+      std::unique_ptr<queued_handler_base> handler(new queued_handler<Function>(priority, order, std::move(function)));
+      if (lock_enabled_) {
+         std::lock_guard g( mtx_ );
+         handlers_.push( std::move( handler ) );
+         if (num_waiting_)
+            cond_.notify_one();
+      } else {
+         handlers_.push( std::move( handler ) );
+      }
+   }
+
+   // only call when no lock required
+   void clear()
+   {
+      handlers_ = prio_queue();
+   }
+
+   // only call when no lock required
+   bool execute_highest()
+   {
+      if( !handlers_.empty() ) {
+         handlers_.top()->execute();
+         handlers_.pop();
+      }
+
+      return !handlers_.empty();
+   }
+
+private:
+   // has to be defined before use, auto return type
+   auto pop() {
+      auto t = std::move(const_cast<std::unique_ptr<queued_handler_base>&>(handlers_.top()));
+      handlers_.pop();
+      return t;
+   }
+
+public:
+
+   bool execute_highest_locked(bool should_block) {
+      std::unique_lock g(mtx_);
+      if (should_block) {
+         ++num_waiting_;
+         cond_.wait(g, [this](){
+            bool exit = exiting_blocking_ || should_exit_();
+            bool empty = handlers_.empty();
+            if (empty || exit) {
+               if (((empty && num_waiting_ == max_waiting_) || exit) && !exiting_blocking_) {
+                  cond_.notify_all();
+                  exiting_blocking_ = true;
+               }
+               return exit || exiting_blocking_; // same as calling should_exit(), but faster
+            }
+            return true;
+         });
+         --num_waiting_;
+         if (exiting_blocking_ || should_exit_())
+            return false;
+      }
+      if( handlers_.empty() )
+         return false;
+      auto t = pop();
+      g.unlock();
+      t->execute();
+      return true;
+   }
+
+   // Only call when locking disabled
+   size_t size() const { return handlers_.size(); }
+
+   // Only call when locking disabled
+   bool empty() const { return handlers_.empty(); }
+
+   // Only call when locking disabled
+   const auto& top() const { return handlers_.top(); }
+
+   class executor
+   {
+   public:
+      executor(exec_pri_queue& q, int p, size_t o)
+            : context_(q), priority_(p), order_(o)
+      {
+      }
+
+      exec_pri_queue& context() const noexcept
+      {
+         return context_;
+      }
+
+      template <typename Function, typename Allocator>
+      void dispatch(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void post(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      template <typename Function, typename Allocator>
+      void defer(Function f, const Allocator&) const
+      {
+         context_.add(priority_, order_, std::move(f));
+      }
+
+      void on_work_started() const noexcept {}
+      void on_work_finished() const noexcept {}
+
+      bool operator==(const executor& other) const noexcept
+      {
+         return order_ == other.order_ && &context_ == &other.context_ && priority_ == other.priority_;
+      }
+
+      bool operator!=(const executor& other) const noexcept
+      {
+         return !operator==(other);
+      }
+
+   private:
+      exec_pri_queue& context_;
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   boost::asio::executor_binder<Function, executor>
+   wrap(int priority, size_t order, Function&& func)
+   {
+      return boost::asio::bind_executor( executor(*this, priority, order), std::forward<Function>(func) );
+   }
+
+private:
+   class queued_handler_base
+   {
+   public:
+      queued_handler_base( int p, size_t order )
+            : priority_( p )
+            , order_( order )
+      {
+      }
+
+      virtual ~queued_handler_base() = default;
+
+      virtual void execute() = 0;
+
+      int priority() const { return priority_; }
+      // C++20
+      // friend std::weak_ordering operator<=>(const queued_handler_base&,
+      //                                       const queued_handler_base&) noexcept = default;
+      friend bool operator<(const queued_handler_base& a,
+                            const queued_handler_base& b) noexcept
+      {
+         return std::tie( a.priority_, a.order_ ) < std::tie( b.priority_, b.order_ );
+      }
+
+   private:
+      int priority_;
+      size_t order_;
+   };
+
+   template <typename Function>
+   class queued_handler : public queued_handler_base
+   {
+   public:
+      queued_handler(int p, size_t order, Function f)
+            : queued_handler_base( p, order )
+            , function_( std::move(f) )
+      {
+      }
+
+      void execute() override
+      {
+         function_();
+      }
+
+   private:
+      Function function_;
+   };
+
+   struct deref_less
+   {
+      template<typename Pointer>
+      bool operator()(const Pointer& a, const Pointer& b) noexcept(noexcept(*a < *b))
+      {
+         return *a < *b;
+      }
+   };
+
+   bool lock_enabled_ = false;
+   std::mutex mtx_;
+   std::condition_variable cond_;
+   uint32_t num_waiting_{0};
+   uint32_t max_waiting_{0};
+   bool exiting_blocking_{false};
+   std::function<bool()> should_exit_; // called holding mtx_
+   using prio_queue = std::priority_queue<std::unique_ptr<queued_handler_base>, std::deque<std::unique_ptr<queued_handler_base>>, deref_less>;
+   prio_queue handlers_;
+};
+
+} // appbase

libraries/custom_appbase/tests/custom_appbase_tests.cpp

@@ -27,28 +27,28 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
 
    // Stop app. Use the lowest priority to make sure this function to execute the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_only_trx_safe_queue should only contain the current lambda function,
-      // and general_queue should have execute all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_only_queue should only contain the current lambda function,
+      // and read_write_queue should have executed all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of both queues' functions processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 8 );
@@ -64,40 +64,40 @@ BOOST_AUTO_TEST_CASE( default_exec_window ) {
    BOOST_CHECK_LT( rslts[6], rslts[7] );
 }
 
-// verify functions only from read_only_trx_safe queue are processed during read window
+// verify functions only from read_only queue are processed during read window
 BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 4 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 4 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 6 );
@@ -109,40 +109,40 @@ BOOST_AUTO_TEST_CASE( execute_from_read_queue ) {
    BOOST_CHECK_LT( rslts[3], rslts[4] );
 }
 
-// verify no functions are executed during  read window if read_only_trx_safe queue is empty
+// verify no functions are executed during  read window if read_only queue is empty
 BOOST_AUTO_TEST_CASE( execute_from_empty_read_queue ) {
    appbase::scoped_app app;
    auto app_thread = start_app_thread(app);
    
-   // set to run functions from read_only_trx_safe queue only
-   app->executor().set_to_read_window();
+   // set to run functions from read_only queue only
+   app->executor().set_to_read_window(1, [](){return false;});
 
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[9]=seq_num; ++seq_num; } );
 
    // Stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
-      // read_queue should have current function and write_queue should have all its functions
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 10 );
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
+      // read_queue should be empty (read window pops before execute) and write_queue should have all its functions
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 0); // pop()s before execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 10 );
       app->quit();
       } );
    app_thread.join();
 
    // both queues are cleared after execution 
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // no results
    BOOST_REQUIRE_EQUAL( rslts.size(), 0 );
@@ -159,32 +159,32 @@ BOOST_AUTO_TEST_CASE( execute_from_both_queues ) {
    // post functions
    std::map<int, int> rslts {};
    int seq_num = 0;
-   app->executor().post( priority::medium, exec_queue::read_only_trx_safe,  [&]() { rslts[0]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[1]=seq_num; ++seq_num; } );
-   app->executor().post( priority::high,   exec_queue::general, [&]() { rslts[2]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[3]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::read_only_trx_safe,  [&]() { rslts[4]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[5]=seq_num; ++seq_num; } );
-   app->executor().post( priority::highest,exec_queue::read_only_trx_safe,  [&]() { rslts[6]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[7]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[8]=seq_num; ++seq_num; } );
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe,  [&]() { rslts[9]=seq_num; ++seq_num; } );
-   app->executor().post( priority::low,    exec_queue::general, [&]() { rslts[10]=seq_num; ++seq_num; } );
-   app->executor().post( priority::medium, exec_queue::general, [&]() { rslts[11]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_only,  [&]() { rslts[0]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[1]=seq_num; ++seq_num; } );
+   app->executor().post( priority::high,   exec_queue::read_write, [&]() { rslts[2]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[3]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_only,  [&]() { rslts[4]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[5]=seq_num; ++seq_num; } );
+   app->executor().post( priority::highest,exec_queue::read_only,  [&]() { rslts[6]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[7]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[8]=seq_num; ++seq_num; } );
+   app->executor().post( priority::lowest, exec_queue::read_only,  [&]() { rslts[9]=seq_num; ++seq_num; } );
+   app->executor().post( priority::low,    exec_queue::read_write, [&]() { rslts[10]=seq_num; ++seq_num; } );
+   app->executor().post( priority::medium, exec_queue::read_write, [&]() { rslts[11]=seq_num; ++seq_num; } );
 
    // stop application. Use lowest at the end to make sure this executes the last
-   app->executor().post( priority::lowest, exec_queue::read_only_trx_safe, [&]() {
+   app->executor().post( priority::lowest, exec_queue::read_only, [&]() {
       // read_queue should have current function and write_queue's functions are all executed 
-      BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().size(), 1);
-      BOOST_REQUIRE_EQUAL( app->executor().general_queue().size(), 0 );
+      BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().size(), 1); // pop()s after execute
+      BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().size(), 0 );
       app->quit();
       } );
 
    app_thread.join();
 
    // queues are emptied after quit
-   BOOST_REQUIRE_EQUAL( app->executor().read_only_trx_safe_queue().empty(), true);
-   BOOST_REQUIRE_EQUAL( app->executor().general_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_only_queue().empty(), true);
+   BOOST_REQUIRE_EQUAL( app->executor().read_write_queue().empty(), true);
 
    // exactly number of posts processed
    BOOST_REQUIRE_EQUAL( rslts.size(), 12 );

libraries/libfc/CMakeLists.txt

@@ -9,7 +9,6 @@ find_package(OpenSSL REQUIRED)
 
 set( fc_sources
      src/uint128.cpp
-     src/real128.cpp
      src/variant.cpp
      src/exception.cpp
      src/variant_object.cpp
@@ -56,9 +55,6 @@ set( fc_sources
      src/crypto/modular_arithmetic.cpp
      src/crypto/blake2.cpp
      src/crypto/k1_recover.cpp
-     src/network/ip.cpp
-     src/network/resolve.cpp
-     src/network/udp_socket.cpp
      src/network/url.cpp
      src/network/http/http_client.cpp
      src/compress/zlib.cpp
@@ -97,7 +93,6 @@ endif()
 
 find_package(Boost 1.66 REQUIRED COMPONENTS
     date_time
-    filesystem
     chrono
     unit_test_framework
     iostreams)
@@ -125,11 +120,7 @@ find_package(ZLIB REQUIRED)
 target_include_directories(fc PUBLIC include)
 
 # try and make this very clear that this json parser is intended only for webauthn parsing..
-if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.11.0)
-   set_source_files_properties(src/crypto/elliptic_webauthn.cpp PROPERTIES INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/include/fc/crypto/webauthn_json/include")
-else()
-   set_source_files_properties(src/crypto/elliptic_webauthn.cpp PROPERTIES COMPILE_FLAGS "-I${CMAKE_CURRENT_SOURCE_DIR}/include/fc/crypto/webauthn_json/include")
-endif()
+set_source_files_properties(src/crypto/elliptic_webauthn.cpp PROPERTIES COMPILE_FLAGS "-I${CMAKE_CURRENT_SOURCE_DIR}/include/fc/crypto/webauthn_json/include")
 
 if(WIN32)
   target_link_libraries( fc PUBLIC ws2_32 mswsock userenv )
@@ -139,7 +130,7 @@ if(APPLE)
   find_library(security_framework Security)
   find_library(corefoundation_framework CoreFoundation)
 endif()
-target_link_libraries( fc PUBLIC Boost::date_time Boost::filesystem Boost::chrono Boost::iostreams Threads::Threads
+target_link_libraries( fc PUBLIC Boost::date_time Boost::chrono Boost::iostreams Threads::Threads
                                  OpenSSL::Crypto ZLIB::ZLIB ${PLATFORM_SPECIFIC_LIBS} ${CMAKE_DL_LIBS} secp256k1 ${security_framework} ${corefoundation_framework})
 
 # Critically, this ensures that OpenSSL 1.1 & 3.0 both have a variant of BN_zero() with void return value. But it also allows access

libraries/libfc/include/fc/any.hpp

@@ -1,7 +0,0 @@
-#pragma once
-#include <boost/any.hpp>
-
-namespace fc { 
-  // TODO: define this without using boost
-  typedef boost::any any;
-} 

libraries/libfc/include/fc/api.hpp

@@ -1,134 +0,0 @@
-#pragma once 
-#include <fc/thread/future.hpp>
-#include <fc/any.hpp>
-#include <functional>
-#include <boost/config.hpp>
-
-// ms visual c++ (as of 2013) doesn't accept the standard syntax for calling a 
-// templated member function (foo->template bar();)
-#ifdef _MSC_VER
-# define FC_CALL_MEMBER_TEMPLATE_KEYWORD
-#else
-# define FC_CALL_MEMBER_TEMPLATE_KEYWORD template
-#endif
-
-namespace fc {
-  struct identity_member { 
-       template<typename R, typename C, typename P, typename... Args>
-       static std::function<R(Args...)> functor( P&& p, R (C::*mem_func)(Args...) );
-       template<typename R, typename C, typename P, typename... Args>
-       static std::function<R(Args...)> functor( P&& p, R (C::*mem_func)(Args...)const );
-  };
-   
-  template< typename Interface, typename Transform  >
-  struct vtable  : public std::enable_shared_from_this<vtable<Interface,Transform>> 
-  { private: vtable(); };
-  
-  template<typename OtherType>
-  struct vtable_copy_visitor {
-      typedef OtherType other_type;
-
-      vtable_copy_visitor( OtherType& s):_source( s ){}
-  
-      template<typename R, typename MemberPtr, typename... Args>
-      void operator()( const char* name, std::function<R(Args...)>& memb, MemberPtr m )const {
-        OtherType* src = &_source;
-        memb = [src,m]( Args... args ){ return (src->*m)(args...); };
-      }
-      OtherType& _source;
-  };
-
-  template<typename Interface, typename Transform >
-  class api;
-
-  class api_connection;
-
-  typedef uint32_t api_id_type;
-
-  class api_base
-  {
-     public:
-        api_base() {}
-        virtual ~api_base() {}
-
-        virtual uint64_t get_handle()const = 0;
-
-        virtual api_id_type register_api( api_connection& conn )const = 0;
-
-        // defined in api_connection.hpp
-        template< typename T >
-        api<T, identity_member> as();
-  };
-  typedef std::shared_ptr< api_base > api_ptr;
-
-  class api_connection;
-
-  template<typename Interface, typename Transform = identity_member >
-  class api : public api_base {
-    public:
-      typedef vtable<Interface,Transform> vtable_type;
-
-      api():_vtable( std::make_shared<vtable_type>() ) {}
-
-      /** T is anything with pointer semantics */
-      template<typename T >
-      api( const T& p )
-      :_vtable( std::make_shared<vtable_type>() )
-      {
-         _data = std::make_shared<fc::any>(p);
-         T& ptr = boost::any_cast<T&>(*_data);
-         auto& pointed_at = *ptr;
-         typedef typename std::remove_reference<decltype(pointed_at)>::type source_vtable_type;
-         _vtable->FC_CALL_MEMBER_TEMPLATE_KEYWORD visit_other( vtable_copy_visitor<source_vtable_type>(pointed_at) );
-      }
-
-      api( const api& cpy ):_vtable(cpy._vtable),_data(cpy._data) {}
-      virtual ~api() {}
-
-      friend bool operator == ( const api& a, const api& b ) { return a._data == b._data && a._vtable == b._vtable;    }
-      friend bool operator != ( const api& a, const api& b ) { return !(a._data == b._data && a._vtable == b._vtable); }
-      virtual uint64_t get_handle()const override { return uint64_t(_data.get()); }
-      virtual api_id_type register_api( api_connection& conn )const override;    // defined in api_connection.hpp
-
-      vtable_type& operator*()const  { FC_ASSERT( _vtable ); return *_vtable; }
-      vtable_type* operator->()const {  FC_ASSERT( _vtable ); return _vtable.get(); }
-
-    protected:
-      std::shared_ptr<vtable_type>    _vtable;
-      std::shared_ptr<fc::any>        _data;
-  };
-
-} // namespace fc
-
-#include <boost/preprocessor/repeat.hpp>
-#include <boost/preprocessor/repetition/enum_binary_params.hpp>
-#include <boost/preprocessor/repetition/enum_params.hpp>
-#include <boost/preprocessor/repetition/enum_trailing_params.hpp>
-#include <boost/preprocessor/facilities/empty.hpp>
-#include <boost/preprocessor/seq/for_each.hpp>
-#include <boost/preprocessor/stringize.hpp>
-
-#define FC_API_VTABLE_DEFINE_MEMBER( r, data, elem ) \
-      decltype(Transform::functor( (data*)nullptr, &data::elem)) elem; 
-#define FC_API_VTABLE_DEFINE_VISIT_OTHER( r, data, elem ) \
-        { typedef typename Visitor::other_type OtherType; \
-        v( BOOST_PP_STRINGIZE(elem), elem, &OtherType::elem ); }
-#define FC_API_VTABLE_DEFINE_VISIT( r, data, elem ) \
-        v( BOOST_PP_STRINGIZE(elem), elem ); 
-
-#define FC_API( CLASS, METHODS ) \
-namespace fc { \
-  template<typename Transform> \
-  struct vtable<CLASS,Transform> : public std::enable_shared_from_this<vtable<CLASS,Transform>> { \
-      BOOST_PP_SEQ_FOR_EACH( FC_API_VTABLE_DEFINE_MEMBER, CLASS, METHODS ) \
-      template<typename Visitor> \
-      void visit_other( Visitor&& v ){ \
-        BOOST_PP_SEQ_FOR_EACH( FC_API_VTABLE_DEFINE_VISIT_OTHER, CLASS, METHODS ) \
-      } \
-      template<typename Visitor> \
-      void visit( Visitor&& v ){ \
-        BOOST_PP_SEQ_FOR_EACH( FC_API_VTABLE_DEFINE_VISIT, CLASS, METHODS ) \
-      } \
-  }; \
-}  
-

libraries/libfc/include/fc/array.hpp

@@ -124,7 +124,7 @@ namespace fc {
   {
      static const char* name()
      {
-        static std::string _name = std::string("fc::array<")+std::string(fc::get_typename<T>::name())+","+ fc::to_string(N) + ">";
+        static std::string _name = std::string("fc::array<")+std::string(fc::get_typename<T>::name())+","+ std::to_string(N) + ">";
         return _name.c_str();
      }
   };

libraries/libfc/include/fc/compress/zlib.hpp

@@ -1,10 +1,10 @@
 #pragma once
 
-#include <fc/string.hpp>
+#include <string>
 
 namespace fc 
 {
 
-  string zlib_compress(const string& in);
+   std::string zlib_compress(const std::string& in);
 
 } // namespace fc

libraries/libfc/include/fc/container/deque.hpp

@@ -1,12 +0,0 @@
-#pragma once
-
-#include <deque>
-#include <fc/io/raw.hpp>
-
-namespace fc {
-   namespace raw {
-
-
-    } // namespace raw
-
-} // namespace fc

libraries/libfc/include/fc/crypto/aes.hpp

@@ -6,7 +6,6 @@
 #include <vector>
 
 namespace fc {
-    class path;
 
     class aes_encoder
     {
@@ -42,13 +41,13 @@ namespace fc {
     std::vector<char> aes_decrypt( const fc::sha512& key, const std::vector<char>& cipher_text );
 
     /** encrypts plain_text and then includes a checksum that enables us to verify the integrety of
-     * the file / key prior to decryption. 
+     * the file / key prior to decryption.
      */
-    void              aes_save( const fc::path& file, const fc::sha512& key, std::vector<char> plain_text );
+    void              aes_save( const std::filesystem::path& file, const fc::sha512& key, std::vector<char> plain_text );
 
     /**
      *  recovers the plain_text saved via aes_save()
      */
-    std::vector<char> aes_load( const fc::path& file, const fc::sha512& key );
+    std::vector<char> aes_load( const std::filesystem::path& file, const fc::sha512& key );
 
-} // namespace fc 
+} // namespace fc

libraries/libfc/include/fc/crypto/bigint.hpp

@@ -55,7 +55,7 @@ namespace fc {
       bigint operator--(int);
       bigint& operator--();
 
-      operator fc::string()const;
+      operator std::string()const;
 
       // returns bignum as bigendian bytes
       operator std::vector<char>()const;

libraries/libfc/include/fc/crypto/common.hpp

@@ -105,7 +105,7 @@ namespace fc { namespace crypto {
          auto data_str = to_base58( packed.data(), packed.size(), _yield );
          _yield();
          if (!is_default) {
-            data_str = string(Prefixes[position]) + "_" + data_str;
+            data_str = std::string(Prefixes[position]) + "_" + data_str;
          }
          _yield();
 
@@ -178,7 +178,7 @@ namespace fc { namespace crypto {
       {}
 
       shim(data_type&& data)
-      :_data(forward<data_type>(data))
+         :_data(std::move(data))
       {}
 
       shim(const data_type& data)

libraries/libfc/include/fc/crypto/hex.hpp

@@ -1,15 +1,15 @@
 #pragma once
-#include <fc/string.hpp>
 #include <fc/utility.hpp>
+#include <string>
 #include <vector>
 
 namespace fc {
     uint8_t from_hex( char c );
-    fc::string to_hex( const char* d, uint32_t s );
+    std::string to_hex( const char* d, uint32_t s );
     std::string to_hex( const std::vector<char>& data );
 
     /**
      *  @return the number of bytes decoded
      */
-    size_t from_hex( const fc::string& hex_str, char* out_data, size_t out_data_len );
+    size_t from_hex( const std::string& hex_str, char* out_data, size_t out_data_len );
 } 

libraries/libfc/include/fc/crypto/openssl.hpp

@@ -10,14 +10,12 @@
 #include <openssl/obj_mac.h>
 #include <openssl/bn.h>
 
-/** 
+/**
  * @file openssl.hpp
  * Provides common utility calls for wrapping openssl c api.
  */
-namespace fc 
+namespace fc
 {
-  class path;
-
     template <typename ssl_type>
     struct ssl_wrapper
     {

libraries/libfc/include/fc/crypto/private_key.hpp

@@ -46,14 +46,14 @@ namespace fc { namespace crypto {
          }
 
          // serialize to/from string
-         explicit private_key(const string& base58str);
+         explicit private_key(const std::string& base58str);
          std::string to_string(const fc::yield_function_t& yield = fc::yield_function_t()) const;
 
       private:
          storage_type _storage;
 
          private_key( storage_type&& other_storage )
-         :_storage(forward<storage_type>(other_storage))
+            :_storage(other_storage)
          {}
 
          friend bool operator == ( const private_key& p1, const private_key& p2);

libraries/libfc/include/fc/crypto/public_key.hpp

@@ -31,7 +31,7 @@ namespace fc { namespace crypto {
          public_key( const signature& c, const sha256& digest, bool check_canonical = true );
 
          public_key( storage_type&& other_storage )
-         :_storage(forward<storage_type>(other_storage))
+            :_storage(std::move(other_storage))
          {}
 
          bool valid()const;
@@ -39,7 +39,7 @@ namespace fc { namespace crypto {
          size_t which()const;
 
          // serialize to/from string
-         explicit public_key(const string& base58str);
+         explicit public_key(const std::string& base58str);
          std::string to_string(const fc::yield_function_t& yield = fc::yield_function_t()) const;
 
          storage_type _storage;

libraries/libfc/include/fc/crypto/ripemd160.hpp

@@ -12,10 +12,10 @@ class ripemd160
 {
   public:
     ripemd160();
-    explicit ripemd160( const string& hex_str );
+    explicit ripemd160( const std::string& hex_str );
 
-    string str()const;
-    explicit operator string()const;
+    std::string str()const;
+    explicit operator std::string()const;
 
     char*    data()const;
     size_t data_size()const { return 160/8; }
@@ -23,7 +23,7 @@ class ripemd160
     static ripemd160 hash( const fc::sha512& h );
     static ripemd160 hash( const fc::sha256& h );
     static ripemd160 hash( const char* d, uint32_t dlen );
-    static ripemd160 hash( const string& );
+    static ripemd160 hash( const std::string& );
 
     template<typename T>
     static ripemd160 hash( const T& t ) 

libraries/libfc/include/fc/crypto/sha1.hpp

@@ -8,17 +8,17 @@ class sha1
 {
   public:
     sha1();
-    explicit sha1( const string& hex_str );
+    explicit sha1( const std::string& hex_str );
 
-    string str()const;
-    operator string()const;
+    std::string str()const;
+    operator std::string()const;
 
     char*       data();
     const char* data()const;
     size_t data_size()const { return 20; }
 
     static sha1 hash( const char* d, uint32_t dlen );
-    static sha1 hash( const string& );
+    static sha1 hash( const std::string& );
 
     template<typename T>
     static sha1 hash( const T& t ) 

libraries/libfc/include/fc/crypto/sha224.hpp

@@ -11,17 +11,17 @@ class sha224
 {
   public:
     sha224();
-    explicit sha224( const string& hex_str );
+    explicit sha224( const std::string& hex_str );
 
-    string str()const;
-    operator string()const;
+    std::string str()const;
+    operator std::string()const;
 
     char*       data();
     const char* data()const;
     size_t data_size()const { return 224 / 8; }
 
     static sha224 hash( const char* d, uint32_t dlen );
-    static sha224 hash( const string& );
+    static sha224 hash( const std::string& );
 
     template<typename T>
     static sha224 hash( const T& t ) 

libraries/libfc/include/fc/crypto/sha256.hpp

@@ -12,11 +12,11 @@ class sha256
 {
   public:
     sha256();
-    explicit sha256( const string& hex_str );
+    explicit sha256( const std::string& hex_str );
     explicit sha256( const char *data, size_t size );
 
-    string str()const;
-    operator string()const;
+    std::string str()const;
+    operator std::string()const;
 
     const char* data()const;
     char*       data();
@@ -27,7 +27,7 @@ class sha256
     }
 
     static sha256 hash( const char* d, uint32_t dlen );
-    static sha256 hash( const string& );
+    static sha256 hash( const std::string& );
     static sha256 hash( const sha256& );
 
     template<typename T>

libraries/libfc/include/fc/crypto/sha3.hpp

@@ -13,11 +13,11 @@ class sha3
 public:
 	sha3();
 	~sha3(){}
-	explicit sha3(const string &hex_str);
+	explicit sha3(const std::string &hex_str);
 	explicit sha3(const char *data, size_t size);
 
-	string str() const;
-	operator string() const;
+	std::string str() const;
+	operator std::string() const;
 
 	const char *data() const;
 	char* data();
@@ -29,7 +29,7 @@ public:
 		const auto& sha = e.result(is_nist);
 		return sha;
 	}
-	static sha3 hash(const string& s, bool is_nist=true) { return hash(s.c_str(), s.size(), is_nist); }
+	static sha3 hash(const std::string& s, bool is_nist=true) { return hash(s.c_str(), s.size(), is_nist); }
 	static sha3 hash(const sha3& s, bool is_nist=true) { return hash(s.data(), sizeof(s._hash), is_nist); }
 
 	template <typename T>

libraries/libfc/include/fc/crypto/sha512.hpp

@@ -9,17 +9,17 @@ class sha512
 {
   public:
     sha512();
-    explicit sha512( const string& hex_str );
+    explicit sha512( const std::string& hex_str );
 
-    string str()const;
-    operator string()const;
+    std::string str()const;
+    operator std::string()const;
 
     char*       data();
     const char* data()const;
     size_t data_size()const { return 512 / 8; }
 
     static sha512 hash( const char* d, uint32_t dlen );
-    static sha512 hash( const string& );
+    static sha512 hash( const std::string& );
 
     template<typename T>
     static sha512 hash( const T& t ) 

libraries/libfc/include/fc/crypto/signature.hpp

@@ -27,7 +27,7 @@ namespace fc { namespace crypto {
          signature& operator= (const signature& ) = default;
 
          // serialize to/from string
-         explicit signature(const string& base58str);
+         explicit signature(const std::string& base58str);
          std::string to_string(const fc::yield_function_t& yield = fc::yield_function_t()) const;
 
          size_t which() const;
@@ -38,7 +38,7 @@ namespace fc { namespace crypto {
          storage_type _storage;
 
          signature( storage_type&& other_storage )
-         :_storage(std::forward<storage_type>(other_storage))
+         :_storage(std::move(other_storage))
          {}
 
          friend bool operator == ( const signature& p1, const signature& p2);

libraries/libfc/include/fc/filesystem.hpp

@@ -1,264 +1,61 @@
 #pragma once
-#include <utility>
 #include <memory>
+#include <utility>
 
-#include <fc/string.hpp>
-#include <fc/reflect/typename.hpp>
 #include <fc/fwd.hpp>
+#include <fc/reflect/typename.hpp>
+#include <fc/string.hpp>
 
-namespace boost {
-  namespace filesystem {
-    class path;
-    class directory_iterator;
-    class recursive_directory_iterator;
-  }
-}
-
+#include <filesystem>
 
 namespace fc {
-  /**
-   *  @brief wraps boost::filesystem::path to provide platform independent path manipulation.
-   *
-   *  Most calls are simply a passthrough to boost::filesystem::path, however exceptions are
-   *  wrapped in an fc::error_report and the additional helper method fc::path::windows_string(),
-   *  can be used to calculate paths intended for systems different than the host.
-   *
-   *  @note Serializes to a fc::value() as the result of generic_string()
-   */
-  class path {
-    public:
-      path();
-      ~path();
-      path( const boost::filesystem::path& );
-      path( const std::string& p );
-      /// Constructor to build path using unicode native characters.
-      path(const std::wstring& p);
-      path( const char* );
-      path( const path& p );
-      path( path&& p );
-      path& operator =( const path& );
-      path& operator =( path&& );
 
-      path& operator /=( const fc::path& );
-      friend path operator /( const fc::path& p, const fc::path& );
-      friend bool operator ==( const fc::path& p, const fc::path& );
-      friend bool operator !=( const fc::path& p, const fc::path& );
-      friend bool operator < ( const fc::path& p, const fc::path& );
+/** @return the home directory on Linux and OS X and the Profile directory on Windows */
+const std::filesystem::path& home_path();
 
-      operator boost::filesystem::path& ();
-      operator const boost::filesystem::path& ()const;
+/** @return the home_path() on Linux, home_path()/Library/Application Support/ on OS X,
+ *  and APPDATA on windows
+ */
+const std::filesystem::path& app_path();
 
-      void         replace_extension( const fc::path& e );
-      fc::path     stem()const;
-      fc::path     extension()const;
-      fc::path     filename()const;
-      fc::path     parent_path()const;
-      std::string   string()const;
-      std::string   generic_string()const;
-      /** On windows, returns a path where all path separators are '\' suitable for displaying
-       * to users.  On other platforms, it does the same as generic_string()
-       */
-      std::string   preferred_string() const;
+class variant;
+void to_variant(const std::filesystem::path&, fc::variant&);
+void from_variant(const fc::variant&, std::filesystem::path&);
 
-      std::wstring wstring() const;
-      std::wstring generic_wstring() const;
-      std::wstring preferred_wstring() const;
+template <>
+struct get_typename<std::filesystem::path> {
+   static const char* name() { return "path"; }
+};
 
-      /** Retrieves native string path representation and next converts it into
-          ANSI UTF-8 representation.
-          It is needed since not all parts of fc library accept unicode paths
-          (fc::file_mapping).
-      */
-      std::string  to_native_ansi_path() const;
+class temp_directory {
+   std::filesystem::path tmp_path;
 
-      /**
-       * @brief replaces '/' with '\' in the result of generic_string()
-       *
-       * @note not part of boost::filesystem::path
-       */
-      std::string windows_string()const;
+ public:
+   temp_directory(const std::filesystem::path& tempFolder = std::filesystem::temp_directory_path()) {
+      std::filesystem::path template_path{ tempFolder / "fc-XXXXXX" };
+      std::string tmp_buf = template_path.string();
+      // The following is valid because the return array of std::string::data() is null-terminated since C++11
+      if (mkdtemp(tmp_buf.data()) == nullptr)
+         throw std::system_error(errno, std::generic_category(), __PRETTY_FUNCTION__);
+      tmp_path = tmp_buf;
+   }
+   temp_directory(const temp_directory&) = delete;
+   temp_directory(temp_directory&& other) { tmp_path.swap(other.tmp_path); }
 
-      bool       is_relative()const;
-      bool       is_absolute()const;
-      bool       empty() const;
-
-      static char    separator_char;
-
-    private:
-    #ifdef _WIN64
-      fwd<boost::filesystem::path,40> _p; 
-    #else
-      fwd<boost::filesystem::path,32> _p; 
-    #endif
-  };
-
-  namespace detail
-  {
-    class path_wrapper
-    {
-    public:
-      path_wrapper(path p) :
-        _path(p)
-      {
+   ~temp_directory() {
+      if (!tmp_path.empty()) {
+         std::error_code ec;
+         std::filesystem::remove_all(tmp_path, ec);
       }
-      const path* operator->() const
-      {
-        return &_path;
-      }
-    private:
-      path _path;
-    };
-  }
+   }
 
-  class directory_iterator {
-    public:
-      directory_iterator( const fc::path& p );
-      directory_iterator();
-      ~directory_iterator();
-
-      fc::path            operator*()const;
-      detail::path_wrapper operator->() const;
-      directory_iterator& operator++(int);
-      directory_iterator& operator++();
-
-      friend bool operator==( const directory_iterator&, const directory_iterator& );
-      friend bool operator!=( const directory_iterator&, const directory_iterator& );
-    private:
-      fwd<boost::filesystem::directory_iterator,16> _p; 
-  };
-  class recursive_directory_iterator {
-    public:
-      recursive_directory_iterator( const fc::path& p );
-      recursive_directory_iterator();
-      ~recursive_directory_iterator();
-
-      fc::path            operator*()const;
-      recursive_directory_iterator& operator++(int);
-      recursive_directory_iterator& operator++();
-      void pop();
-      int  level();
+   temp_directory& operator=(const temp_directory&) = delete;
+   temp_directory& operator=(temp_directory&& other) {
+      tmp_path.swap(other.tmp_path);
+      return *this;
+   }
+   const std::filesystem::path& path() const { return tmp_path; }
+};
 
 
-      friend bool operator==( const recursive_directory_iterator&, const recursive_directory_iterator& );
-      friend bool operator!=( const recursive_directory_iterator&, const recursive_directory_iterator& );
-    private:
-      fwd<boost::filesystem::recursive_directory_iterator,16> _p; 
-  };
-
-  bool     exists( const path& p );
-  bool     is_directory( const path& p );
-  bool     is_regular_file( const path& p );
-  void     create_directories( const path& p );
-  void     remove_all( const path& p );
-  path     absolute( const path& p );
-  path     make_relative(const path& from, const path& to);
-  path     canonical( const path& p );
-  uint64_t file_size( const path& p );
-  uint64_t directory_size( const path& p );
-  bool     remove( const path& p );
-  void     copy( const path& from, const path& to );
-  void     rename( const path& from, const path& to );
-  void     resize_file( const path& file, size_t s );
-  
-  // setuid, setgid not implemented.
-  // translates octal permission like 0755 to S_ stuff defined in sys/stat.h
-  // no-op on Windows.
-  void     chmod( const path& p, int perm );
-
-  void     create_hard_link( const path& from, const path& to );
-
-  path        unique_path();
-  path        temp_directory_path();
-
-  /** @return the home directory on Linux and OS X and the Profile directory on Windows */
-  const path& home_path();
-
-  /** @return the home_path() on Linux, home_path()/Library/Application Support/ on OS X, 
-   *  and APPDATA on windows
-   */
-  const path& app_path();
-
-  /** @return application executable path */
-  const fc::path& current_path();
-
-  class variant;
-  void to_variant( const fc::path&,  fc::variant&  );
-  void from_variant( const fc::variant& , fc::path& );
-
-  template<> struct get_typename<path> { static const char* name()   { return "path";   } };
-
-  /**
-   * Class which creates a temporary directory inside an existing temporary directory.
-   */
-  class temp_file_base
-  {
-  public:
-     inline ~temp_file_base() { remove(); }
-     inline operator bool() const { return _path.has_value(); }
-     inline bool operator!() const { return !_path.has_value(); }
-     const fc::path& path() const;
-     void remove();
-     void release();
-  protected:
-     typedef std::optional<fc::path> path_t;
-     inline temp_file_base(const path_t& path) : _path(path) {}
-     inline temp_file_base(path_t&& path) : _path(std::move(path)) {}
-     path_t _path;
-  };
-
-  /**
-   * Class which creates a temporary directory inside an existing temporary directory.
-   */
-  class temp_file : public temp_file_base
-  {
-  public:
-     temp_file(temp_file&& other);
-     temp_file& operator=(temp_file&& other);
-     temp_file(const fc::path& tempFolder = fc::temp_directory_path(), bool create = false);
-  };
-
-  /**
-   * Class which creates a temporary directory inside an existing temporary directory.
-   */
-  class temp_directory : public temp_file_base
-  {
-  public:
-     temp_directory(temp_directory&& other);
-     temp_directory& operator=(temp_directory&& other);
-     temp_directory(const fc::path& tempFolder = fc::temp_directory_path());
-  };
-
-
-#if !defined(__APPLE__)
-  // this code is known to work on linux and windows.  It may work correctly on mac, 
-  // or it may need slight tweaks or extra includes.  It's disabled now to avoid giving
-  // a false sense of security.
-# define FC_HAS_SIMPLE_FILE_LOCK
-#endif
-#ifdef FC_HAS_SIMPLE_FILE_LOCK  
-  /** simple class which only allows one process to open any given file. 
-   * approximate usage:
-   * int main() {
-   *   fc::simple_file_lock instance_lock("~/.my_app/.lock");
-   *   if (!instance_lock.try_lock()) {
-   *     elog("my_app is already running");
-   *     return 1;
-   *   }
-   *   // do stuff here, file will be unlocked when instance_lock goes out of scope
-   * }
-  */
-  class simple_lock_file
-  {
-  public:
-    simple_lock_file(const path& lock_file_path);
-    ~simple_lock_file();
-    bool try_lock();
-    void unlock();
-  private:
-    class impl;
-    std::unique_ptr<impl> my;
-  };
-#endif // FC_HAS_SIMPLE_FILE_LOCK
-
-}
-
+} // namespace fc

libraries/libfc/include/fc/interprocess/iprocess.hpp

@@ -1,67 +0,0 @@
-#pragma once
-#include <fc/thread/future.hpp>
-#include <fc/io/buffered_iostream.hpp>
-#include <vector>
-#include <fc/string.hpp>
-#include <fc/filesystem.hpp>
-
-namespace fc
-{
-    /**
-     *   @brief abstract interface for interacting with external processes
-     *
-     *   At the very least we have ssh::process and direct child processes, and
-     *   there may be other processes that need to implement this protocol.
-     */
-    class iprocess
-    {
-       public:
-         enum exec_opts {
-           open_none   = 0,
-           open_stdin  = 0x01, 
-           open_stdout = 0x02, 
-           open_stderr = 0x04,
-           open_all    = open_stdin|open_stdout|open_stderr,
-           suppress_console = 0x08
-         };
-
-          virtual ~iprocess(){}
-
-          /**
-           *  
-           *  @return *this
-           */
-          virtual iprocess& exec( const path& exe, std::vector<std::string> args, 
-                                  const path& work_dir = path(), int opts = open_all ) = 0;
-
-          /**
-           *  @return blocks until the process exits
-           */
-          virtual int result(const microseconds& timeout = microseconds::maximum()) = 0;
-
-
-          /**
-           *  Forcefully kills the process.
-           */
-          virtual void kill() = 0;
-          
-          /**
-           *  @brief returns a stream that writes to the process' stdin
-           */
-          virtual buffered_ostream_ptr in_stream() = 0;
-          
-          /**
-           *  @brief returns a stream that reads from the process' stdout
-           */
-          virtual buffered_istream_ptr out_stream() = 0;
-          /**
-           *  @brief returns a stream that reads from the process' stderr
-           */
-          virtual buffered_istream_ptr err_stream() = 0;
-
-    };
-
-    typedef std::shared_ptr<iprocess> iprocess_ptr;
-    
-
-} // namespace fc

libraries/libfc/include/fc/interprocess/process.hpp

@@ -1,37 +0,0 @@
-#pragma once
-#include <fc/interprocess/iprocess.hpp>
-
-namespace fc {
-
-  fc::path find_executable_in_path( const fc::string name );
-
-  /**
-   *  @brief start and manage an local process
-   *  @note this class implements reference semantics.
-   */
-  class process  : public iprocess
-  {
-    public:
-      process();
-      ~process();
-
-      virtual iprocess& exec( const fc::path&  exe, 
-                              std::vector<std::string>   args, 
-                              const fc::path&  work_dir = fc::path(), 
-                              int              opts     = open_all    );
-
-      
-      virtual int                        result(const microseconds& timeout = microseconds::maximum());
-      virtual void                       kill();
-      virtual fc::buffered_ostream_ptr   in_stream();
-      virtual fc::buffered_istream_ptr   out_stream();
-      virtual fc::buffered_istream_ptr   err_stream();
-
-      class impl;
-    private:
-      std::unique_ptr<impl> my;
-  };
-
-  typedef std::shared_ptr<process> process_ptr;
-
-} // namespace fc

libraries/libfc/include/fc/io/buffered_iostream.hpp

@@ -1,73 +0,0 @@
-#pragma once
-#include <fc/io/iostream.hpp>
-
-namespace fc
-{
-   namespace detail 
-   { 
-       class buffered_istream_impl; 
-       class buffered_ostream_impl; 
-   }
-
-   /**
-    *  @brief Reads data from an unbuffered stream
-    *         and enables peek functionality.
-    */
-   class buffered_istream : public virtual istream
-   {
-      public:
-        buffered_istream( istream_ptr is );
-        buffered_istream( buffered_istream&& o );
-
-        buffered_istream& operator=( buffered_istream&& i );
-
-        virtual ~buffered_istream();
-        
-        /** read at least 1 byte or throw, if no data is available
-         *  this method should block cooperatively until data is
-         *  available or fc::eof_exception is thrown.
-         *
-         *  @pre len > 0 
-         *  @pre buf != nullptr
-         *  @throws fc::eof if at least 1 byte cannot be read
-         **/
-        virtual std::size_t     readsome( char* buf, std::size_t len );
-        virtual size_t          readsome( const std::shared_ptr<char>& buf, size_t len, size_t offset );
-
-        /**
-         *  This method may block until at least 1 character is
-         *  available.
-         */
-        virtual char            peek() const;
-
-      private:
-        std::unique_ptr<detail::buffered_istream_impl> my;
-   };
-   typedef std::shared_ptr<buffered_istream> buffered_istream_ptr;
-
-
-   /**
-    * 
-    */
-   class buffered_ostream : public virtual ostream
-   {
-      public:
-        buffered_ostream( ostream_ptr o, size_t bufsize = 4096 );
-        buffered_ostream( buffered_ostream&& m );
-        ~buffered_ostream();
-
-        buffered_ostream& operator=( buffered_ostream&& m );
-        /**
-         *  This method will return immediately unless the buffer
-         *  is full, in which case it will flush which may block.
-         */
-        virtual size_t  writesome( const char* buf, size_t len );
-        virtual size_t  writesome( const std::shared_ptr<const char>& buf, size_t len, size_t offset );
-
-        virtual void close();
-        virtual void flush();
-      private:
-        std::unique_ptr<detail::buffered_ostream_impl> my;
-   };
-   typedef std::shared_ptr<buffered_ostream> buffered_ostream_ptr;
-}

libraries/libfc/include/fc/io/cfile.hpp

@@ -29,16 +29,30 @@ class cfile_datastream;
  * std::ios_base::failure exception thrown for errors.
  */
 class cfile {
+   friend class temp_cfile;
 public:
    cfile()
      : _file(nullptr, &fclose)
    {}
 
-   void set_file_path( fc::path file_path ) {
+   cfile(cfile&& other)
+     : _open(other._open)
+     , _file_path(std::move(other._file_path))
+     , _file_blk_size(other._file_blk_size)
+     , _file(std::move(other._file))
+   {}
+
+   cfile& operator=(cfile&& other){
+      swap(*this, other);
+      return *this;
+   }
+
+
+   void set_file_path( std::filesystem::path file_path ) {
       _file_path = std::move( file_path );
    }
 
-   fc::path get_file_path() const {
+   std::filesystem::path get_file_path() const {
       return _file_path;
    }
 
@@ -221,11 +235,43 @@ public:
 
 private:
    bool                  _open = false;
-   fc::path              _file_path;
+   std::filesystem::path _file_path;
    size_t                _file_blk_size = 4096;
    detail::unique_file   _file;
 };
 
+class temp_cfile {
+   cfile _impl;
+public:
+   explicit temp_cfile(const char* mode = "wb"){
+      std::filesystem::path template_path{ std::filesystem::temp_directory_path() / "fc-XXXXXX" };
+      std::string tmp = template_path.string();
+      int fd = mkstemp(tmp.data());
+      if (fd == -1)
+         throw std::system_error(errno, std::generic_category(), __PRETTY_FUNCTION__);
+      _impl.set_file_path(tmp);
+      _impl._file.reset(fdopen(fd, mode));
+      if( !_impl._file ) {
+         using namespace std::literals::string_literals;
+         throw std::ios_base::failure( "cfile unable to open: "s +  tmp + " in mode: " + std::string( mode ) );
+      }
+      _impl._open = true;
+   }
+   ~temp_cfile() {
+      _impl.close();
+      std::error_code ec;
+      std::filesystem::remove(_impl.get_file_path(), ec);
+   }
+
+   temp_cfile(temp_cfile&& other): _impl(std::move(other._impl)) { }
+   temp_cfile& operator = (temp_cfile&& other){
+      _impl = std::move(other._impl);
+      return *this;
+   }
+
+   cfile& file() { return _impl; }
+};
+
 /*
  *  @brief datastream adapter that adapts cfile for use with fc unpack
  *

libraries/libfc/include/fc/io/fstream.hpp

@@ -7,5 +7,5 @@ namespace fc {
    * NB reading a full file into memory is a poor choice
    * if the file may be very large.
    */
-  void read_file_contents( const fc::path& filename, std::string& result );
+  void read_file_contents( const std::filesystem::path& filename, std::string& result );
 }