testing

Print non-default options on startup/shutdown

.cicd/defaults.json

@@ -0,0 +1,9 @@
+{
+    "cdt":{
+       "target":"4",
+       "prerelease":false
+    },
+    "eossystemcontracts":{
+       "ref":"release/3.1"
+    }
+ }

.cicd/platforms.json

@@ -1,7 +1,4 @@
 {
-   "ubuntu18": {
-      "dockerfile": ".cicd/platforms/ubuntu18.Dockerfile"
-   },
    "ubuntu20": {
       "dockerfile": ".cicd/platforms/ubuntu20.Dockerfile"
    },

.cicd/platforms/ubuntu18.Dockerfile

@@ -1,35 +0,0 @@
-FROM ubuntu:bionic
-
-RUN apt-get update && apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential             \
-                                                      cmake                       \
-                                                      curl                        \
-                                                      g++-8                       \
-                                                      git                         \
-                                                      jq                          \
-                                                      libgmp-dev                  \
-                                                      libssl-dev                  \
-                                                      llvm-7-dev                  \
-                                                      ninja-build                 \
-                                                      python3                     \
-                                                      software-properties-common  \
-                                                      zlib1g-dev                  \
-                                                      zstd
-
-# GitHub's actions/checkout requires git 2.18+ but Ubuntu 18 only provides 2.17
-RUN add-apt-repository ppa:git-core/ppa && apt update && apt install -y git
-
-# Leap requires boost 1.67+ but Ubuntu 18 only provides 1.65
-# Probably need 1.70+ to work properly with old cmake provided in Ubuntu 18
-RUN curl -L https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 | tar jx && \
-   cd boost_* &&                                                                                              \
-   ./bootstrap.sh --prefix=/boost &&                                                                          \
-   ./b2 --with-iostreams --with-date_time --with-filesystem --with-system                                     \
-        --with-program_options --with-chrono --with-test -j$(nproc) install &&                                \
-   cd .. &&                                                                                                   \
-   rm -rf boost_*
-
-ENV CC=gcc-8
-ENV CXX=g++-8
-ENV BOOST_ROOT=/boost
-ENV LLVM_DIR=/usr/lib/llvm-7/

.cicd/platforms/ubuntu20.Dockerfile

@@ -1,14 +1,17 @@
+# spoon was here
 FROM ubuntu:focal
-
+ENV TZ="America/New_York"
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential      \
-                                                      cmake                \
-                                                      curl                 \
-                                                      git                  \
-                                                      jq                   \
-                                                      libboost-all-dev     \
-                                                      libgmp-dev           \
-                                                      libssl-dev           \
-                                                      llvm-11-dev          \
-                                                      ninja-build          \
-                                                      zstd
+    apt-get install -y build-essential      \
+                       cmake                \
+                       git                  \
+                       jq                   \
+                       libboost-all-dev     \
+                       libcurl4-openssl-dev \
+                       libgmp-dev           \
+                       libssl-dev           \
+                       llvm-11-dev          \
+                       ninja-build          \
+                       python3-numpy        \
+                       zstd

.cicd/platforms/ubuntu22.Dockerfile

@@ -1,14 +1,17 @@
+# spoon was here
 FROM ubuntu:jammy
-
+ENV TZ="America/New_York"
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get upgrade -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential      \
-                                                      cmake                \
-                                                      curl                 \
-                                                      git                  \
-                                                      jq                   \
-                                                      libboost-all-dev     \
-                                                      libgmp-dev           \
-                                                      libssl-dev           \
-                                                      llvm-11-dev          \
-                                                      ninja-build          \
-                                                      zstd
+    apt-get install -y build-essential      \
+                       cmake                \
+                       git                  \
+                       jq                   \
+                       libboost-all-dev     \
+                       libcurl4-openssl-dev \
+                       libgmp-dev           \
+                       libssl-dev           \
+                       llvm-11-dev          \
+                       ninja-build          \
+                       python3-numpy        \
+                       zstd

.github/actions/parallel-ctest-containers/action.yml

@@ -9,6 +9,8 @@ inputs:
     required: true
   tests-label:
     required: true
+  test-timeout:
+    required: true
 runs:
   using: 'node16'
   main: 'dist/index.mjs'

.github/actions/parallel-ctest-containers/main.mjs

@@ -10,25 +10,27 @@ const container = core.getInput('container', {required: true});
 const error_log_paths = JSON.parse(core.getInput('error-log-paths', {required: true}));
 const log_tarball_prefix = core.getInput('log-tarball-prefix', {required: true});
 const tests_label = core.getInput('tests-label', {required: true});
+const test_timeout = core.getInput('test-timeout', {required: true});
+
+const repo_name = process.env.GITHUB_REPOSITORY.split('/')[1];
 
 try {
-   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", "/__w/leap/leap", container, "sh", "-c", "zstdcat /build.tar.zst | tar x"], {stdio:"inherit"}).status)
+   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", `/__w/${repo_name}/${repo_name}`, container, "sh", "-c", "zstdcat /build.tar.zst | tar x"], {stdio:"inherit"}).status)
       throw new Error("Failed to create base container");
    if(child_process.spawnSync("docker", ["commit", "base", "baseimage"], {stdio:"inherit"}).status)
       throw new Error("Failed to create base image");
    if(child_process.spawnSync("docker", ["rm", "base"], {stdio:"inherit"}).status)
       throw new Error("Failed to remove base container");
 
-   // the correct approach is by far "--show-only=json-v1" and then pluck out .tests[].name; but that doesn't work on U18's cmake 3.10 since it lacks json-v1 output
-   const test_query_result = child_process.spawnSync("docker", ["run", "--rm", "baseimage", "bash", "-e", "-o", "pipefail", "-c", `cd build; ctest -L '${tests_label}' --show-only | head -n -1 | cut -d ':' -f 2 -s | jq -cnR '[inputs | select(length>0)[1:]]'`]);
+   const test_query_result = child_process.spawnSync("docker", ["run", "--rm", "baseimage", "bash", "-e", "-o", "pipefail", "-c", `cd build; ctest -L '${tests_label}' --show-only=json-v1`]);
    if(test_query_result.status)
       throw new Error("Failed to discover tests with label")
-   const tests = JSON.parse(test_query_result.stdout);
+   const tests = JSON.parse(test_query_result.stdout).tests;
 
    let subprocesses = [];
    tests.forEach(t => {
       subprocesses.push(new Promise(resolve => {
-         child_process.spawn("docker", ["run", "--name", t, "--init", "baseimage", "bash", "-c", `cd build; ctest --output-on-failure -R '^${t}$'`], {stdio:"inherit"}).on('close', code => resolve(code));
+         child_process.spawn("docker", ["run", "--security-opt", "seccomp=unconfined", "-e", "GITHUB_ACTIONS=True", "--name", t.name, "--init", "baseimage", "bash", "-c", `cd build; ctest --output-on-failure -R '^${t.name}$' --timeout ${test_timeout}`], {stdio:"inherit"}).on('close', code => resolve(code));
       }));
    });
 
@@ -45,13 +47,13 @@ try {
       let packer = tar.pack();
 
       extractor.on('entry', (header, stream, next) => {
-         if(!header.name.startsWith(`__w/leap/leap/build`)) {
+         if(!header.name.startsWith(`__w/${repo_name}/${repo_name}/build`)) {
             stream.on('end', () => next());
             stream.resume();
             return;
          }
 
-         header.name = header.name.substring(`__w/leap/leap/`.length);
+         header.name = header.name.substring(`__w/${repo_name}/${repo_name}/`.length);
          if(header.name !== "build/" && error_log_paths.filter(p => header.name.startsWith(p)).length === 0) {
             stream.on('end', () => next());
             stream.resume();
@@ -61,8 +63,8 @@ try {
          stream.pipe(packer.entry(header, next));
       }).on('finish', () => {packer.finalize()});
 
-      child_process.spawn("docker", ["export", tests[i]]).stdout.pipe(extractor);
-      stream.promises.pipeline(packer, zlib.createGzip(), fs.createWriteStream(`${log_tarball_prefix}-${tests[i]}-logs.tar.gz`));
+      child_process.spawn("docker", ["export", tests[i].name]).stdout.pipe(extractor);
+      stream.promises.pipeline(packer, zlib.createGzip(), fs.createWriteStream(`${log_tarball_prefix}-${tests[i].name}-logs.tar.gz`));
    }
 } catch(e) {
    core.setFailed(`Uncaught exception ${e.message}`);

.github/workflows/build.yaml

@@ -7,6 +7,20 @@ on:
       - "release/*"
   pull_request:
   workflow_dispatch:
+    inputs:
+      override-cdt:
+        description: 'Override cdt target'
+        type: string
+      override-cdt-prerelease:
+        type: choice
+        description: Override cdt prelease
+        options:
+        - default
+        - true
+        - false
+      override-eos-system-contracts:
+        description: 'Override eos-system-contracts ref'
+        type: string
 
 permissions:
   packages: read
@@ -17,81 +31,51 @@ defaults:
     shell: bash
 
 jobs:
-  d:
-    name: Discover Platforms
-    runs-on: ubuntu-latest
-    outputs:
-      missing-platforms: ${{steps.discover.outputs.missing-platforms}}
-      p: ${{steps.discover.outputs.platforms}}
-    steps:
-      - name: Discover Platforms
-        id: discover
-        uses: AntelopeIO/discover-platforms-action@v1
-        with:
-          platform-file: .cicd/platforms.json
-          password: ${{secrets.GITHUB_TOKEN}}
-          package-name: builders
-  build-platforms:
-    name: Build Platforms
-    needs: d
-    if: needs.d.outputs.missing-platforms != '[]'
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ${{fromJSON(needs.d.outputs.missing-platforms)}}
-    runs-on: ["self-hosted", "enf-x86-beefy"]
+  build-base:
+    name: Run Build Workflow
+    uses: ./.github/workflows/build_base.yaml
     permissions:
       packages: write
       contents: read
-    steps:
-      - name: Login to Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{github.repository_owner}}
-          password: ${{secrets.GITHUB_TOKEN}}
-      - name: Build and push
-        uses: docker/build-push-action@v3
-        with:
-          push: true
-          tags: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
-          file: ${{fromJSON(needs.d.outputs.p)[matrix.platform].dockerfile}}
 
-  Build:
-    needs: [d, build-platforms]
-    if: always() && needs.d.result == 'success' && (needs.build-platforms.result == 'success' || needs.build-platforms.result == 'skipped')
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: [ubuntu18, ubuntu20, ubuntu22]
-    runs-on: ["self-hosted", "enf-x86-beefy"]
-    container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+  v:
+    name: Discover Versions
+    runs-on: ubuntu-latest
+    outputs:
+      cdt-target: ${{steps.versions.outputs.cdt-target}}
+      cdt-prerelease: ${{steps.versions.outputs.cdt-prerelease}}
+      eos-system-contracts-ref: ${{steps.versions.outputs.eos-system-contracts-ref}}
     steps:
-        - uses: actions/checkout@v3
-          with:
-            submodules: recursive
-        - name: Build
-          id: build
-          run: |
-            # https://github.com/actions/runner/issues/2033
-            chown -R $(id -u):$(id -g) $PWD
-            mkdir build
-            cd build
-            cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -GNinja ..
-            ninja
-            tar -pc -C .. --exclude "*.o" build | zstd --long -T0 -9 > ../build.tar.zst
-        - name: Upload builddir
-          uses: AntelopeIO/upload-artifact-large-chunks-action@v1
-          with:
-            name: ${{matrix.platform}}-build
-            path: build.tar.zst
+      - name: Setup cdt and eos-system-contracts versions
+        id: versions
+        env:
+          GH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          DEFAULTS_JSON=$(curl -sSfL $(gh api https://api.github.com/repos/${{github.repository}}/contents/.cicd/defaults.json?ref=${{github.sha}} --jq .download_url))
+          echo cdt-target=$(echo "$DEFAULTS_JSON" | jq -r '.cdt.target') >> $GITHUB_OUTPUT
+          echo cdt-prerelease=$(echo "$DEFAULTS_JSON" | jq -r '.cdt.prerelease') >> $GITHUB_OUTPUT
+          echo eos-system-contracts-ref=$(echo "$DEFAULTS_JSON" | jq -r '.eossystemcontracts.ref') >> $GITHUB_OUTPUT
+
+          if [[ "${{inputs.override-cdt}}" != "" ]]; then
+            echo cdt-target=${{inputs.override-cdt}} >> $GITHUB_OUTPUT
+          fi
+          if [[ "${{inputs.override-cdt-prerelease}}" == +(true|false) ]]; then
+            echo cdt-prerelease=${{inputs.override-cdt-prerelease}} >> $GITHUB_OUTPUT
+          fi
+          if [[ "${{inputs.override-eos-system-contracts}}" != "" ]]; then
+            echo eos-system-contracts-ref=${{inputs.override-eos-system-contracts}} >> $GITHUB_OUTPUT
+          fi
 
   dev-package:
     name: Build leap-dev package
-    needs: [d, Build]
-    if: always() && needs.Build.result == 'success'
+    needs: [build-base]
+    if: always() && needs.build-base.result == 'success'
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu20, ubuntu22]
     runs-on: ubuntu-latest
-    container: ${{fromJSON(needs.d.outputs.p)['ubuntu20'].image}}
+    container: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -99,28 +83,37 @@ jobs:
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
-          name: ubuntu20-build
+          name: ${{matrix.platform}}-build
       - name: Build dev package
         run: |
           zstdcat build.tar.zst | tar x
           cd build
           cpack
+      - name: Install dev package
+        run: |
+          apt update && apt upgrade -y
+          apt install -y ./build/leap_*.deb ./build/leap-dev*.deb
+      - name: Test using TestHarness
+        run: |
+          python3 -c "from TestHarness import Cluster"
       - name: Upload dev package
         uses: actions/upload-artifact@v3
         with:
-          name: leap-dev-ubuntu20-amd64
+          name: leap-dev-${{matrix.platform}}-amd64
           path: build/leap-dev*.deb
 
   tests:
     name: Tests
-    needs: [d, Build]
-    if: always() && needs.Build.result == 'success'
+    needs: [build-base]
+    if: always() && needs.build-base.result == 'success'
     strategy:
       fail-fast: false
       matrix:
         platform: [ubuntu20, ubuntu22]
     runs-on: ["self-hosted", "enf-x86-hightier"]
-    container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+    container:
+      image: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}}
+      options: --security-opt seccomp=unconfined
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
@@ -131,21 +124,18 @@ jobs:
         run: |
           # https://github.com/actions/runner/issues/2033  -- need this because of full version label test looking at git revs
           chown -R $(id -u):$(id -g) $PWD
-          # jammy's boost 1.74 can stumble on an EXDEV via copy_file_range() it doesn't have a fallback for; re-eval once moving to std::filesystem
-          export TMPDIR="$PWD/tmp"
-          mkdir -p $TMPDIR
           zstdcat build.tar.zst | tar x
           cd build
-          ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)"
+          ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)" --timeout 420
 
   np-tests:
     name: NP Tests
-    needs: [d, Build]
-    if: always() && needs.Build.result == 'success'
+    needs: [build-base]
+    if: always() && needs.build-base.result == 'success'
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu18, ubuntu20, ubuntu22]
+        platform: [ubuntu20, ubuntu22]
     runs-on: ["self-hosted", "enf-x86-midtier"]
     steps:
       - uses: actions/checkout@v3
@@ -156,11 +146,12 @@ jobs:
       - name: Run tests in parallel containers
         uses: ./.github/actions/parallel-ctest-containers
         with:
-          container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
-          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd"]'
+          container: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}}
+          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
           log-tarball-prefix: ${{matrix.platform}}
           tests-label: nonparallelizable_tests
-      - name: Upload logs from failed tests  
+          test-timeout: 420
+      - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
@@ -169,12 +160,12 @@ jobs:
 
   lr-tests:
     name: LR Tests
-    needs: [d, Build]
-    if: always() && needs.Build.result == 'success'
+    needs: [build-base]
+    if: always() && needs.build-base.result == 'success'
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu18, ubuntu20, ubuntu22]
+        platform: [ubuntu20, ubuntu22]
     runs-on: ["self-hosted", "enf-x86-lowtier"]
     steps:
       - uses: actions/checkout@v3
@@ -185,10 +176,11 @@ jobs:
       - name: Run tests in parallel containers
         uses: ./.github/actions/parallel-ctest-containers
         with:
-          container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
-          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd"]'
+          container: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}}
+          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd", "build/TestLogs"]'
           log-tarball-prefix: ${{matrix.platform}}
           tests-label: long_running_tests
+          test-timeout: 1800
       - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
@@ -196,11 +188,99 @@ jobs:
           name: ${{matrix.platform}}-lr-logs
           path: '*-logs.tar.gz'
 
+  libtester-tests:
+    name: libtester tests
+    needs: [build-base, v, dev-package]
+    if: always() && needs.v.result == 'success' && needs.dev-package.result == 'success'
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu20, ubuntu22]
+        test: [build-tree, make-dev-install, deb-install]
+    runs-on:  ["self-hosted", "enf-x86-midtier"]
+    container: ${{ matrix.test != 'deb-install' && fromJSON(needs.build-base.outputs.p)[matrix.platform].image || matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }}
+    steps:
+      # LEAP
+      - if: ${{ matrix.test != 'deb-install' }}
+        name: Clone leap
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - if: ${{ matrix.test != 'deb-install' }}
+        name: Download leap builddir
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{matrix.platform}}-build
+      - if: ${{ matrix.test != 'deb-install' }}
+        name: Extract leap build
+        run: |
+          zstdcat build.tar.zst | tar x
+      - if: ${{ matrix.test == 'build-tree' }}
+        name: Set leap_DIR env var
+        run: |
+          echo "leap_DIR=$PWD/build/lib/cmake/leap" >> "$GITHUB_ENV"
+      - if: ${{ matrix.test == 'make-dev-install' }}
+        name: leap dev-install
+        run: |
+          cmake --install build
+          cmake --install build --component dev
+      - if: ${{ matrix.test == 'make-dev-install' }}
+        name: Delete leap artifacts
+        run: |
+          rm -r *
+      - if: ${{ matrix.test == 'deb-install' }}
+        name: Download leap-dev
+        uses: actions/download-artifact@v3
+        with:
+          name: leap-dev-${{matrix.platform}}-amd64
+      - if: ${{ matrix.test == 'deb-install' }}
+        name: Install leap-dev Package
+        run: |
+          apt-get update
+          export DEBIAN_FRONTEND='noninteractive'
+          export TZ='Etc/UTC'
+          apt-get install -y ./*.deb
+          rm ./*.deb
+
+      # CDT
+      - name: Download cdt
+        uses: AntelopeIO/asset-artifact-download-action@v2
+        with:
+          owner: AntelopeIO
+          repo: cdt
+          file: 'cdt_.*amd64.deb'
+          target: '${{needs.v.outputs.cdt-target}}'
+          prereleases: ${{fromJSON(needs.v.outputs.cdt-prerelease)}}
+          artifact-name: cdt_ubuntu_package_amd64
+          token: ${{github.token}}
+      - name: Install cdt Packages
+        run: |
+          apt install -y ./*.deb
+          rm ./*.deb
+
+      # Reference Contracts
+      - name: checkout eos-system-contracts
+        uses: actions/checkout@v3
+        with:
+          repository: eosnetworkfoundation/eos-system-contracts
+          path: eos-system-contracts
+          ref: '${{needs.v.outputs.eos-system-contracts-ref}}'
+      - if: ${{ matrix.test == 'deb-install' }}
+        name: Install eos-system-contracts deps
+        run: |
+          apt-get -y install cmake build-essential
+      - name: Build & Test eos-system-contracts
+        run: |
+          cmake -S eos-system-contracts -B eos-system-contracts/build -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=On -DSYSTEM_ENABLE_LEAP_VERSION_CHECK=Off -DSYSTEM_ENABLE_CDT_VERSION_CHECK=Off
+          cmake --build eos-system-contracts/build -- -j $(nproc)
+          cd eos-system-contracts/build/tests
+          ctest --output-on-failure -j $(nproc)
+
   all-passing:
     name: All Required Tests Passed
-    needs: [dev-package, tests, np-tests]
+    needs: [dev-package, tests, np-tests, libtester-tests]
     if: always()
     runs-on: ubuntu-latest
     steps:
-      - if: needs.dev-package.result != 'success' || needs.tests.result != 'success' || needs.np-tests.result != 'success'
+      - if: needs.dev-package.result != 'success' || needs.tests.result != 'success' || needs.np-tests.result != 'success' || needs.libtester-tests.result != 'success'
         run: false

.github/workflows/build_base.yaml

@@ -0,0 +1,87 @@
+name: "Build leap"
+
+on:
+  workflow_dispatch:
+  workflow_call:
+    outputs:
+      p:
+        description: "Discovered Build Platforms"
+        value: ${{ jobs.d.outputs.p }}
+
+permissions:
+  packages: read
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  d:
+    name: Discover Platforms
+    runs-on: ubuntu-latest
+    outputs:
+      missing-platforms: ${{steps.discover.outputs.missing-platforms}}
+      p: ${{steps.discover.outputs.platforms}}
+    steps:
+      - name: Discover Platforms
+        id: discover
+        uses: AntelopeIO/discover-platforms-action@v1
+        with:
+          platform-file: .cicd/platforms.json
+          password: ${{secrets.GITHUB_TOKEN}}
+          package-name: builders
+
+  build-platforms:
+    name: Build Platforms
+    needs: d
+    if: needs.d.outputs.missing-platforms != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{fromJSON(needs.d.outputs.missing-platforms)}}
+    runs-on: ["self-hosted", "enf-x86-beefy"]
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Login to Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{github.repository_owner}}
+          password: ${{secrets.GITHUB_TOKEN}}
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          tags: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+          file: ${{fromJSON(needs.d.outputs.p)[matrix.platform].dockerfile}}
+
+  Build:
+    name: Build leap
+    needs: [d, build-platforms]
+    if: always() && needs.d.result == 'success' && (needs.build-platforms.result == 'success' || needs.build-platforms.result == 'skipped')
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu20, ubuntu22]
+    runs-on: ["self-hosted", "enf-x86-beefy"]
+    container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+    steps:
+        - uses: actions/checkout@v3
+          with:
+            submodules: recursive
+        - name: Build
+          id: build
+          run: |
+            # https://github.com/actions/runner/issues/2033
+            chown -R $(id -u):$(id -g) $PWD
+            cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -GNinja
+            cmake --build build
+            tar -pc --exclude "*.o" build | zstd --long -T0 -9 > build.tar.zst
+        - name: Upload builddir
+          uses: AntelopeIO/upload-artifact-large-chunks-action@v1
+          with:
+            name: ${{matrix.platform}}-build
+            path: build.tar.zst

.github/workflows/performance_harness_run.yaml

@@ -0,0 +1,21 @@
+name: "Performance Harness Run"
+
+on:
+  workflow_dispatch:
+
+permissions:
+  packages: read
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  tmp:
+    name: Stub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Workflow Stub
+        run: |
+          echo "Workflow Stub"

.github/workflows/ph_backward_compatibility.yaml

@@ -0,0 +1,70 @@
+name: "Performance Harness Backwards Compatibility"
+
+on:
+  workflow_dispatch:
+
+permissions:
+  packages: read
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build-base:
+    name: Run Build Workflow
+    uses: ./.github/workflows/build_base.yaml
+    permissions:
+      packages: write
+      contents: read
+
+  tests:
+    name: Tests
+    needs: [build-base]
+    if: always() && needs.build-base.result == 'success'
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu20, ubuntu22]
+        release: [3.1, 3.2, 4.0]
+    runs-on: ["self-hosted", "enf-x86-lowtier"]
+    container:
+      image: ${{fromJSON(needs.build-base.outputs.p)[matrix.platform].image}}
+      options: --security-opt seccomp=unconfined
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download builddir
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{matrix.platform}}-build
+      - name: Extract Build Directory
+        run: |
+          zstdcat build.tar.zst | tar x
+      - name: Download Prev Leap Version
+        uses: AntelopeIO/asset-artifact-download-action@v2
+        with:
+          owner: AntelopeIO
+          repo: leap
+          file: '(leap).*${{matrix.platform}}.04.*(x86_64|amd64).deb'
+          target: '${{matrix.release}}'
+          token: ${{github.token}}
+      - name: Install leap & replace binaries for PH use
+        run: |
+          apt-get update
+          apt-get install -y ./leap*.deb
+          rm build/bin/nodeos
+          rm build/bin/cleos
+          cp /usr/bin/nodeos build/bin
+          cp /usr/bin/cleos build/bin
+          ./build/bin/nodeos --version
+      - if: ${{ matrix.release == '3.1' || matrix.release == '3.2' }}
+        name: Run Performance Tests (<v4.0)
+        run: |
+          cd build
+          ctest --output-on-failure -R performance_test -E read_only --timeout 480
+      - if: ${{ matrix.release != '3.1' && matrix.release != '3.2' }}
+        name: Run Performance Tests (>=v4.0)
+        run: |
+          cd build
+          ctest --output-on-failure -R performance_test --timeout 480

.github/workflows/pinned_build.yaml

@@ -0,0 +1,51 @@
+name: "Pinned Build"
+
+on:
+  workflow_dispatch:
+
+permissions:
+  packages: read
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  Build:
+    name: Build
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu20, ubuntu22]
+    runs-on: ["self-hosted", "enf-x86-beefy-long"]
+    container: ${{ matrix.platform == 'ubuntu20' && 'ubuntu:focal' || 'ubuntu:jammy' }}
+    steps:
+      - name: Update and Install git
+        run: |
+          apt-get update
+          apt-get install -y git
+          git --version
+      - name: Clone leap
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Install dependencies
+        run: |
+          # https://github.com/actions/runner/issues/2033
+          chown -R $(id -u):$(id -g) $PWD
+          ./scripts/install_deps.sh
+      - name: Build Pinned Build
+        env:
+          LEAP_PINNED_INSTALL_PREFIX: /usr
+        run: |
+          ./scripts/pinned_build.sh deps build "$(nproc)"
+      - name: Upload package
+        uses: actions/upload-artifact@v3
+        with:
+          name: leap-${{matrix.platform}}-pinned-amd64
+          path: build/leap_*.deb
+      - name: Run Parallel Tests
+        run: |
+          cd build
+          ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)" --timeout 420

.gitignore

@@ -75,19 +75,21 @@ witness_node_data_dir
 *.pyc
 *.pyo
 
+*.gdb_history
+
 Testing/*
 build.tar.gz
-build/*
-build-debug/*
+[Bb]uild*/*
 deps/*
 dependencies/*
 
 etc/eosio/node_*
-var/lib/node_*
+TestLogs*
 .vscode
 .idea/
 *.iws
 .DS_Store
+CMakeLists.txt.user
 
 !*.swagger.*
 

.gitmodules

@@ -22,6 +22,12 @@
 [submodule "libraries/libfc/secp256k1/secp256k1"]
 	path = libraries/libfc/secp256k1/secp256k1
 	url = https://github.com/bitcoin-core/secp256k1
-[submodule "libraries/libfc/libraries/ff"]
-	path = libraries/libfc/libraries/ff
-	url = https://github.com/AntelopeIO/libff
+[submodule "libraries/prometheus/prometheus-cpp"]
+	path = libraries/prometheus/prometheus-cpp
+	url = https://github.com/jupp0r/prometheus-cpp.git
+[submodule "libraries/libfc/libraries/bn256"]
+	path = libraries/libfc/libraries/bn256
+	url = https://github.com/AntelopeIO/bn256
+[submodule "libraries/cli11/cli11"]
+	path = libraries/cli11/cli11
+	url = https://github.com/AntelopeIO/CLI11.git

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required( VERSION 3.8 )
+cmake_minimum_required( VERSION 3.16 )
 
 project( leap )
 include(CTest) # suppresses DartConfiguration.tcl error
@@ -14,7 +14,7 @@ set( CMAKE_CXX_EXTENSIONS ON )
 set( CXX_STANDARD_REQUIRED ON)
 
 set(VERSION_MAJOR 4)
-set(VERSION_MINOR 0)
+set(VERSION_MINOR 1)
 set(VERSION_PATCH 0)
 set(VERSION_SUFFIX dev)
 
@@ -24,6 +24,12 @@ else()
     set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}")
 endif()
 
+
+# Options
+option(ENABLE_WERROR "Enable `-Werror` compilation flag." Off)
+option(ENABLE_WEXTRA "Enable `-Wextra` compilation flag." Off)
+
+
 set( CLI_CLIENT_EXECUTABLE_NAME cleos )
 set( NODE_EXECUTABLE_NAME nodeos )
 set( KEY_STORE_EXECUTABLE_NAME keosd )
@@ -31,12 +37,8 @@ set( LEAP_UTIL_EXECUTABLE_NAME leap-util )
 
 # http://stackoverflow.com/a/18369825
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0)
-        message(FATAL_ERROR "GCC version must be at least 8.0!")
-    endif()
-elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)
-        message(FATAL_ERROR "Clang version must be at least 5.0!")
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
+        message(FATAL_ERROR "GCC version must be at least 9.0!")
     endif()
 endif()
 
@@ -103,7 +105,7 @@ set(Boost_USE_MULTITHREADED      ON)
 set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 # Most boost deps get implictly picked up via fc, as just about everything links to fc. In addition we pick up
 # the pthread dependency through fc.
-find_package(Boost 1.67 REQUIRED COMPONENTS program_options unit_test_framework system)
+find_package(Boost 1.71 REQUIRED COMPONENTS program_options unit_test_framework system)
 
 if( APPLE AND UNIX )
 # Apple Specific Options Here
@@ -126,6 +128,16 @@ else()
     endif()
 endif()
 
+if(ENABLE_WERROR)
+  set( CMAKE_C_FLAGS   "${CMAKE_C_FLAGS}   -Werror" )
+  set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror" )
+endif()
+if(ENABLE_WEXTRA)
+  set( CMAKE_C_FLAGS   "${CMAKE_C_FLAGS}   -Wextra" )
+  set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra" )
+endif()
+
+
 option(EOSIO_ENABLE_DEVELOPER_OPTIONS "enable developer options for Leap" OFF)
 
 # based on http://www.delorie.com/gnu/docs/gdb/gdb_70.html
@@ -162,9 +174,10 @@ endif()
 
 message( STATUS "Using '${EOSIO_ROOT_KEY}' as public key for 'eosio' account" )
 
-find_package( Gperftools QUIET )
-if( GPERFTOOLS_FOUND )
-    message( STATUS "Found gperftools; compiling Leap with TCMalloc")
+option(ENABLE_TCMALLOC "use tcmalloc (requires gperftools)" OFF)
+if( ENABLE_TCMALLOC )
+    find_package( Gperftools REQUIRED )
+    message( STATUS "Compiling Leap with TCMalloc")
     #if doing this by the book, simply link_libraries( ${GPERFTOOLS_TCMALLOC} ) here. That will
     #give the performance benefits of tcmalloc but since it won't be linked last
     #the heap profiler & checker may not be accurate. This here is rather undocumented behavior
@@ -184,9 +197,6 @@ add_subdirectory( benchmark )
 
 option(DISABLE_WASM_SPEC_TESTS "disable building of wasm spec unit tests" OFF)
 
-install(FILES testnet.template DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio/launcher COMPONENT dev EXCLUDE_FROM_ALL)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testnet.template ${CMAKE_CURRENT_BINARY_DIR}/etc/eosio/launcher/testnet.template COPYONLY)
-
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/eosio.version.in ${CMAKE_CURRENT_BINARY_DIR}/eosio.version.hpp)
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/eosio.version.hpp DESTINATION ${CMAKE_INSTALL_FULL_INCLUDEDIR} COMPONENT dev EXCLUDE_FROM_ALL)
 
@@ -218,13 +228,48 @@ configure_file(libraries/softfloat/COPYING.txt                             licen
 configure_file(libraries/wasm-jit/LICENSE                                  licenses/leap/LICENSE.wavm      COPYONLY)
 configure_file(libraries/libfc/secp256k1/secp256k1/COPYING                 licenses/leap/LICENSE.secp256k1 COPYONLY)
 configure_file(libraries/libfc/include/fc/crypto/webauthn_json/license.txt licenses/leap/LICENSE.rapidjson COPYONLY)
-configure_file(libraries/libfc/src/network/LICENSE.go                      licenses/leap/LICENSE.go        COPYONLY)
 configure_file(libraries/eos-vm/LICENSE                                    licenses/leap/LICENSE.eos-vm    COPYONLY)
-configure_file(libraries/libfc/libraries/ff/LICENSE                        licenses/leap/LICENSE.libff     COPYONLY)
+configure_file(libraries/prometheus/prometheus-cpp/LICENSE                 licenses/leap/LICENSE.prom      COPYONLY)
 configure_file(programs/cleos/LICENSE.CLI11                                licenses/leap/LICENSE.CLI11     COPYONLY)
 
 install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/licenses/leap" DESTINATION "${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/" COMPONENT base)
 
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/libraries/testing/contracts DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/libraries/testing COMPONENT dev EXCLUDE_FROM_ALL
+        PATTERN "CMakeFiles" EXCLUDE
+        PATTERN "*.cmake" EXCLUDE
+        PATTERN "Makefile" EXCLUDE)
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/unittests/contracts DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/unittests COMPONENT dev EXCLUDE_FROM_ALL
+        PATTERN "CMakeFiles" EXCLUDE
+        PATTERN "*.cmake" EXCLUDE
+        PATTERN "Makefile" EXCLUDE)
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests/TestHarness DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing/tests COMPONENT dev EXCLUDE_FROM_ALL
+        FILES_MATCHING
+        PATTERN "*.py"
+        PATTERN "*.json"
+        PATTERN "__pycache__" EXCLUDE
+        PATTERN "CMakeFiles" EXCLUDE)
+
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.21)
+   # Cmake versions < 3.21 did not support installing symbolic links to a directory via install(FILES ...)
+   # Cmake 3.21 fixed this (https://gitlab.kitware.com/cmake/cmake/-/commit/d71a7cc19d6e03f89581efdaa8d63db216184d40)
+   # If/when the lowest cmake version supported becomes >= 3.21, the else block as well as the postinit and prerm scripts
+   # would be a good option to remove in favor of using the facilities in cmake / cpack directly.
+
+   add_custom_target(link_target ALL
+      COMMAND ${CMAKE_COMMAND} -E make_directory lib/python3/dist-packages
+      COMMAND ${CMAKE_COMMAND} -E create_symlink ../../../share/leap_testing/tests/TestHarness lib/python3/dist-packages/TestHarness
+      COMMAND ${CMAKE_COMMAND} -E make_directory share/leap_testing
+      COMMAND ${CMAKE_COMMAND} -E create_symlink ../../bin share/leap_testing/bin)
+
+   install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib/python3/dist-packages/TestHarness DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python3/dist-packages COMPONENT dev EXCLUDE_FROM_ALL)
+   install(FILES ${CMAKE_CURRENT_BINARY_DIR}/share/leap_testing/bin DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/leap_testing COMPONENT dev EXCLUDE_FROM_ALL)
+else()
+   # The following install(SCRIPT ...) steps are necessary for `make dev-install` to work on cmake versions < 3.21.
+   install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/scripts/install_testharness_symlinks.cmake COMPONENT dev EXCLUDE_FROM_ALL)
+
+   # The `make package` installation of symlinks happens via the `postinst` script installed in cmake.package via the line below
+endif()
+
 configure_file(${CMAKE_SOURCE_DIR}/libraries/cli11/bash-completion/completions/leap-util
                ${CMAKE_BINARY_DIR}/programs/leap-util/bash-completion/completions/leap-util COPYONLY)
 configure_file(${CMAKE_SOURCE_DIR}/libraries/cli11/bash-completion/completions/cleos

CMakeModules/EosioTester.cmake.in

@@ -14,12 +14,6 @@ if (LLVM_DIR STREQUAL "" OR NOT LLVM_DIR)
    set(LLVM_DIR @LLVM_DIR@)
 endif()
 
-find_package( Gperftools QUIET )
-if( GPERFTOOLS_FOUND )
-    message( STATUS "Found gperftools; compiling tests with TCMalloc")
-    list( APPEND PLATFORM_SPECIFIC_LIBS tcmalloc )
-endif()
-
 if(NOT "@LLVM_FOUND@" STREQUAL "")
    find_package(LLVM @LLVM_VERSION@ EXACT REQUIRED CONFIG)
    llvm_map_components_to_libnames(LLVM_LIBS support core passes mcjit native DebugInfoDWARF orcjit)
@@ -41,8 +35,6 @@ else ( APPLE )
    set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} -Wall")
 endif ( APPLE )
 
-### Remove after Boost 1.70 CMake fixes are in place
-set( Boost_NO_BOOST_CMAKE ON CACHE STRING "ON or OFF" )
 set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 find_package(Boost @Boost_MAJOR_VERSION@.@Boost_MINOR_VERSION@ EXACT REQUIRED COMPONENTS
    date_time
@@ -56,18 +48,15 @@ find_library(libtester eosio_testing @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH
 find_library(libchain eosio_chain @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libfc fc @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libsecp256k1 secp256k1 @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
+find_library(libbn256 bn256 @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 
-find_library(libff ff @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libwasm WASM @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libwast WAST @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libir IR     @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(liblogging Logging @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
-find_library(libruntime Runtime @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libsoftfloat softfloat @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 get_filename_component(cryptodir @OPENSSL_CRYPTO_LIBRARY@ DIRECTORY)
 find_library(liboscrypto crypto "${cryptodir}" NO_DEFAULT_PATH)
-get_filename_component(ssldir @OPENSSL_SSL_LIBRARY@ DIRECTORY)
-find_library(libosssl ssl "${ssldir}" NO_DEFAULT_PATH)
 find_library(libchainbase chainbase @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 find_library(libbuiltins builtins @CMAKE_INSTALL_FULL_LIBDIR@ NO_DEFAULT_PATH)
 
@@ -89,16 +78,14 @@ macro(add_eosio_test_executable test_name)
        ${libfc}
        ${libwast}
        ${libwasm}
-       ${libruntime}
        ${libir}
        ${libsoftfloat}
        ${liboscrypto}
-       ${libosssl}
        ${liblogging}
        ${libchainbase}
        ${libbuiltins}
        ${libsecp256k1}
-       ${libff}
+       ${libbn256}
        @GMP_LIBRARY@
 
        ${Boost_FILESYSTEM_LIBRARY}
@@ -115,7 +102,7 @@ macro(add_eosio_test_executable test_name)
        ${WRAP_MAIN}
        Threads::Threads
       )
-   
+
    #adds -ltr. Ubuntu eosio.contracts build breaks without this
    if(UNIX AND NOT APPLE)
       target_link_libraries(${test_name} ${LIBRT})

CMakeModules/EosioTesterBuild.cmake.in

@@ -12,12 +12,6 @@ if (LLVM_DIR STREQUAL "" OR NOT LLVM_DIR)
    set(LLVM_DIR @LLVM_DIR@)
 endif()
 
-find_package( Gperftools QUIET )
-if( GPERFTOOLS_FOUND )
-    message( STATUS "Found gperftools; compiling tests with TCMalloc")
-    list( APPEND PLATFORM_SPECIFIC_LIBS tcmalloc )
-endif()
-
 if(NOT "@LLVM_FOUND@" STREQUAL "")
    find_package(LLVM @LLVM_VERSION@ EXACT REQUIRED CONFIG)
    llvm_map_components_to_libnames(LLVM_LIBS support core passes mcjit native DebugInfoDWARF orcjit)
@@ -38,8 +32,6 @@ else ( APPLE )
    set( CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS} -Wall")
 endif ( APPLE )
 
-### Remove after Boost 1.70 CMake fixes are in place
-set( Boost_NO_BOOST_CMAKE ON CACHE STRING "ON or OFF" )
 set( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 find_package(Boost @Boost_MAJOR_VERSION@.@Boost_MINOR_VERSION@ EXACT REQUIRED COMPONENTS
    date_time
@@ -53,18 +45,15 @@ find_library(libtester eosio_testing @CMAKE_BINARY_DIR@/libraries/testing NO_DEF
 find_library(libchain eosio_chain @CMAKE_BINARY_DIR@/libraries/chain NO_DEFAULT_PATH)
 find_library(libfc fc @CMAKE_BINARY_DIR@/libraries/libfc NO_DEFAULT_PATH)
 find_library(libsecp256k1 secp256k1 @CMAKE_BINARY_DIR@/libraries/libfc/secp256k1 NO_DEFAULT_PATH)
+find_library(libbn256 bn256 @CMAKE_BINARY_DIR@/libraries/libfc/libraries/bn256/src NO_DEFAULT_PATH)
 
-find_library(libff ff @CMAKE_BINARY_DIR@/libraries/libfc/libraries/ff/libff NO_DEFAULT_PATH)
 find_library(libwasm WASM @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WASM NO_DEFAULT_PATH)
 find_library(libwast WAST @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WAST NO_DEFAULT_PATH)
 find_library(libir IR     @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/IR NO_DEFAULT_PATH)
 find_library(liblogging Logging @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/Logging NO_DEFAULT_PATH)
-find_library(libruntime Runtime @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/Runtime NO_DEFAULT_PATH)
 find_library(libsoftfloat softfloat @CMAKE_BINARY_DIR@/libraries/softfloat NO_DEFAULT_PATH)
 get_filename_component(cryptodir @OPENSSL_CRYPTO_LIBRARY@ DIRECTORY)
 find_library(liboscrypto crypto "${cryptodir}" NO_DEFAULT_PATH)
-get_filename_component(ssldir @OPENSSL_SSL_LIBRARY@ DIRECTORY)
-find_library(libosssl ssl "${ssldir}" NO_DEFAULT_PATH)
 find_library(libchainbase chainbase @CMAKE_BINARY_DIR@/libraries/chainbase NO_DEFAULT_PATH)
 find_library(libbuiltins builtins @CMAKE_BINARY_DIR@/libraries/builtins NO_DEFAULT_PATH)
 
@@ -86,16 +75,14 @@ macro(add_eosio_test_executable test_name)
        ${libfc}
        ${libwast}
        ${libwasm}
-       ${libruntime}
        ${libir}
        ${libsoftfloat}
        ${liboscrypto}
-       ${libosssl}
        ${liblogging}
        ${libchainbase}
        ${libbuiltins}
        ${libsecp256k1}
-       ${libff}
+       ${libbn256}
        @GMP_LIBRARY@
 
        ${Boost_FILESYSTEM_LIBRARY}
@@ -112,7 +99,7 @@ macro(add_eosio_test_executable test_name)
        ${WRAP_MAIN}
        Threads::Threads
       )
-   
+
    #adds -ltr. Ubuntu eosio.contracts build breaks without this
    if(UNIX AND NOT APPLE)
       target_link_libraries(${test_name} ${LIBRT})

README.md

@@ -8,7 +8,6 @@ The `main` branch is the development branch; do not use it for production. Refer
 We currently support the following operating systems.
 - Ubuntu 22.04 Jammy
 - Ubuntu 20.04 Focal
-- Ubuntu 18.04 Bionic
 
 Other Unix derivatives such as macOS are tended to on a best-effort basis and may not be full featured. If you aren't using Ubuntu, please visit the "[Build Unsupported OS](./docs/00_install/01_build-from-source/00_build-unsupported-os.md)" page to explore your options.
 
@@ -45,15 +44,16 @@ You will need to build on a [supported operating system](#supported-operating-sy
 
 Requirements to build:
 - C++17 compiler and standard library
-- boost 1.67+
-- CMake 3.8+
+- boost 1.70+
+- CMake 3.16+
 - LLVM 7 - 11 - for Linux only
   - newer versions do not work
 - openssl 1.1+
-- curl
+- libcurl 7.40.0+
 - git
 - GMP
 - Python 3
+- python3-numpy
 - zlib
 
 ### Step 1 - Clone
@@ -109,12 +109,14 @@ Make sure you are in the root of the `leap` repo, then run the `install_depts.sh
 sudo scripts/install_deps.sh
 ```
 
-Next, run the pinned build script. You have to give it three arguments, in the following order:
-  - A temporary folder, for all dependencies that need to be built from source.
-  - A build folder, where the binaries you need to install will be built to.
-  - The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
+Next, run the pinned build script. You have to give it three arguments in the following order:
+1. A temporary folder, for all dependencies that need to be built from source.
+1. A build folder, where the binaries you need to install will be built to.
+1. The number of jobs or CPU cores/threads to use (note the [jobs flag](#step-3---build) warning above).
 
-The following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads (Note: you don't need `sudo` for this command):
+> 🔒 You do not need to run this script with `sudo` or as root.
+
+For example, the following command runs the `pinned_build.sh` script, specifies a `deps` and `build` folder in the root of the Leap repo for the first two arguments, then builds the packages using all of your computer's CPU threads:
 ```bash
 scripts/pinned_build.sh deps build "$(nproc)"
 ```
@@ -123,20 +125,19 @@ Now you can optionally [test](#step-4---test) your build, or [install](#step-5--
 #### Unpinned Build
 The following instructions are valid for this branch. Other release branches may have different requirements, so ensure you follow the directions in the branch or release you intend to build. If you are in an Ubuntu docker container, omit `sudo` because you run as `root` by default.
 
-<details> <summary>Ubuntu 22.04 Jammy & Ubuntu 20.04 Focal</summary>
-
 Install dependencies:
 ```bash
 sudo apt-get update
 sudo apt-get install -y \
         build-essential \
         cmake \
-        curl \
         git \
         libboost-all-dev \
+        libcurl4-openssl-dev \
         libgmp-dev \
         libssl-dev \
-        llvm-11-dev
+        llvm-11-dev \
+        python3-numpy
 ```
 To build, make sure you are in the root of the `leap` repo, then run the following command:
 ```bash
@@ -145,47 +146,6 @@ cd build
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/usr/lib/llvm-11 ..
 make -j "$(nproc)" package
 ```
-</details>
-
-<details> <summary>Ubuntu 18.04 Bionic</summary>
-
-Install dependencies:
-```bash
-sudo apt-get update
-sudo apt-get install -y \
-        build-essential \
-        cmake \
-        curl \
-        g++-8 \
-        git \
-        libgmp-dev \
-        libssl-dev \
-        llvm-7-dev \
-        python3 \
-        zlib1g-dev
-```
-You need to build Boost from source on this distribution:
-```bash
-curl -fL https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.bz2 -o ~/Downloads/boost_1_79_0.tar.bz2
-tar -jvxf ~/Downloads/boost_1_79_0.tar.bz2 -C ~/Downloads/
-pushd ~/Downloads/boost_1_79_0
-./bootstrap.sh --prefix="$HOME/boost1.79"
-./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -j "$(nproc)" install
-popd
-```
-The Boost `*.tar.bz2` download and `boost_1_79_0` folder can be removed now if you want more space.
-```bash
-rm -r ~/Downloads/boost_1_79_0.tar.bz2 ~/Downloads/boost_1_79_0
-```
-From a terminal in the root of the `leap` repo, build.
-```bash
-mkdir -p build
-cd build
-cmake -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 -DCMAKE_PREFIX_PATH="$HOME/boost1.79;/usr/lib/llvm-7/" -DCMAKE_BUILD_TYPE=Release ..
-make -j "$(nproc)" package
-```
-After building, you may remove the `~/boost1.79` directory or you may keep it around for your next build.
-</details>
 
 Now you can optionally [test](#step-4---test) your build, or [install](#step-5---install) the `*.deb` binary packages, which will be in the root of your build directory.
 

benchmark/CMakeLists.txt

@@ -1,7 +1,7 @@
 file(GLOB BENCHMARK "*.cpp")
 add_executable( benchmark ${BENCHMARK} )
 
-target_link_libraries( benchmark fc Boost::program_options )
+target_link_libraries( benchmark fc Boost::program_options bn256)
 target_include_directories( benchmark PUBLIC
                             "${CMAKE_CURRENT_SOURCE_DIR}"
                           )

benchmark/alt_bn_128.cpp

@@ -1,6 +1,6 @@
 #include <iostream>
 
-#include <fc/crypto/alt_bn128.hpp>
+#include <bn256/bn256.h>
 
 #include <benchmark.hpp>
 
@@ -10,22 +10,20 @@ using bytes = std::vector<char>;
 using g1g2_pair = std::vector<std::string>;
 
 void add_benchmarking() {
-   std::vector<unsigned char>  point1_raw = {
+   const unsigned char  point1[64] = {
       12,175,215,144,98,95,151,228,179,85,31,170,172,159,40,255,250,252,68,28,235,65,172,180,69,164,153,29,187,239,220,201,   // x
       18,102,219,76,79,148,120,28,39,149,42,172,41,248,120,249,255,69,42,51,160,239,13,219,239,183,77,174,217,158,130,10,};  // y
-   std::vector<unsigned char>  point2_raw = {
+   const unsigned char  point2[64] = {
       12,144,10,32,104,85,103,36,222,232,48,152,108,217,40,145,230,48,8,54,0,7,134,164,7,10,139,110,95,205,124,121,       // x
-      22,254,176,251,18,168,78,220,142,100,102,113,58,176,83,186,212,62,154,138,235,135,34,46,237,117,54,36,198,40,79,73,};   // y 
-   
-   bytes point1, point2;
-   point1.insert(point1.begin(), point1_raw.begin(), point1_raw.end());
-   point2.insert(point2.begin(), point2_raw.begin(), point2_raw.end());
+      22,254,176,251,18,168,78,220,142,100,102,113,58,176,83,186,212,62,154,138,235,135,34,46,237,117,54,36,198,40,79,73,};   // y
+
+   unsigned char  ans[64] ={};
+
 
    auto f = [&]() {
-      auto res = fc::alt_bn128_add(point1, point2);
-      if (std::holds_alternative<fc::alt_bn128_error>(res)) {
-         std::cout << "alt_bn128_add failed: " 
-            << (int)std::get<fc::alt_bn128_error>(res) << std::endl;
+      auto res = bn256::g1_add(point1, point2, ans);
+      if (res == -1) {
+         std::cout << "alt_bn128_add failed" << std::endl;
       }
    };
 
@@ -33,21 +31,18 @@ void add_benchmarking() {
 }
 
 void mul_benchmarking() {
-   std::vector<unsigned char>  point_raw = {
+   const unsigned char  point[64] = {
       12,175,215,144,98,95,151,228,179,85,31,170,172,159,40,255,250,252,68,28,235,65,172,180,69,164,153,29,187,239,220,201,   // x
       18,102,219,76,79,148,120,28,39,149,42,172,41,248,120,249,255,69,42,51,160,239,13,219,239,183,77,174,217,158,130,10,}; // y
-   std::vector<unsigned char>  scaler_raw = {
+   const unsigned char  scalar[32] = {
      25,62,182,170,104,140,135,90,37,150,0,77,2,77,146,71,54,101,113,69,177,216,157,4,229,213,33,215,169,99,150,91, }; // scaler size 256 bits
-   
-   bytes point, scaler;
-   point.insert(point.begin(), point_raw.begin(), point_raw.end());
-   scaler.insert(scaler.begin(), scaler_raw.begin(), scaler_raw.end());
+
+   unsigned char  ans[64] = {};
 
    auto f = [&]() {
-      auto res = fc::alt_bn128_mul(point, scaler);
-      if (std::holds_alternative<fc::alt_bn128_error>(res)) {
-         std::cout << "alt_bn128_mul failed: "
-            << (int)std::get<fc::alt_bn128_error>(res) << std::endl;
+      auto res = bn256::g1_scalar_mul(point, scalar, ans);
+      if (res == -1) {
+         std::cout << "alt_bn128_mul failed" << std::endl;
       }
    };
 
@@ -55,7 +50,7 @@ void mul_benchmarking() {
 }
 
 void pair_benchmarking() {
-   std::vector<unsigned char>  g1_g2_pairs_raw = {
+   const unsigned char  g1_g2_pairs[] = {
       /* pair 1 */
       12,175,215,144,98,95,151,228,179,85,31,170,172,159,40,255,250,252,68,28,235,65,172,180,69,164,153,29,187,239,220,201,18,102,219,76,79,148,120,28,39,149,42,172,41,248,120,249,255,69,42,51,160,239,13,219,239,183,77,174,217,158,130,10,
       46,24,234,176,148,82,198,136,44,30,157,18,217,25,49,241,63,197,2,151,14,150,136,210,114,4,74,124,145,225,131,139,13,60,0,50,236,103,39,15,150,226,246,189,209,113,0,46,151,39,41,6,87,119,228,213,45,225,29,234,161,110,43,87,3,237,74,227,93,23,171,129,49,118,228,173,154,111,168,220,161,46,140,103,155,56,168,207,254,90,228,76,188,232,25,34,44,102,159,35,193,216,177,31,131,247,226,19,170,222,26,227,86,37,81,149,202,144,188,126,120,168,9,195,32,169,147,161,
@@ -87,27 +82,21 @@ void pair_benchmarking() {
      30,204,212,77,212,159,135,173,195,194,165,112,62,248,180,204,66,73,253,99,65,111,39,171,19,211,171,203,35,66,146,20,33,241,46,6,167,133,80,76,238,165,59,232,120,58,211,157,50,212,86,191,95,6,134,164,36,227,79,58,119,98,108,171,
      48,49,12,141,166,151,158,56,136,255,197,138,114,195,39,59,71,236,82,57,149,249,170,55,187,95,193,171,14,124,45,87,7,157,47,178,153,237,194,157,142,194,100,14,40,51,61,201,244,93,61,196,154,59,14,135,209,72,102,186,14,228,228,152,40,246,109,82,93,249,92,105,191,121,77,108,20,87,3,87,167,173,171,255,189,34,155,239,218,95,181,153,222,20,120,195,27,28,47,82,113,3,218,129,54,210,185,165,206,99,126,61,217,19,237,5,12,90,148,246,128,231,63,53,37,223,204,195
    };
-   
+
    // benchmarking 1 pair of points
-   bytes g1_g2_1_pair;
-   g1_g2_1_pair.insert(g1_g2_1_pair.begin(), g1_g2_pairs_raw.begin(), g1_g2_pairs_raw.begin() + 384);
    auto f_1_pair = [&]() {
-      auto res = fc::alt_bn128_pair(g1_g2_1_pair, [](){});
-      if (std::holds_alternative<fc::alt_bn128_error>(res)) {
-         std::cout << "alt_bn128_pair 1 pair failed: "
-            << (int)std::get<fc::alt_bn128_error>(res) << std::endl;
+      auto res = bn256::pairing_check({ g1_g2_pairs, 384}, [](){});
+      if (res == -1) {
+         std::cout << "alt_bn128_pair 1 pair failed" << std::endl;
       }
    };
    benchmarking("alt_bn128_pair (1 pair)", f_1_pair);
 
    // benchmarking 10 pair of points
-   bytes g1_g2_10_pairs;
-   g1_g2_10_pairs.insert(g1_g2_10_pairs.begin(), g1_g2_pairs_raw.begin(), g1_g2_pairs_raw.end());
    auto f_10_pairs = [&]() {
-      auto res = fc::alt_bn128_pair(g1_g2_10_pairs, [](){});
-      if (std::holds_alternative<fc::alt_bn128_error>(res)) {
-         std::cout << "alt_bn128_pair 10 pairs failed: "
-            << (int)std::get<fc::alt_bn128_error>(res) << std::endl;
+      auto res = bn256::pairing_check(g1_g2_pairs, [](){});
+      if (res == -1) {
+         std::cout << "alt_bn128_pair 10 pair failed" << std::endl;
       }
    };
 

docs/00_install/index.md

@@ -12,7 +12,6 @@ Antelope currently supports the following operating systems:
 
 1. Ubuntu 22.04 Jammy
 2. Ubuntu 20.04 Focal
-3. Ubuntu 18.04 Bionic
 
 [[info | Note]]
 | It may be possible to build and install Antelope on other Unix-based operating systems. We gathered helpful information on the following page but please keep in mind that it is experimental and not officially supported. 

docs/01_nodeos/02_usage/03_development-environment/01_local-multi-node-testnet.md

@@ -143,7 +143,7 @@ We now have an account that is available to have a contract assigned to it, enab
 In the fourth terminal window, start a second `nodeos` instance. Notice that this command line is substantially longer than the one we used above to create the first producer. This is necessary to avoid collisions with the first `nodeos` instance. Fortunately, you can just cut and paste this command line and adjust the keys:
 
 ```sh
-nodeos --producer-name inita --plugin eosio::chain_api_plugin --plugin eosio::net_api_plugin --http-server-address 127.0.0.1:8889 --p2p-listen-endpoint 127.0.0.1:9877 --p2p-peer-address 127.0.0.1:9876 --config-dir node2 --data-dir node2 --private-key [\"EOS6hMjoWRF2L8x9YpeqtUEcsDKAyxSuM1APicxgRU1E3oyV5sDEg\",\"5JgbL2ZnoEAhTudReWH1RnMuQS6DBeLZt4ucV6t8aymVEuYg7sr\"]
+nodeos --producer-name inita --plugin eosio::chain_api_plugin --plugin eosio::net_api_plugin --http-server-address 127.0.0.1:8889 --p2p-listen-endpoint 127.0.0.1:9877 --p2p-peer-address 127.0.0.1:9876 --config-dir node2 --data-dir node2 --signature-provider EOS6hMjoWRF2L8x9YpeqtUEcsDKAyxSuM1APicxgRU1E3oyV5sDEg=KEY:5JgbL2ZnoEAhTudReWH1RnMuQS6DBeLZt4ucV6t8aymVEuYg7sr
 ```
 
 The output from this new node will show a little activity but will stop reporting until the last step in this tutorial, when the `inita` account is registered as a producer account and activated. Here is some example output from a newly started node. Your output might look a little different, depending on how much time you took entering each of these commands. Furthermore, this example is only the last few lines of output:

docs/01_nodeos/03_plugins/chain_plugin/index.md

@@ -21,33 +21,35 @@ These can only be specified from the `nodeos` command-line:
 ```console
 Command Line Options for eosio::chain_plugin:
   --genesis-json arg                    File to read Genesis State from
-  --genesis-timestamp arg               override the initial timestamp in the 
+  --genesis-timestamp arg               override the initial timestamp in the
                                         Genesis State file
-  --print-genesis-json                  extract genesis_state from blocks.log 
+  --print-genesis-json                  extract genesis_state from blocks.log
                                         as JSON, print to console, and exit
-  --extract-genesis-json arg            extract genesis_state from blocks.log 
+  --extract-genesis-json arg            extract genesis_state from blocks.log
                                         as JSON, write into specified file, and
                                         exit
-  --print-build-info                    print build environment information to 
+  --print-build-info                    print build environment information to
                                         console as JSON and exit
-  --extract-build-info arg              extract build environment information 
+  --extract-build-info arg              extract build environment information
                                         as JSON, write into specified file, and
                                         exit
-  --force-all-checks                    do not skip any checks that can be 
-                                        skipped while replaying irreversible 
-                                        blocks
+  --force-all-checks                    do not skip any validation checks while
+                                        replaying blocks (useful for replaying
+                                        blocks from untrusted source)
   --disable-replay-opts                 disable optimizations that specifically
                                         target replay
-  --replay-blockchain                   clear chain state database and replay 
+  --replay-blockchain                   clear chain state database and replay
                                         all blocks
-  --hard-replay-blockchain              clear chain state database, recover as 
-                                        many blocks as possible from the block 
+  --hard-replay-blockchain              clear chain state database, recover as
+                                        many blocks as possible from the block
                                         log, and then replay those blocks
-  --delete-all-blocks                   clear chain state database and block 
+  --delete-all-blocks                   clear chain state database and block
                                         log
-  --truncate-at-block arg (=0)          stop hard replay / block log recovery 
-                                        at this block number (if set to 
+  --truncate-at-block arg (=0)          stop hard replay / block log recovery
+                                        at this block number (if set to
                                         non-zero number)
+  --terminate-at-block arg (=0)         terminate after reaching this block
+                                        number (if set to a non-zero number)
   --snapshot arg                        File to read Snapshot State from
 
 ```
@@ -58,162 +60,182 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::chain_plugin:
-  --blocks-dir arg (="blocks")          the location of the blocks directory 
-                                        (absolute path or relative to 
+  --blocks-dir arg (="blocks")          the location of the blocks directory
+                                        (absolute path or relative to
+                                        application data dir)
+  --state-dir arg (="state")            the location of the state directory
+                                        (absolute path or relative to
                                         application data dir)
   --protocol-features-dir arg (="protocol_features")
-                                        the location of the protocol_features 
+                                        the location of the protocol_features
                                         directory (absolute path or relative to
                                         application config dir)
-  --checkpoint arg                      Pairs of [BLOCK_NUM,BLOCK_ID] that 
+  --checkpoint arg                      Pairs of [BLOCK_NUM,BLOCK_ID] that
                                         should be enforced as checkpoints.
-  --wasm-runtime runtime (=eos-vm-jit)  Override default WASM runtime ( 
+  --wasm-runtime runtime (=eos-vm-jit)  Override default WASM runtime (
                                         "eos-vm-jit", "eos-vm")
-                                        "eos-vm-jit" : A WebAssembly runtime 
-                                        that compiles WebAssembly code to 
+                                        "eos-vm-jit" : A WebAssembly runtime
+                                        that compiles WebAssembly code to
                                         native x86 code prior to execution.
                                         "eos-vm" : A WebAssembly interpreter.
-                                        
-  --profile-account arg                 The name of an account whose code will 
+
+  --profile-account arg                 The name of an account whose code will
                                         be profiled
   --abi-serializer-max-time-ms arg (=15)
-                                        Override default maximum ABI 
+                                        Override default maximum ABI
                                         serialization time allowed in ms
-  --chain-state-db-size-mb arg (=1024)  Maximum size (in MiB) of the chain 
+  --chain-state-db-size-mb arg (=1024)  Maximum size (in MiB) of the chain
                                         state database
   --chain-state-db-guard-size-mb arg (=128)
-                                        Safely shut down node when free space 
-                                        remaining in the chain state database 
+                                        Safely shut down node when free space
+                                        remaining in the chain state database
                                         drops below this size (in MiB).
   --signature-cpu-billable-pct arg (=50)
                                         Percentage of actual signature recovery
-                                        cpu to bill. Whole number percentages, 
+                                        cpu to bill. Whole number percentages,
                                         e.g. 50 for 50%
-  --chain-threads arg (=2)              Number of worker threads in controller 
+  --chain-threads arg (=2)              Number of worker threads in controller
                                         thread pool
   --contracts-console                   print contract's output to console
-  --deep-mind                           print deeper information about chain 
+  --deep-mind                           print deeper information about chain
                                         operations
-  --actor-whitelist arg                 Account added to actor whitelist (may 
+  --actor-whitelist arg                 Account added to actor whitelist (may
                                         specify multiple times)
-  --actor-blacklist arg                 Account added to actor blacklist (may 
+  --actor-blacklist arg                 Account added to actor blacklist (may
                                         specify multiple times)
-  --contract-whitelist arg              Contract account added to contract 
+  --contract-whitelist arg              Contract account added to contract
                                         whitelist (may specify multiple times)
-  --contract-blacklist arg              Contract account added to contract 
+  --contract-blacklist arg              Contract account added to contract
                                         blacklist (may specify multiple times)
   --action-blacklist arg                Action (in the form code::action) added
-                                        to action blacklist (may specify 
+                                        to action blacklist (may specify
                                         multiple times)
-  --key-blacklist arg                   Public key added to blacklist of keys 
-                                        that should not be included in 
-                                        authorities (may specify multiple 
+  --key-blacklist arg                   Public key added to blacklist of keys
+                                        that should not be included in
+                                        authorities (may specify multiple
                                         times)
-  --sender-bypass-whiteblacklist arg    Deferred transactions sent by accounts 
-                                        in this list do not have any of the 
-                                        subjective whitelist/blacklist checks 
-                                        applied to them (may specify multiple 
+  --sender-bypass-whiteblacklist arg    Deferred transactions sent by accounts
+                                        in this list do not have any of the
+                                        subjective whitelist/blacklist checks
+                                        applied to them (may specify multiple
                                         times)
-  --read-mode arg (=speculative)        Database read mode ("speculative", 
-                                        "head", "read-only", "irreversible").
-                                        In "speculative" mode: database 
-                                        contains state changes by transactions 
-                                        in the blockchain up to the head block 
-                                        as well as some transactions not yet 
-                                        included in the blockchain.
+  --read-mode arg (=head)               Database read mode ("head",
+                                        "irreversible", "speculative").
                                         In "head" mode: database contains state
-                                        changes by only transactions in the 
-                                        blockchain up to the head block; 
-                                        transactions received by the node are 
+                                        changes up to the head block;
+                                        transactions received by the node are
                                         relayed if valid.
-                                        In "read-only" mode: (DEPRECATED: see 
-                                        p2p-accept-transactions & 
-                                        api-accept-transactions) database 
-                                        contains state changes by only 
-                                        transactions in the blockchain up to 
-                                        the head block; transactions received 
-                                        via the P2P network are not relayed and
-                                        transactions cannot be pushed via the 
-                                        chain API.
-                                        In "irreversible" mode: database 
-                                        contains state changes by only 
-                                        transactions in the blockchain up to 
-                                        the last irreversible block; 
-                                        transactions received via the P2P 
-                                        network are not relayed and 
-                                        transactions cannot be pushed via the 
-                                        chain API.
-                                        
-  --api-accept-transactions arg (=1)    Allow API transactions to be evaluated 
+                                        In "irreversible" mode: database
+                                        contains state changes up to the last
+                                        irreversible block; transactions
+                                        received via the P2P network are not
+                                        relayed and transactions cannot be
+                                        pushed via the chain API.
+                                        In "speculative" mode: database
+                                        contains state changes by transactions
+                                        in the blockchain up to the head block
+                                        as well as some transactions not yet
+                                        included in the blockchain;
+                                        transactions received by the node are
+                                        relayed if valid.                                        
+  --api-accept-transactions arg (=1)    Allow API transactions to be evaluated
                                         and relayed if valid.
-  --validation-mode arg (=full)         Chain validation mode ("full" or 
+  --validation-mode arg (=full)         Chain validation mode ("full" or
                                         "light").
                                         In "full" mode all incoming blocks will
                                         be fully validated.
-                                        In "light" mode all incoming blocks 
-                                        headers will be fully validated; 
-                                        transactions in those validated blocks 
-                                        will be trusted 
-                                        
-  --disable-ram-billing-notify-checks   Disable the check which subjectively 
+                                        In "light" mode all incoming blocks
+                                        headers will be fully validated;
+                                        transactions in those validated blocks
+                                        will be trusted
+
+  --disable-ram-billing-notify-checks   Disable the check which subjectively
                                         fails a transaction if a contract bills
-                                        more RAM to another account within the 
+                                        more RAM to another account within the
                                         context of a notification handler (i.e.
-                                        when the receiver is not the code of 
+                                        when the receiver is not the code of
                                         the action).
   --maximum-variable-signature-length arg (=16384)
-                                        Subjectively limit the maximum length 
-                                        of variable components in a variable 
+                                        Subjectively limit the maximum length
+                                        of variable components in a variable
                                         legnth signature to this size in bytes
-  --trusted-producer arg                Indicate a producer whose blocks 
-                                        headers signed by it will be fully 
-                                        validated, but transactions in those 
+  --trusted-producer arg                Indicate a producer whose blocks
+                                        headers signed by it will be fully
+                                        validated, but transactions in those
                                         validated blocks will be trusted.
   --database-map-mode arg (=mapped)     Database map mode ("mapped", "heap", or
                                         "locked").
-                                        In "mapped" mode database is memory 
+                                        In "mapped" mode database is memory
                                         mapped as a file.
                                         In "heap" mode database is preloaded in
-                                        to swappable memory and will use huge 
+                                        to swappable memory and will use huge
                                         pages if available.
                                         In "locked" mode database is preloaded,
-                                        locked in to memory, and will use huge 
+                                        locked in to memory, and will use huge
                                         pages if available.
-                                        
-  --enable-account-queries arg (=0)     enable queries to find accounts by 
+
+  --eos-vm-oc-cache-size-mb arg (=1024) Maximum size (in MiB) of the EOS VM OC
+                                        code cache
+  --eos-vm-oc-compile-threads arg (=1)  Number of threads to use for EOS VM OC
+                                        tier-up
+  --eos-vm-oc-enable arg (=auto)        Enable EOS VM OC tier-up runtime
+                                        ('auto', 'all', 'none').
+                                        'auto' - EOS VM OC tier-up is enabled
+                                        for eosio.* accounts, read-only trxs,
+                                        and applying blocks.
+                                        'all'  - EOS VM OC tier-up is enabled
+                                        for all contract execution.
+                                        'none' - EOS VM OC tier-up is
+                                        completely disabled.
+
+  --enable-account-queries arg (=0)     enable queries to find accounts by
                                         various metadata.
   --max-nonprivileged-inline-action-size arg (=4096)
-                                        maximum allowed size (in bytes) of an 
-                                        inline action for a nonprivileged 
+                                        maximum allowed size (in bytes) of an
+                                        inline action for a nonprivileged
                                         account
   --transaction-retry-max-storage-size-gb arg
-                                        Maximum size (in GiB) allowed to be 
-                                        allocated for the Transaction Retry 
-                                        feature. Setting above 0 enables this 
+                                        Maximum size (in GiB) allowed to be
+                                        allocated for the Transaction Retry
+                                        feature. Setting above 0 enables this
                                         feature.
   --transaction-retry-interval-sec arg (=20)
                                         How often, in seconds, to resend an 
                                         incoming transaction to network if not 
                                         seen in a block.
-  --transaction-retry-max-expiration-sec arg (=90)
+                                        Needs to be at least twice as large as 
+                                        p2p-dedup-cache-expire-time-sec.
+  --transaction-retry-max-expiration-sec arg (=120)
                                         Maximum allowed transaction expiration 
                                         for retry transactions, will retry 
                                         transactions up to this value.
+                                        Should be larger than 
+                                        transaction-retry-interval-sec.
   --transaction-finality-status-max-storage-size-gb arg
-                                        Maximum size (in GiB) allowed to be 
-                                        allocated for the Transaction Finality 
+                                        Maximum size (in GiB) allowed to be
+                                        allocated for the Transaction Finality
                                         Status feature. Setting above 0 enables
                                         this feature.
   --transaction-finality-status-success-duration-sec arg (=180)
-                                        Duration (in seconds) a successful 
-                                        transaction's Finality Status will 
-                                        remain available from being first 
+                                        Duration (in seconds) a successful
+                                        transaction's Finality Status will
+                                        remain available from being first
                                         identified.
   --transaction-finality-status-failure-duration-sec arg (=180)
-                                        Duration (in seconds) a failed 
-                                        transaction's Finality Status will 
-                                        remain available from being first 
+                                        Duration (in seconds) a failed
+                                        transaction's Finality Status will
+                                        remain available from being first
                                         identified.
+  --integrity-hash-on-start             Log the state integrity hash on startup
+  --integrity-hash-on-stop              Log the state integrity hash on
+                                        shutdown
+  --block-log-retain-blocks arg         If set to greater than 0, periodically
+                                        prune the block log to store only
+                                        configured number of most recent
+                                        blocks.
+                                        If set to 0, no blocks are be written
+                                        to the block log; block log file is
+                                        removed after startup.
 
 ```
 

docs/01_nodeos/03_plugins/history_api_plugin/index.md

@@ -1,59 +0,0 @@
-[[warning | Deprecation Notice]]
-| The `history_plugin` that the `history_api_plugin` depends upon is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead.
-
-## Description
-
-The `history_api_plugin` exposes functionality from the [`history_plugin`](../history_plugin/index.md) to the RPC API interface managed by the [`http_plugin`](../http_plugin/index.md), providing read-only access to blockchain data.
-
-It provides four RPC API endpoints:
-
-* get_actions
-* get_transaction
-* get_key_accounts
-* get_controlled_accounts
-
-The four actions listed above are used by the following `cleos` commands (matching order):
-
-* get actions
-* get transaction
-* get accounts
-* get servants
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::history_api_plugin
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_api_plugin
-```
-
-## Options
-
-None
-
-## Dependencies
-
-* [`history_plugin`](../history_plugin/index.md)
-* [`chain_plugin`](../chain_plugin/index.md)
-* [`http_plugin`](../http_plugin/index.md)
-
-### Load Dependency Examples
-
-```console
-# config.ini
-plugin = eosio::history_plugin
-[options]
-plugin = eosio::chain_plugin
-[options]
-plugin = eosio::http_plugin
-[options]
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_plugin [options]  \
-           --plugin eosio::chain_plugin [operations] [options]  \
-           --plugin eosio::http_plugin [options]
-```

docs/01_nodeos/03_plugins/history_plugin/index.md

@@ -1,41 +0,0 @@
-[[warning | Deprecation Notice]]
-| The `history_plugin` is deprecated and will no longer be maintained. Please use the [`state_history_plugin`](../state_history_plugin/index.md) instead.
-
-## Description
-
-The `history_plugin` provides a cache layer to obtain historical data about the blockchain objects. It depends on [`chain_plugin`](../chain_plugin/index.md) for the data.
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::history_plugin
-[options]
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::history_plugin [options]
-```
-
-## Options
-
-These can be specified from both the `nodeos` command-line or the `config.ini` file:
-
-```console
-Config Options for eosio::history_plugin:
-  -f [ --filter-on ] arg                Track actions which match 
-                                        receiver:action:actor. Actor may be 
-                                        blank to include all. Action and Actor 
-                                        both blank allows all from Recieiver. 
-                                        Receiver may not be blank.
-  -F [ --filter-out ] arg               Do not track actions which match 
-                                        receiver:action:actor. Action and Actor
-                                        both blank excludes all from Reciever. 
-                                        Actor blank excludes all from 
-                                        reciever:action. Receiver may not be 
-                                        blank.
-```
-
-## Dependencies
-
-* [`chain_plugin`](../chain_plugin/index.md)

docs/01_nodeos/03_plugins/http_client_plugin/index.md

@@ -1,39 +0,0 @@
-## Description
-
-The `http_client_plugin`  is an internal utility plugin, providing the `producer_plugin` the ability to use securely an external `keosd` instance as its block signer. It can only be used when the `producer_plugin` is configured to produce blocks.
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::http_client_plugin
-https-client-root-cert = "path/to/my/certificate.pem"
-https-client-validate-peers = 1
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::http_client_plugin  \
-           --https-client-root-cert "path/to/my/certificate.pem"  \
-           --https-client-validate-peers 1
-```
-
-## Options
-
-These can be specified from both the `nodeos` command-line or the `config.ini` file:
-
-```console
-Config Options for eosio::http_client_plugin:
-  --https-client-root-cert arg          PEM encoded trusted root certificate 
-                                        (or path to file containing one) used 
-                                        to validate any TLS connections made.  
-                                        (may specify multiple times)
-                                        
-  --https-client-validate-peers arg (=1)
-                                        true: validate that the peer 
-                                        certificates are valid and trusted, 
-                                        false: ignore cert errors
-```
-
-## Dependencies
-
-* [`producer_plugin`](../producer_plugin/index.md)

docs/01_nodeos/03_plugins/http_plugin/index.md

@@ -22,49 +22,49 @@ These can be specified from both the command-line or the `config.ini` file:
 
 ```console
 Config Options for eosio::http_plugin:
-  --unix-socket-path arg                The filename (relative to data-dir) to 
-                                        create a unix socket for HTTP RPC; set 
-                                        blank to disable (=keosd.sock for keosd)
-  --http-server-address arg (=127.0.0.1:8888 for nodeos)
-                                        The local IP and port to listen for 
+  --unix-socket-path arg                The filename (relative to data-dir) to
+                                        create a unix socket for HTTP RPC; set
+                                        blank to disable.
+  --http-server-address arg (=127.0.0.1:8888)
+                                        The local IP and port to listen for
                                         incoming http connections; set blank to
                                         disable.
-  --https-server-address arg            The local IP and port to listen for 
-                                        incoming https connections; leave blank
-                                        to disable.
-  --https-certificate-chain-file arg    Filename with the certificate chain to 
-                                        present on https connections. PEM 
-                                        format. Required for https.
-  --https-private-key-file arg          Filename with https private key in PEM 
-                                        format. Required for https
-  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use: 
-                                        secp384r1 or prime256v1
   --access-control-allow-origin arg     Specify the Access-Control-Allow-Origin
-                                        to be returned on each request.
+                                        to be returned on each request
   --access-control-allow-headers arg    Specify the Access-Control-Allow-Header
-                                        s to be returned on each request.
-  --access-control-max-age arg          Specify the Access-Control-Max-Age to 
+                                        s to be returned on each request
+  --access-control-max-age arg          Specify the Access-Control-Max-Age to
                                         be returned on each request.
   --access-control-allow-credentials    Specify if Access-Control-Allow-Credent
-                                        ials: true should be returned on each 
+                                        ials: true should be returned on each
                                         request.
-  --max-body-size arg (=1048576)        The maximum body size in bytes allowed 
+  --max-body-size arg (=2097152)        The maximum body size in bytes allowed
                                         for incoming RPC requests
   --http-max-bytes-in-flight-mb arg (=500)
-                                        Maximum size in megabytes http_plugin 
-                                        should use for processing http 
-                                        requests. 503 error response when 
+                                        Maximum size in megabytes http_plugin
+                                        should use for processing http
+                                        requests. -1 for unlimited. 429 error
+                                        response when exceeded.
+  --http-max-in-flight-requests arg (=-1)
+                                        Maximum number of requests http_plugin
+                                        should use for processing http
+                                        requests. 429 error response when
                                         exceeded.
+  --http-max-response-time-ms arg (=30) Maximum time for processing a request,
+                                        -1 for unlimited
   --verbose-http-errors                 Append the error log to HTTP responses
-  --http-validate-host arg (=1)         If set to false, then any incoming 
+  --http-validate-host arg (=1)         If set to false, then any incoming
                                         "Host" header is considered valid
-  --http-alias arg                      Additionaly acceptable values for the 
-                                        "Host" header of incoming HTTP 
-                                        requests, can be specified multiple 
-                                        times.  Includes http/s_server_address 
+  --http-alias arg                      Additionaly acceptable values for the
+                                        "Host" header of incoming HTTP
+                                        requests, can be specified multiple
+                                        times.  Includes http/s_server_address
                                         by default.
   --http-threads arg (=2)               Number of worker threads in http thread
                                         pool
+  --http-keep-alive arg (=1)            If set to false, do not keep HTTP
+                                        connections alive, even if client
+                                        requests.
 ```
 
 ## Dependencies

docs/01_nodeos/03_plugins/index.md

@@ -11,7 +11,6 @@ For information on specific plugins, just select from the list below:
 * [`chain_api_plugin`](chain_api_plugin/index.md)
 * [`chain_plugin`](chain_plugin/index.md)
 * [`db_size_api_plugin`](db_size_api_plugin/index.md)
-* [`http_client_plugin`](http_client_plugin/index.md)
 * [`http_plugin`](http_plugin/index.md)
 * [`net_api_plugin`](net_api_plugin/index.md)
 * [`net_plugin`](net_plugin/index.md)
@@ -20,7 +19,6 @@ For information on specific plugins, just select from the list below:
 * [`test_control_api_plugin`](test_control_api_plugin/index.md)
 * [`test_control_plugin`](test_control_plugin/index.md)
 * [`trace_api_plugin`](trace_api_plugin/index.md)
-* [`txn_test_gen_plugin`](txn_test_gen_plugin/index.md)
 
 [[info | Nodeos is modular]]
 | Plugins add incremental functionality to `nodeos`. Unlike runtime plugins, `nodeos` plugins are built at compile-time.

docs/01_nodeos/03_plugins/net_plugin/index.md

@@ -44,7 +44,7 @@ Config Options for eosio::net_plugin:
   --p2p-accept-transactions arg (=1)    Allow transactions received over p2p
                                         network to be evaluated and relayed if
                                         valid.
-  --agent-name arg (="EOS Test Agent")  The name supplied to identify this node
+  --agent-name arg (=EOS Test Agent)    The name supplied to identify this node
                                         amongst the peers.
   --allowed-connection arg (=any)       Can be 'any' or 'producers' or
                                         'specified' or 'none'. If 'specified',
@@ -73,13 +73,15 @@ Config Options for eosio::net_plugin:
                                         synchronization
   --use-socket-read-watermark arg (=0)  Enable experimental socket read
                                         watermark optimization
-  --peer-log-format arg (=["${_name}" ${_ip}:${_port}])
+  --peer-log-format arg (=["${_name}" - ${_cid} ${_ip}:${_port}] )
                                         The string used to format peers when
                                         logging messages about them.  Variables
                                         are escaped with ${<variable name>}.
                                         Available Variables:
                                            _name  self-reported name
 
+                                           _cid   assigned connection id
+
                                            _id    self-reported ID (64 hex
                                                   characters)
 
@@ -94,7 +96,10 @@ Config Options for eosio::net_plugin:
                                                   peer
 
                                            _lport local port number connected
-                                                  to peer                                        
+                                                  to peer
+  --p2p-keepalive-interval-ms arg (=10000)
+                                        peer heartbeat keepalive message
+                                        interval in milliseconds
 ```
 
 ## Dependencies

docs/01_nodeos/03_plugins/producer_plugin/index.md

@@ -23,7 +23,6 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::producer_plugin:
-
   -e [ --enable-stale-production ]      Enable block production, even if the
                                         chain is stale.
   -x [ --pause-on-startup ]             Start this node in a state where
@@ -40,26 +39,22 @@ Config Options for eosio::producer_plugin:
   -p [ --producer-name ] arg            ID of producer controlled by this node
                                         (e.g. inita; may specify multiple
                                         times)
-  --private-key arg                     (DEPRECATED - Use signature-provider
-                                        instead) Tuple of [public key, WIF
-                                        private key] (may specify multiple
-                                        times)
   --signature-provider arg (=EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV=KEY:5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3)
                                         Key=Value pairs in the form
                                         <public-key>=<provider-spec>
                                         Where:
                                            <public-key>    is a string form of
-                                                           a vaild Antelope public
+                                                           a valid EOS public
                                                            key
 
                                            <provider-spec> is a string in the
                                                            form <provider-type>
                                                            :<data>
 
-                                           <provider-type> is KEY, or KEOSD
+                                           <provider-type> is KEY, KEOSD, or SE
 
                                            KEY:<data>      is a string form of
-                                                           a valid Antelope
+                                                           a valid EOS
                                                            private key which
                                                            maps to the provided
                                                            public key
@@ -69,10 +64,6 @@ Config Options for eosio::producer_plugin:
                                                            and the approptiate
                                                            wallet(s) are
                                                            unlocked
-  --keosd-provider-timeout arg (=5)     Limits the maximum time (in
-                                        milliseconds) that is allowed for
-                                        sending blocks to a keosd provider for
-                                        signing
   --greylist-account arg                account that can not access to extended
                                         CPU/NET virtual resources
   --greylist-limit arg (=1000)          Limit (between 1 and 1000) on the
@@ -107,9 +98,10 @@ Config Options for eosio::producer_plugin:
   --max-scheduled-transaction-time-per-block-ms arg (=100)
                                         Maximum wall-clock time, in
                                         milliseconds, spent retiring scheduled
-                                        transactions in any block before
-                                        returning to normal transaction
-                                        processing.
+                                        transactions (and incoming transactions
+                                        according to incoming-defer-ratio) in
+                                        any block before returning to normal
+                                        transaction processing.
   --subjective-cpu-leeway-us arg (=31000)
                                         Time in microseconds allowed for a
                                         transaction that starts with
@@ -119,11 +111,6 @@ Config Options for eosio::producer_plugin:
                                         Sets the maximum amount of failures
                                         that are allowed for a given account
                                         per block.
-                                        Disregarded for accounts that have been
-                                        whitelisted by disabling subjective
-                                        billing for the account using the
-                                        disable-subjective-account-billing
-                                        configuration option.
   --subjective-account-decay-time-minutes arg (=1440)
                                         Sets the time to return full subjective
                                         cpu for accounts
@@ -135,16 +122,9 @@ Config Options for eosio::producer_plugin:
                                         transaction queue. Exceeding this value
                                         will subjectively drop transaction with
                                         resource exhaustion.
-  --disable-api-persisted-trx           Disable the re-apply of API
-                                        transactions.
-  --disable-subjective-billing arg (=1) Disable subjective CPU billing for
-                                        API/P2P transactions
   --disable-subjective-account-billing arg
                                         Account which is excluded from
                                         subjective CPU billing
-                                        Account is considered whitelisted and
-                                        will not be subject to enforcement of
-                                        subjective-account-max-failures.
   --disable-subjective-p2p-billing arg (=1)
                                         Disable subjective CPU billing for P2P
                                         transactions

docs/01_nodeos/03_plugins/resource_monitor_plugin/index.md

@@ -1,19 +1,10 @@
 
 ## Overview
 
-The `resource_monitor_plugin` monitors space usage in the computing system where `nodeos` is running. Specifically, every `resource-monitor-interval-seconds` seconds,
-it measures the individual space used by each of the file systems mounted
-by `data-dir`, `state-dir`, `blocks-log-dir`, `snapshots-dir`,
-`state-history-dir`, and `trace-dir`.
-When space usage in any of the monitored file system is within `5%` of the threshold
-specified by `resource-monitor-space-threshold`, a warning containing the file system
-path and percentage of space has used is printed out.
-When space usage exceeds the threshold,
-if `resource-monitor-not-shutdown-on-threshold-exceeded` is not set,
-`nodeos` gracefully shuts down; if `resource-monitor-not-shutdown-on-threshold-exceeded` is set, `nodeos` prints out warnings periodically
-until space usage goes under the threshold.
+The `resource_monitor_plugin` monitors space usage in the computing system where `nodeos` is running. Specifically, every `resource-monitor-interval-seconds` seconds, it measures the individual space used by each of the file systems mounted by `data-dir`, `state-dir`, `blocks-log-dir`, `snapshots-dir`, `state-history-dir`, and `trace-dir`. When space usage in any of the monitored file system is within `5%` of the threshold specified by `resource-monitor-space-threshold`, a warning containing the file system path and percentage of space has used is printed out. When space usage exceeds the threshold, if `resource-monitor-not-shutdown-on-threshold-exceeded` is not set, `nodeos` gracefully shuts down; if `resource-monitor-not-shutdown-on-threshold-exceeded` is set, `nodeos` prints out warnings periodically until space usage goes under the threshold.
 
 `resource_monitor_plugin` is always loaded.
+
 ## Usage
 
 ```console
@@ -32,33 +23,28 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::resource_monitor_plugin:
-
   --resource-monitor-interval-seconds arg (=2)
-                                        Time in seconds between two consecutive checks
-                                        of space usage. Should be between 1 and 300.
+                                        Time in seconds between two consecutive
+                                        checks of resource usage. Should be
+                                        between 1 and 300
   --resource-monitor-space-threshold arg (=90)
-                                        Threshold in terms of percentage of used space
-                                        vs total space. If the used space is within
-                                        `5%` of the threshold, a warning is generated.
-                                        If the used space is above the threshold and
-                                        `resource-monitor-not-shutdown-on-threshold-exceeded`
-                                        is enabled, a shutdown is initiated; otherwise
-                                        a warning will be continuously printed out.
-                                        The value should be between 6 and 99.
+                                        Threshold in terms of percentage of
+                                        used space vs total space. If used
+                                        space is above (threshold - 5%), a
+                                        warning is generated.  Unless
+                                        resource-monitor-not-shutdown-on-thresh
+                                        old-exceeded is enabled, a graceful
+                                        shutdown is initiated if used space is
+                                        above the threshold. The value should
+                                        be between 6 and 99
   --resource-monitor-not-shutdown-on-threshold-exceeded
-                                        A switch used to indicate `nodeos` will "not"
-                                        shutdown when threshold is exceeded. When not
-                                        set, `nodeos` will shutdown.
+                                        Used to indicate nodeos will not
+                                        shutdown when threshold is exceeded.
   --resource-monitor-warning-interval arg (=30)
-                                        Number of monitor intervals between which a
-                                        warning is displayed.  For example, if
-                                        `resource-monitor-warning-interval` is to 10
-                                        and `resource-monitor-interval-seconds` is 2,
-                                        a warning will be displayed every 20 seconds,
-                                        even though the space usage is checked every
-                                        2 seconds.  This is used to throttle the
-                                        number of warnings in the `nodeos` log file.
-                                        Should be between 1 and 450.
+                                        Number of resource monitor intervals
+                                        between two consecutive warnings when
+                                        the threshold is hit. Should be between
+                                        1 and 450
 ```
 
 ## Plugin Dependencies

docs/01_nodeos/03_plugins/state_history_plugin/index.md

@@ -31,19 +31,24 @@ These can be specified from both the `nodeos` command-line or the `config.ini` f
 
 ```console
 Config Options for eosio::state_history_plugin:
-
   --state-history-dir arg (="state-history")
-                                        the location of the state-history 
+                                        the location of the state-history
                                         directory (absolute path or relative to
                                         application data dir)
   --trace-history                       enable trace history
   --chain-state-history                 enable chain state history
   --state-history-endpoint arg (=127.0.0.1:8080)
-                                        the endpoint upon which to listen for 
-                                        incoming connections. Caution: only 
-                                        expose this port to your internal 
+                                        the endpoint upon which to listen for
+                                        incoming connections. Caution: only
+                                        expose this port to your internal
                                         network.
+  --state-history-unix-socket-path arg  the path (relative to data-dir) to
+                                        create a unix socket upon which to
+                                        listen for incoming connections.
   --trace-history-debug-mode            enable debug mode for trace history
+  --state-history-log-retain-blocks arg if set, periodically prune the state
+                                        history files to store only configured
+                                        number of most recent blocks
 ```
 
 ## Examples

docs/01_nodeos/03_plugins/txn_test_gen_plugin/index.md

@@ -1,40 +0,0 @@
-
-## Description
-
-The `txn_test_gen_plugin` is used for transaction test purposes.
-
-[[info | For More Information]]
-For more information, check the [txn_test_gen_plugin/README.md](https://github.com/AntelopeIO/leap/tree/main/plugins/txn_test_gen_plugin) on the `AntelopeIO/leap` repository.
-
-## Usage
-
-```console
-# config.ini
-plugin = eosio::txn_test_gen_plugin
-[options]
-```
-```sh
-# command-line
-nodeos ... --plugin eosio::txn_test_gen_plugin [options]
-```
-
-## Options
-
-These can be specified from both the `nodeos` command-line or the `config.ini` file:
-
-```console
-Config Options for eosio::txn_test_gen_plugin:
-  --txn-reference-block-lag arg (=0)    Lag in number of blocks from the head 
-                                        block when selecting the reference 
-                                        block for transactions (-1 means Last 
-                                        Irreversible Block)
-  --txn-test-gen-threads arg (=2)       Number of worker threads in 
-                                        txn_test_gen thread pool
-  --txn-test-gen-account-prefix arg (=txn.test.)
-                                        Prefix to use for accounts generated 
-                                        and used by this plugin
-```
-
-## Dependencies
-
-None

docs/01_nodeos/06_logging/01_logging-levels.md

@@ -81,7 +81,7 @@ Sample `logging.json`:
   - Answers the question: is this useful information for a user that is monitoring the log output. Care should be taken not to create excessive log output; similar to `info` level logging.
   - Enabling `debug` level logging should provide greater insight into behavior without overwhelming the output with log entries.
   - `debug` level should not be used for *trace* level logging; to that end, use `all` (see below).
-  - Like `info`, no `debug` level logging should be produced for every transaction. There are specific transaction level loggers dedicated to transaction level logging: `transaction`, `transaction_trace_failure`, `transaction_trace_success`, `transaction_failure_tracing`, `transaction_success_tracing`.
+  - Like `info`, no `debug` level logging should be produced for every transaction. There are specific transaction level loggers dedicated to transaction level logging: `transaction`, `transaction_trace_failure`, `transaction_trace_success`, `transaction_failure_tracing`, `transaction_success_tracing`, `transient_trx_success_tracing`, `transient_trx_failure_tracing`.
 * `all` (trace) - For logging that would be overwhelming if `debug` level logging were used.
   - Can be used for trace level logging. Only used in a few places and not completely supported.
   - *Note*: In the future a different logging library may provide better trace level logging support. The current logging framework is not performant enough to enable excess trace level output.

docs/01_nodeos/07_concepts/05_storage-and-read-modes.md

@@ -27,44 +27,34 @@ The `nodeos` service provides query access to the chain database via the HTTP [R
 
 The `nodeos` service can be run in different "read" modes. These modes control how the node operates and how it processes blocks and transactions:
 
-- `speculative`: this includes the side effects of confirmed and unconfirmed transactions.
 - `head`: this only includes the side effects of confirmed transactions, this mode processes unconfirmed transactions but does not include them.
-- `read-only`: this mode is deprecated. Similar functionality can be achieved by combining options: `read-mode = head`, `p2p-accept-transactions = false`, `api-accept-transactions = false`. When these options are set, the local database will contain state changes made by transactions in the chain up to the head block. Also, transactions received via the P2P network are not relayed and transactions cannot be pushed via the chain API.
 - `irreversible`: this mode also includes confirmed transactions only up to those included in the last irreversible block.
+- `speculative`: this includes the side effects of confirmed and unconfirmed transactions.
 
 A transaction is considered confirmed when a `nodeos` instance has received, processed, and written it to a block on the blockchain, i.e. it is in the head block or an earlier block.
 
-### Speculative Mode
-
-Clients such as `cleos` and the RPC API, will see database state as of the current head block plus changes made by all transactions known to this node but potentially not included in the chain, unconfirmed transactions for example.
-
-Speculative mode is low latency but fragile, there is no guarantee that the transactions reflected in the state will be included in the chain OR that they will reflected in the same order the state implies.  
-
-This mode features the lowest latency, but is the least consistent.
-
-In speculative mode `nodeos` is able to execute transactions which have TaPoS (Transaction as Proof of Stake) pointing to any valid block in a fork considered to be the best fork by this node.
-
 ### Head Mode
 
-Clients such as `cleos` and the RPC API will see database state as of the current head block of the chain.  Since current head block is not yet irreversible and short-lived forks are possible, state read in this mode may become inaccurate  if `nodeos` switches to a better fork.  Note that this is also true of speculative mode.  
-
-This mode represents a good trade-off between highly consistent views of the data and latency.
+Clients such as `cleos` and the RPC API will see database state as of the current head block of the chain.  Since current head block is not yet irreversible and short-lived forks are possible, state read in this mode may become inaccurate  if `nodeos` switches to a better fork.  
 
 In this mode `nodeos` is able to execute transactions which have TaPoS pointing to any valid block in a fork considered to be the best fork by this node.
 
-### Read-Only Mode
-
-[[caution | Deprecation Notice]]
-| The explicit `read-mode = read-only` mode is deprecated. Similar functionality can now be achieved in `head` mode by combining options: `read-mode = head`, `p2p-accept-transactions = false`, `api-accept-transactions = false`.
-
-Clients such as `cleos` and the RPC API will see database state as of the current head block of the chain. It **will not** include changes made by transactions known to this node but not included in the chain, such as unconfirmed transactions.
-
 ### Irreversible Mode
 
 When `nodeos` is configured to be in irreversible read mode, it will still track the most up-to-date blocks in the fork database, but the state will lag behind the current best head block, sometimes referred to as the fork DB head, to always reflect the state of the last irreversible block.
 
 Clients such as `cleos` and the RPC API will see database state as of the current head block of the chain. It **will not** include changes made by transactions known to this node but not included in the chain, such as unconfirmed transactions.
 
+### Speculative Mode ( Deprecated )
+
+Clients such as `cleos` and the RPC API, will see database state as of the current head block plus changes made by all transactions known to this node but potentially not included in the chain, unconfirmed transactions for example.
+
+Speculative mode is low latency but fragile, there is no guarantee that the transactions reflected in the state will be included in the chain OR that they will reflected in the same order the state implies.
+
+This mode features the lowest latency, but is the least consistent.
+
+In speculative mode `nodeos` is able to execute transactions which have TaPoS (Transaction as Proof of Stake) pointing to any valid block in a fork considered to be the best fork by this node.
+
 ## How To Specify the Read Mode
 
 The mode in which `nodeos` is run can be specified using the `--read-mode` option from the `eosio::chain_plugin`.

docs/03_keosd/10_usage.md

@@ -44,16 +44,6 @@ Config Options for eosio::http_plugin:
   --http-server-address arg             The local IP and port to listen for
                                         incoming http connections; leave blank
                                         to disable.
-  --https-server-address arg            The local IP and port to listen for
-                                        incoming https connections; leave blank
-                                        to disable.
-  --https-certificate-chain-file arg    Filename with the certificate chain to
-                                        present on https connections. PEM
-                                        format. Required for https.
-  --https-private-key-file arg          Filename with https private key in PEM
-                                        format. Required for https
-  --https-ecdh-curve arg (=secp384r1)   Configure https ECDH curve to use:
-                                        secp384r1 or prime256v1
   --access-control-allow-origin arg     Specify the Access-Control-Allow-Origin
                                         to be returned on each request.
   --access-control-allow-headers arg    Specify the Access-Control-Allow-Header

docs/10_utilities/eosio-blocklog.md

@@ -1,38 +0,0 @@
----
-content_title: eosio-blocklog
-link_text: eosio-blocklog
----
-
-`eosio-blocklog` is a command-line interface (CLI) utility that allows node operators to perform low-level tasks on the block logs created by a `nodeos` instance. `eosio-blocklog` can perform one of the following operations:
-
-* Convert a range of blocks to JSON format, as single objects or array.
-* Generate `blocks.index` from `blocks.log` in blocks directory.
-* Trim `blocks.log` and `blocks.index` between a range of blocks.
-* Perform consistency test between `blocks.log` and `blocks.index`.
-* Output the results of the operation to a file or `stdout` (default).
-
-## Usage
-```sh
-eosio-blocklog <options> ...
-```
-
-## Options
-
-Option (=default) | Description
--|-
-`--blocks-dir arg (="blocks")` | The location of the blocks directory (absolute path or relative to the current directory)
-`-o [ --output-file ] arg` | The file to write the generated output to (absolute or relative path). If not specified then output is to `stdout`
-`-f [ --first ] arg (=0)` | The first block number to log or the first block to keep if `trim-blocklog` specified
-`-l [ --last ] arg (=4294967295)` | the last block number to log or the last block to keep if `trim-blocklog` specified
-`--no-pretty-print` | Do not pretty print the output. Useful if piping to `jq` to improve performance
-`--as-json-array` | Print out JSON blocks wrapped in JSON array (otherwise the output is free-standing JSON objects)
-`--make-index` | Create `blocks.index` from `blocks.log`. Must give `blocks-dir` location. Give `output-file` relative to current directory or absolute path (default is `<blocks-dir>/blocks.index`)
-`--trim-blocklog` | Trim `blocks.log` and `blocks.index`. Must give `blocks-dir` and `first` and/or `last` options.
-`--smoke-test` | Quick test that `blocks.log` and `blocks.index` are well formed and agree with each other
-`-h [ --help ]` | Print this help message and exit
-
-## Remarks
-
-When `eosio-blocklog` is launched, the utility attempts to perform the specified operation, then yields the following possible outcomes:
-* If successful, the selected operation is performed and the utility terminates with a zero error code (no error).
-* If unsuccessful, the utility outputs an error to `stderr` and terminates with a non-zero error code (indicating an error).

docs/10_utilities/index.md

@@ -5,5 +5,4 @@ link_text: Antelope Utilities
 
 This section contains documentation for additional utilities that complement or extend `nodeos` and potentially other Antelope software:
 
-* [eosio-blocklog](eosio-blocklog.md) - Low-level utility for node operators to interact with block log files.
 * [trace_api_util](trace_api_util.md) - Low-level utility for performing tasks associated with the [Trace API](../01_nodeos/03_plugins/trace_api_plugin/index.md).

libraries/CMakeLists.txt

@@ -3,6 +3,7 @@ set(FC_INSTALL_COMPONENT "dev")
 set(APPBASE_INSTALL_COMPONENT "dev")
 set(SOFTFLOAT_INSTALL_COMPONENT "dev")
 set(EOSVM_INSTALL_COMPONENT "dev")
+set(BN256_INSTALL_COMPONENT "dev")
 
 add_subdirectory( libfc )
 add_subdirectory( builtins )
@@ -11,6 +12,7 @@ add_subdirectory( wasm-jit )
 add_subdirectory( chainbase )
 set(APPBASE_ENABLE_AUTO_VERSION OFF CACHE BOOL "enable automatic discovery of version via 'git describe'")
 add_subdirectory( appbase )
+add_subdirectory( custom_appbase )
 add_subdirectory( chain )
 add_subdirectory( testing )
 add_subdirectory( version )
@@ -26,3 +28,4 @@ set(ENABLE_PROFILE OFF CACHE BOOL "Enable for profile builds")
 if(eos-vm IN_LIST EOSIO_WASM_RUNTIMES OR eos-vm-jit IN_LIST EOSIO_WASM_RUNTIMES)
 add_subdirectory( eos-vm )
 endif()
+add_subdirectory( prometheus )

libraries/chain/CMakeLists.txt

@@ -5,8 +5,8 @@ file(GLOB HEADERS "include/eosio/chain/*.hpp"
                   "include/eosio/chain/webassembly/*.hpp"
                   "${CMAKE_CURRENT_BINARY_DIR}/include/eosio/chain/core_symbol.hpp" )
 
-if(APPLE AND UNIX)
-   set(PLATFORM_TIMER_IMPL platform_timer_macos.cpp)
+if((APPLE AND UNIX) OR (${CMAKE_SYSTEM_NAME} STREQUAL "FreeBSD"))
+   set(PLATFORM_TIMER_IMPL platform_timer_kqueue.cpp)
 else()
    try_run(POSIX_TIMER_TEST_RUN_RESULT POSIX_TIMER_TEST_COMPILE_RESULT ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/platform_timer_posix_test.c)
    if(POSIX_TIMER_TEST_RUN_RESULT EQUAL 0)
@@ -42,7 +42,7 @@ if("eos-vm-oc" IN_LIST EOSIO_WASM_RUNTIMES)
    endif()
 
    llvm_map_components_to_libnames(LLVM_LIBS support core passes mcjit native orcjit)
-   include_directories(${LLVM_INCLUDE_DIRS})
+   include_directories(SYSTEM ${LLVM_INCLUDE_DIRS})
    add_definitions(${LLVM_DEFINITIONS})
 
    option(EOSVMOC_ENABLE_DEVELOPER_OPTIONS "enable developer options for EOS VM OC" OFF)
@@ -107,6 +107,7 @@ add_library( eosio_chain
               abi_serializer.cpp
               asset.cpp
               snapshot.cpp
+              snapshot_scheduler.cpp
               deep_mind.cpp
 
              ${CHAIN_EOSVMOC_SOURCES}
@@ -121,6 +122,7 @@ add_library( eosio_chain
              protocol_feature_manager.cpp
              producer_schedule.cpp
              genesis_intrinsics.cpp
+             symbol.cpp
              whitelisted_intrinsics.cpp
              thread_utils.cpp
              platform_timer_accuracy.cpp
@@ -128,7 +130,7 @@ add_library( eosio_chain
              ${HEADERS}
              )
 
-target_link_libraries( eosio_chain PUBLIC fc chainbase eosio_rapidjson Logging IR WAST WASM Runtime
+target_link_libraries( eosio_chain PUBLIC bn256 fc chainbase eosio_rapidjson Logging IR WAST WASM
                        softfloat builtins ${CHAIN_EOSVM_LIBRARIES} ${LLVM_LIBS} ${CHAIN_RT_LINKAGE}
                      )
 target_include_directories( eosio_chain

libraries/chain/abi_serializer.cpp

@@ -4,6 +4,7 @@
 #include <fc/io/raw.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <fc/io/varint.hpp>
+#include <fc/time.hpp>
 
 namespace eosio { namespace chain {
 
@@ -23,11 +24,11 @@ namespace eosio { namespace chain {
 
    template <typename T>
    inline fc::variant variant_from_stream(fc::datastream<const char*>& stream, const abi_serializer::yield_function_t& yield) {
-      fc::yield_function_t y = [&yield](){ yield(0); }; // create yield function matching fc::variant requirements, 0 for recursive depth
       T temp;
       fc::raw::unpack( stream, temp );
-      y();
-      return fc::variant( temp, y );
+      yield(0);
+      // create yield function matching fc::variant requirements, 0 for recursive depth
+      return fc::variant( temp, [yield](){ yield(0); } );
    }
 
    template <typename T>
@@ -70,9 +71,9 @@ namespace eosio { namespace chain {
       );
    }
 
-   abi_serializer::abi_serializer( const abi_def& abi, const yield_function_t& yield ) {
+   abi_serializer::abi_serializer( abi_def abi, const yield_function_t& yield ) {
       configure_built_in_types();
-      set_abi(abi, yield);
+      set_abi(std::move(abi), yield);
    }
 
    abi_serializer::abi_serializer( const abi_def& abi, const fc::microseconds& max_serialization_time) {
@@ -128,11 +129,19 @@ namespace eosio { namespace chain {
       built_in_types.emplace("extended_asset",            pack_unpack<extended_asset>());
    }
 
-   void abi_serializer::set_abi(const abi_def& abi, const yield_function_t& yield) {
-      impl::abi_traverse_context ctx(yield);
+   void abi_serializer::set_abi(abi_def abi, const yield_function_t& yield) {
+      impl::abi_traverse_context ctx(yield, fc::microseconds{});
 
       EOS_ASSERT(starts_with(abi.version, "eosio::abi/1."), unsupported_abi_version_exception, "ABI has an unsupported version");
 
+      size_t types_size = abi.types.size();
+      size_t structs_size = abi.structs.size();
+      size_t actions_size = abi.actions.size();
+      size_t tables_size = abi.tables.size();
+      size_t error_messages_size = abi.error_messages.size();
+      size_t variants_size = abi.variants.value.size();
+      size_t action_results_size = abi.action_results.value.size();
+
       typedefs.clear();
       structs.clear();
       actions.clear();
@@ -141,41 +150,41 @@ namespace eosio { namespace chain {
       variants.clear();
       action_results.clear();
 
-      for( const auto& st : abi.structs )
-         structs[st.name] = st;
+      for( auto& st : abi.structs )
+         structs[st.name] = std::move(st);
 
-      for( const auto& td : abi.types ) {
+      for( auto& td : abi.types ) {
          EOS_ASSERT(!_is_type(td.new_type_name, ctx), duplicate_abi_type_def_exception,
                     "type already exists", ("new_type_name",impl::limit_size(td.new_type_name)));
-         typedefs[td.new_type_name] = td.type;
+         typedefs[std::move(td.new_type_name)] = std::move(td.type);
       }
 
-      for( const auto& a : abi.actions )
-         actions[a.name] = a.type;
+      for( auto& a : abi.actions )
+         actions[std::move(a.name)] = std::move(a.type);
 
-      for( const auto& t : abi.tables )
-         tables[t.name] = t.type;
+      for( auto& t : abi.tables )
+         tables[std::move(t.name)] = std::move(t.type);
 
-      for( const auto& e : abi.error_messages )
-         error_messages[e.error_code] = e.error_msg;
+      for( auto& e : abi.error_messages )
+         error_messages[std::move(e.error_code)] = std::move(e.error_msg);
 
-      for( const auto& v : abi.variants.value )
-         variants[v.name] = v;
+      for( auto& v : abi.variants.value )
+         variants[v.name] = std::move(v);
 
-      for( const auto& r : abi.action_results.value )
-         action_results[r.name] = r.result_type;
+      for( auto& r : abi.action_results.value )
+         action_results[std::move(r.name)] = std::move(r.result_type);
 
       /**
        *  The ABI vector may contain duplicates which would make it
        *  an invalid ABI
        */
-      EOS_ASSERT( typedefs.size() == abi.types.size(), duplicate_abi_type_def_exception, "duplicate type definition detected" );
-      EOS_ASSERT( structs.size() == abi.structs.size(), duplicate_abi_struct_def_exception, "duplicate struct definition detected" );
-      EOS_ASSERT( actions.size() == abi.actions.size(), duplicate_abi_action_def_exception, "duplicate action definition detected" );
-      EOS_ASSERT( tables.size() == abi.tables.size(), duplicate_abi_table_def_exception, "duplicate table definition detected" );
-      EOS_ASSERT( error_messages.size() == abi.error_messages.size(), duplicate_abi_err_msg_def_exception, "duplicate error message definition detected" );
-      EOS_ASSERT( variants.size() == abi.variants.value.size(), duplicate_abi_variant_def_exception, "duplicate variant definition detected" );
-      EOS_ASSERT( action_results.size() == abi.action_results.value.size(), duplicate_abi_action_results_def_exception, "duplicate action results definition detected" );
+      EOS_ASSERT( typedefs.size() == types_size, duplicate_abi_type_def_exception, "duplicate type definition detected" );
+      EOS_ASSERT( structs.size() == structs_size, duplicate_abi_struct_def_exception, "duplicate struct definition detected" );
+      EOS_ASSERT( actions.size() == actions_size, duplicate_abi_action_def_exception, "duplicate action definition detected" );
+      EOS_ASSERT( tables.size() == tables_size, duplicate_abi_table_def_exception, "duplicate table definition detected" );
+      EOS_ASSERT( error_messages.size() == error_messages_size, duplicate_abi_err_msg_def_exception, "duplicate error message definition detected" );
+      EOS_ASSERT( variants.size() == variants_size, duplicate_abi_variant_def_exception, "duplicate variant definition detected" );
+      EOS_ASSERT( action_results.size() == action_results_size, duplicate_abi_action_results_def_exception, "duplicate action results definition detected" );
 
       validate(ctx);
    }
@@ -227,7 +236,7 @@ namespace eosio { namespace chain {
    }
 
    bool abi_serializer::is_type(const std::string_view& type, const yield_function_t& yield)const {
-      impl::abi_traverse_context ctx(yield);
+      impl::abi_traverse_context ctx(yield, fc::microseconds{});
       return _is_type(type, ctx);
    }
 
@@ -456,23 +465,27 @@ namespace eosio { namespace chain {
    }
 
    fc::variant abi_serializer::binary_to_variant( const std::string_view& type, const bytes& binary, const yield_function_t& yield, bool short_path )const {
-      impl::binary_to_variant_context ctx(*this, yield, type);
+      impl::binary_to_variant_context ctx(*this, yield, fc::microseconds{}, type);
       ctx.short_path = short_path;
       return _binary_to_variant(type, binary, ctx);
    }
 
-   fc::variant abi_serializer::binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path )const {
-      return binary_to_variant( type, binary, create_yield_function(max_serialization_time), short_path );
+   fc::variant abi_serializer::binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path )const {
+      impl::binary_to_variant_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type);
+      ctx.short_path = short_path;
+      return _binary_to_variant(type, binary, ctx);
    }
 
    fc::variant abi_serializer::binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const yield_function_t& yield, bool short_path )const {
-      impl::binary_to_variant_context ctx(*this, yield, type);
+      impl::binary_to_variant_context ctx(*this, yield, fc::microseconds{}, type);
       ctx.short_path = short_path;
       return _binary_to_variant(type, binary, ctx);
    }
 
-   fc::variant abi_serializer::binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const fc::microseconds& max_serialization_time, bool short_path )const {
-      return binary_to_variant( type, binary, create_yield_function(max_serialization_time), short_path );
+   fc::variant abi_serializer::binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path )const {
+      impl::binary_to_variant_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type);
+      ctx.short_path = short_path;
+      return _binary_to_variant(type, binary, ctx);
    }
 
    void abi_serializer::_variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char *>& ds, impl::variant_to_binary_context& ctx )const
@@ -535,14 +548,15 @@ namespace eosio { namespace chain {
             bool disallow_additional_fields = false;
             for( uint32_t i = 0; i < st.fields.size(); ++i ) {
                const auto& field = st.fields[i];
-               if( vo.contains( string(field.name).c_str() ) ) {
+               bool present = vo.contains(string(field.name).c_str());
+               if( present || is_optional(field.type) ) {
                   if( disallow_additional_fields )
                      EOS_THROW( pack_exception, "Unexpected field '${f}' found in input object while processing struct '${p}'",
                                 ("f", ctx.maybe_shorten(field.name))("p", ctx.get_path_string()) );
                   {
                      auto h1 = ctx.push_to_path( impl::field_path_item{ .parent_struct_itr = s_itr, .field_ordinal = i } );
                      auto h2 = ctx.disallow_extensions_unless( &field == &st.fields.back() );
-                     _variant_to_binary(_remove_bin_extension(field.type), vo[field.name], ds, ctx);
+                     _variant_to_binary(_remove_bin_extension(field.type), present ? vo[field.name] : fc::variant(nullptr), ds, ctx);
                   }
                } else if( ends_with(field.type, "$") && ctx.extensions_allowed() ) {
                   disallow_additional_fields = true;
@@ -595,23 +609,27 @@ namespace eosio { namespace chain {
    } FC_CAPTURE_AND_RETHROW() }
 
    bytes abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, const yield_function_t& yield, bool short_path )const {
-      impl::variant_to_binary_context ctx(*this, yield, type);
+      impl::variant_to_binary_context ctx(*this, yield, fc::microseconds{}, type);
       ctx.short_path = short_path;
       return _variant_to_binary(type, var, ctx);
    }
 
-   bytes abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path ) const {
-       return variant_to_binary( type, var, create_yield_function(max_serialization_time), short_path );
+   bytes abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_action_data_serialization_time, bool short_path ) const {
+      impl::variant_to_binary_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type);
+      ctx.short_path = short_path;
+      return _variant_to_binary(type, var, ctx);
    }
 
    void  abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const yield_function_t& yield, bool short_path )const {
-      impl::variant_to_binary_context ctx(*this, yield, type);
+      impl::variant_to_binary_context ctx(*this, yield, fc::microseconds{}, type);
       ctx.short_path = short_path;
       _variant_to_binary(type, var, ds, ctx);
    }
 
-   void  abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const fc::microseconds& max_serialization_time, bool short_path ) const {
-       variant_to_binary( type, var, create_yield_function(max_serialization_time), short_path );
+   void  abi_serializer::variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const fc::microseconds& max_action_data_serialization_time, bool short_path ) const {
+      impl::variant_to_binary_context ctx(*this, create_depth_yield_function(), max_action_data_serialization_time, type);
+      ctx.short_path = short_path;
+      _variant_to_binary(type, var, ds, ctx);
    }
 
    type_name abi_serializer::get_action_type(name action)const {
@@ -642,9 +660,7 @@ namespace eosio { namespace chain {
    namespace impl {
 
       fc::scoped_exit<std::function<void()>> abi_traverse_context::enter_scope() {
-         std::function<void()> callback = [old_recursion_depth=recursion_depth, this](){
-            recursion_depth = old_recursion_depth;
-         };
+         std::function<void()> callback = [this](){ --recursion_depth; };
 
          ++recursion_depth;
          yield( recursion_depth );

libraries/chain/apply_context.cpp

@@ -19,15 +19,26 @@ namespace eosio { namespace chain {
 
 static inline void print_debug(account_name receiver, const action_trace& ar) {
    if (!ar.console.empty()) {
-      auto prefix = fc::format_string(
-                                      "\n[(${a},${n})->${r}]",
-                                      fc::mutable_variant_object()
-                                      ("a", ar.act.account)
-                                      ("n", ar.act.name)
-                                      ("r", receiver));
-      dlog(prefix + ": CONSOLE OUTPUT BEGIN =====================\n"
-           + ar.console
-           + prefix + ": CONSOLE OUTPUT END   =====================" );
+      if (fc::logger::get(DEFAULT_LOGGER).is_enabled( fc::log_level::debug )) {
+         std::string prefix;
+         prefix.reserve(3 + 13 + 1 + 13 + 3 + 13 + 1);
+         prefix += "\n[(";
+         prefix += ar.act.account.to_string();
+         prefix += ",";
+         prefix += ar.act.name.to_string();
+         prefix += ")->";
+         prefix += receiver.to_string();
+         prefix += "]";
+
+         std::string output;
+         output.reserve(512);
+         output += prefix;
+         output += ": CONSOLE OUTPUT BEGIN =====================\n";
+         output += ar.console;
+         output += prefix;
+         output += ": CONSOLE OUTPUT END   =====================";
+         dlog( std::move(output) );
+      }
    }
 }
 
@@ -75,7 +86,7 @@ void apply_context::exec_one()
             privileged = receiver_account->is_privileged();
             auto native = control.find_apply_handler( receiver, act->account, act->name );
             if( native ) {
-               if( trx_context.enforce_whiteblacklist && control.is_producing_block() ) {
+               if( trx_context.enforce_whiteblacklist && control.is_speculative_block() ) {
                   control.check_contract_list( receiver );
                   control.check_action_list( act->account, act->name );
                }
@@ -89,7 +100,7 @@ void apply_context::exec_one()
                      || control.is_builtin_activated( builtin_protocol_feature_t::forward_setcode )
                   )
             ) {
-               if( trx_context.enforce_whiteblacklist && control.is_producing_block() ) {
+               if( trx_context.enforce_whiteblacklist && control.is_speculative_block() ) {
                   control.check_contract_list( receiver );
                   control.check_action_list( act->account, act->name );
                }
@@ -182,7 +193,7 @@ void apply_context::exec_one()
       print_debug(receiver, trace);
    }
 
-   if (auto dm_logger = control.get_deep_mind_logger())
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient()))
    {
       dm_logger->on_end_action();
    }
@@ -289,7 +300,7 @@ void apply_context::require_recipient( account_name recipient ) {
          schedule_action( action_ordinal, recipient, false )
       );
 
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_require_recipient();
       }
    }
@@ -316,12 +327,12 @@ void apply_context::execute_inline( action&& a ) {
    EOS_ASSERT( code != nullptr, action_validate_exception,
                "inline action's code account ${account} does not exist", ("account", a.account) );
 
-   bool enforce_actor_whitelist_blacklist = trx_context.enforce_whiteblacklist && control.is_producing_block();
+   bool enforce_actor_whitelist_blacklist = trx_context.enforce_whiteblacklist && control.is_speculative_block();
    flat_set<account_name> actors;
 
    bool disallow_send_to_self_bypass = control.is_builtin_activated( builtin_protocol_feature_t::restrict_action_to_self );
    bool send_to_self = (a.account == receiver);
-   bool inherit_parent_authorizations = (!disallow_send_to_self_bypass && send_to_self && (receiver == act->account) && control.is_producing_block());
+   bool inherit_parent_authorizations = (!disallow_send_to_self_bypass && send_to_self && (receiver == act->account) && control.is_speculative_block());
 
    flat_set<permission_level> inherited_authorizations;
    if( inherit_parent_authorizations ) {
@@ -347,14 +358,14 @@ void apply_context::execute_inline( action&& a ) {
       control.check_actor_list( actors );
    }
 
-   if( !privileged && control.is_producing_block() ) {
+   if( !privileged && control.is_speculative_block() ) {
       const auto& chain_config = control.get_global_properties().configuration;
       EOS_ASSERT( a.data.size() < std::min(chain_config.max_inline_action_size, control.get_max_nonprivileged_inline_action_size()),
                   inline_action_too_big_nonprivileged,
                   "inline action too big for nonprivileged account ${account}", ("account", a.account));
    }
    // No need to check authorization if replaying irreversible blocks or contract is privileged
-   if( !control.skip_auth_check() && !privileged ) {
+   if( !control.skip_auth_check() && !privileged && !trx_context.is_read_only() ) {
       try {
          control.get_authorization_manager()
                 .check_authorization( {a},
@@ -363,7 +374,7 @@ void apply_context::execute_inline( action&& a ) {
                                       control.pending_block_time() - trx_context.published,
                                       std::bind(&transaction_context::checktime, &this->trx_context),
                                       false,
-                                      trx_context.is_read_only,
+                                      trx_context.is_dry_run(), // check_but_dont_fail
                                       inherited_authorizations
                                     );
 
@@ -373,7 +384,7 @@ void apply_context::execute_inline( action&& a ) {
       } catch( const fc::exception& e ) {
          if( disallow_send_to_self_bypass || !send_to_self ) {
             throw;
-         } else if( control.is_producing_block() ) {
+         } else if( control.is_speculative_block() ) {
             subjective_block_production_exception new_exception(FC_LOG_MESSAGE( error, "Authorization failure with inline action sent to self"));
             for (const auto& log: e.get_log()) {
                new_exception.append_log(log);
@@ -383,7 +394,7 @@ void apply_context::execute_inline( action&& a ) {
       } catch( ... ) {
          if( disallow_send_to_self_bypass || !send_to_self ) {
             throw;
-         } else if( control.is_producing_block() ) {
+         } else if( control.is_speculative_block() ) {
             EOS_THROW(subjective_block_production_exception, "Unexpected exception occurred validating inline action sent to self");
          }
       }
@@ -394,7 +405,7 @@ void apply_context::execute_inline( action&& a ) {
       schedule_action( std::move(a), inline_receiver, false )
    );
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_send_inline();
    }
 }
@@ -407,7 +418,7 @@ void apply_context::execute_context_free_inline( action&& a ) {
    EOS_ASSERT( a.authorization.size() == 0, action_validate_exception,
                "context-free actions cannot have authorizations" );
 
-   if( !privileged && control.is_producing_block() ) {
+   if( !privileged && control.is_speculative_block() ) {
       const auto& chain_config = control.get_global_properties().configuration;
       EOS_ASSERT( a.data.size() < std::min(chain_config.max_inline_action_size, control.get_max_nonprivileged_inline_action_size()),
                   inline_action_too_big_nonprivileged,
@@ -419,16 +430,17 @@ void apply_context::execute_context_free_inline( action&& a ) {
       schedule_action( std::move(a), inline_receiver, true )
    );
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_send_context_free_inline();
    }
 }
 
 
 void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, account_name payer, transaction&& trx, bool replace_existing ) {
+   EOS_ASSERT( !trx_context.is_read_only(), transaction_exception, "cannot schedule a deferred transaction from within a readonly transaction" );
    EOS_ASSERT( trx.context_free_actions.size() == 0, cfa_inside_generated_tx, "context free actions are not currently allowed in generated transactions" );
 
-   bool enforce_actor_whitelist_blacklist = trx_context.enforce_whiteblacklist && control.is_producing_block()
+   bool enforce_actor_whitelist_blacklist = trx_context.enforce_whiteblacklist && control.is_speculative_block()
                                              && !control.sender_avoids_whitelist_blacklist_enforcement( receiver );
    trx_context.validate_referenced_accounts( trx, enforce_actor_whitelist_blacklist );
 
@@ -466,7 +478,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
       trx.ref_block_num = 0;
       trx.ref_block_prefix = 0;
    } else {
-      trx.expiration = control.pending_block_time() + fc::microseconds(999'999); // Rounds up to nearest second (makes expiration check unnecessary)
+      trx.expiration = time_point_sec{control.pending_block_time() + fc::microseconds(999'999)}; // Rounds up to nearest second (makes expiration check unnecessary)
       trx.set_reference_block(control.head_block_id()); // No TaPoS check necessary
    }
 
@@ -530,7 +542,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
       } catch( const fc::exception& e ) {
          if( disallow_send_to_self_bypass || !is_sending_only_to_self(receiver) ) {
             throw;
-         } else if( control.is_producing_block() ) {
+         } else if( control.is_speculative_block() ) {
             subjective_block_production_exception new_exception(FC_LOG_MESSAGE( error, "Authorization failure with sent deferred transaction consisting only of actions to self"));
             for (const auto& log: e.get_log()) {
                new_exception.append_log(log);
@@ -540,7 +552,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
       } catch( ... ) {
          if( disallow_send_to_self_bypass || !is_sending_only_to_self(receiver) ) {
             throw;
-         } else if( control.is_producing_block() ) {
+         } else if( control.is_speculative_block() ) {
             EOS_THROW(subjective_block_production_exception, "Unexpected exception occurred validating sent deferred transaction consisting only of actions to self");
          }
       }
@@ -552,12 +564,12 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
 
       bool replace_deferred_activated = control.is_builtin_activated(builtin_protocol_feature_t::replace_deferred);
 
-      EOS_ASSERT( replace_deferred_activated || !control.is_producing_block()
+      EOS_ASSERT( replace_deferred_activated || !control.is_speculative_block()
                      || control.all_subjective_mitigations_disabled(),
                   subjective_block_production_exception,
                   "Replacing a deferred transaction is temporarily disabled." );
 
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", ptr->id)), "deferred_trx", "cancel", "deferred_trx_cancel");
       }
 
@@ -575,7 +587,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
          trx_id_for_new_obj = ptr->trx_id;
       }
 
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_cancel_deferred(deep_mind_handler::operation_qualifier::modify, *ptr);
       }
 
@@ -592,7 +604,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
 
          trx_size = gtx.set( trx );
 
-         if (auto dm_logger = control.get_deep_mind_logger()) {
+         if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
             dm_logger->on_send_deferred(deep_mind_handler::operation_qualifier::modify, gtx);
             dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", gtx.id)), "deferred_trx", "update", "deferred_trx_add");
          }
@@ -609,7 +621,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
 
          trx_size = gtx.set( trx );
 
-         if (auto dm_logger = control.get_deep_mind_logger()) {
+         if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
             dm_logger->on_send_deferred(deep_mind_handler::operation_qualifier::none, gtx);
             dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", gtx.id)), "deferred_trx", "add", "deferred_trx_add");
          }
@@ -626,10 +638,11 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a
 }
 
 bool apply_context::cancel_deferred_transaction( const uint128_t& sender_id, account_name sender ) {
+   EOS_ASSERT( !trx_context.is_read_only(), transaction_exception, "cannot cancel a deferred transaction from within a readonly transaction" );
    auto& generated_transaction_idx = db.get_mutable_index<generated_transaction_multi_index>();
    const auto* gto = db.find<generated_transaction_object,by_sender_id>(boost::make_tuple(sender, sender_id));
    if ( gto ) {
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_cancel_deferred(deep_mind_handler::operation_qualifier::none, *gto);
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", gto->id)), "deferred_trx", "cancel", "deferred_trx_cancel");
       }
@@ -670,7 +683,7 @@ const table_id_object& apply_context::find_or_create_table( name code, name scop
       return *existing_tid;
    }
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       std::string event_id = RAM_EVENT_ID("${code}:${scope}:${table}",
          ("code", code)
          ("scope", scope)
@@ -687,14 +700,14 @@ const table_id_object& apply_context::find_or_create_table( name code, name scop
       t_id.table = table;
       t_id.payer = payer;
 
-      if (auto dm_logger = control.get_deep_mind_logger()) {
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
          dm_logger->on_create_table(t_id);
       }
    });
 }
 
 void apply_context::remove_table( const table_id_object& tid ) {
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       std::string event_id = RAM_EVENT_ID("${code}:${scope}:${table}",
          ("code", tid.code)
          ("scope", tid.scope)
@@ -705,7 +718,7 @@ void apply_context::remove_table( const table_id_object& tid ) {
 
    update_db_usage(tid.payer, - config::billable_size_v<table_id_object>);
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_remove_table(tid);
    }
 
@@ -778,11 +791,13 @@ int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t b
 }
 
 int apply_context::db_store_i64( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ) {
+   EOS_ASSERT( !trx_context.is_read_only(), table_operation_not_permitted, "cannot store a db record when executing a readonly transaction" );
    return db_store_i64( receiver, scope, table, payer, id, buffer, buffer_size);
 }
 
 int apply_context::db_store_i64( name code, name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ) {
 //   require_write_lock( scope );
+   EOS_ASSERT( !trx_context.is_read_only(), table_operation_not_permitted, "cannot store a db record when executing a readonly transaction" );
    const auto& tab = find_or_create_table( code, scope, table, payer );
    auto tableid = tab.id;
 
@@ -801,7 +816,7 @@ int apply_context::db_store_i64( name code, name scope, name table, const accoun
 
    int64_t billable_size = (int64_t)(buffer_size + config::billable_size_v<key_value_object>);
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       std::string event_id = RAM_EVENT_ID("${table_code}:${scope}:${table_name}:${primkey}",
          ("table_code", tab.code)
          ("scope", tab.scope)
@@ -813,7 +828,7 @@ int apply_context::db_store_i64( name code, name scope, name table, const accoun
 
    update_db_usage( payer, billable_size);
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_db_store_i64(tab, obj);
    }
 
@@ -822,6 +837,7 @@ int apply_context::db_store_i64( name code, name scope, name table, const accoun
 }
 
 void apply_context::db_update_i64( int iterator, account_name payer, const char* buffer, size_t buffer_size ) {
+   EOS_ASSERT( !trx_context.is_read_only(), table_operation_not_permitted, "cannot update a db record when executing a readonly transaction" );
    const key_value_object& obj = keyval_cache.get( iterator );
 
    const auto& table_obj = keyval_cache.get_table( obj.t_id );
@@ -836,7 +852,7 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char*
    if( payer == account_name() ) payer = obj.payer;
 
    std::string event_id;
-   if (control.get_deep_mind_logger() != nullptr) {
+   if (control.get_deep_mind_logger(trx_context.is_transient()) != nullptr) {
       event_id = RAM_EVENT_ID("${table_code}:${scope}:${table_name}:${primkey}",
          ("table_code", table_obj.code)
          ("scope", table_obj.scope)
@@ -847,27 +863,27 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char*
 
    if( account_name(obj.payer) != payer ) {
       // refund the existing payer
-      if (auto dm_logger = control.get_deep_mind_logger())
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient()))
       {
          dm_logger->on_ram_trace(std::string(event_id), "table_row", "remove", "primary_index_update_remove_old_payer");
       }
       update_db_usage( obj.payer,  -(old_size) );
       // charge the new payer
-      if (auto dm_logger = control.get_deep_mind_logger())
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient()))
       {
          dm_logger->on_ram_trace(std::move(event_id), "table_row", "add", "primary_index_update_add_new_payer");
       }
       update_db_usage( payer,  (new_size));
    } else if(old_size != new_size) {
       // charge/refund the existing payer the difference
-      if (auto dm_logger = control.get_deep_mind_logger())
+      if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient()))
       {
          dm_logger->on_ram_trace(std::move(event_id) , "table_row", "update", "primary_index_update");
       }
       update_db_usage( obj.payer, new_size - old_size);
    }
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_db_update_i64(table_obj, obj, payer, buffer, buffer_size);
    }
 
@@ -878,6 +894,7 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char*
 }
 
 void apply_context::db_remove_i64( int iterator ) {
+   EOS_ASSERT( !trx_context.is_read_only(), table_operation_not_permitted, "cannot remove a db record when executing a readonly transaction" );
    const key_value_object& obj = keyval_cache.get( iterator );
 
    const auto& table_obj = keyval_cache.get_table( obj.t_id );
@@ -885,7 +902,7 @@ void apply_context::db_remove_i64( int iterator ) {
 
 //   require_write_lock( table_obj.scope );
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       std::string event_id = RAM_EVENT_ID("${table_code}:${scope}:${table_name}:${primkey}",
          ("table_code", table_obj.code)
          ("scope", table_obj.scope)
@@ -897,7 +914,7 @@ void apply_context::db_remove_i64( int iterator ) {
 
    update_db_usage( obj.payer,  -(obj.value.size() + config::billable_size_v<key_value_object>) );
 
-   if (auto dm_logger = control.get_deep_mind_logger()) {
+   if (auto dm_logger = control.get_deep_mind_logger(trx_context.is_transient())) {
       dm_logger->on_db_remove_i64(table_obj, obj);
    }
 
@@ -1029,17 +1046,27 @@ int apply_context::db_end_i64( name code, name scope, name table ) {
 
 uint64_t apply_context::next_global_sequence() {
    const auto& p = control.get_dynamic_global_properties();
-   db.modify( p, [&]( auto& dgp ) {
-      ++dgp.global_action_sequence;
-   });
-   return p.global_action_sequence;
+   if ( trx_context.is_read_only() ) {
+      // To avoid confusion of duplicated global sequence number, hard code to be 0.
+      return 0;
+   } else {
+      db.modify( p, [&]( auto& dgp ) {
+         ++dgp.global_action_sequence;
+      });
+      return p.global_action_sequence;
+   }
 }
 
 uint64_t apply_context::next_recv_sequence( const account_metadata_object& receiver_account ) {
-   db.modify( receiver_account, [&]( auto& ra ) {
-      ++ra.recv_sequence;
-   });
-   return receiver_account.recv_sequence;
+   if ( trx_context.is_read_only() ) {
+      // To avoid confusion of duplicated receive sequence number, hard code to be 0.
+      return 0;
+   } else {
+      db.modify( receiver_account, [&]( auto& ra ) {
+         ++ra.recv_sequence;
+      });
+      return receiver_account.recv_sequence;
+   }
 }
 uint64_t apply_context::next_auth_sequence( account_name actor ) {
    const auto& amo = db.get<account_metadata_object,by_name>( actor );
@@ -1067,4 +1094,19 @@ action_name apply_context::get_sender() const {
    return action_name();
 }
 
+// Context             |    OC?
+//-------------------------------------------------------------------------------
+// Building block      | baseline, OC for eosio.*
+// Applying block      | OC unless a producer, OC for eosio.* including producers
+// Speculative API trx | baseline, OC for eosio.*
+// Speculative P2P trx | baseline, OC for eosio.*
+// Compute trx         | baseline, OC for eosio.*
+// Read only trx       | OC
+bool apply_context::should_use_eos_vm_oc()const {
+   return receiver.prefix() == config::system_account_name // "eosio"_n, all cases use OC
+          || (is_applying_block() && !control.is_producer_node()) // validating/applying block
+          || trx_context.is_read_only();
+}
+
+
 } } /// eosio::chain

libraries/chain/asset.cpp

@@ -1,9 +1,14 @@
+#include <string>
 #include <eosio/chain/asset.hpp>
 #include <boost/rational.hpp>
+#include <boost/algorithm/string.hpp>
 #include <fc/reflect/variant.hpp>
+#include <fc/safe.hpp>
 
 namespace eosio { namespace chain {
 
+using fc::safe;
+
 uint8_t asset::decimals()const {
    return sym.decimals();
 }
@@ -19,11 +24,11 @@ int64_t asset::precision()const {
 string asset::to_string()const {
    string sign = amount < 0 ? "-" : "";
    int64_t abs_amount = std::abs(amount);
-   string result = fc::to_string( static_cast<int64_t>(abs_amount) / precision());
+   string result = std::to_string( static_cast<int64_t>(abs_amount) / precision());
    if( decimals() )
    {
       auto fract = static_cast<int64_t>(abs_amount) % precision();
-      result += "." + fc::to_string(precision() + fract).erase(0,1);
+      result += "." + std::to_string(precision() + fract).erase(0,1);
    }
    return sign + result + " " + symbol_name();
 }
@@ -31,12 +36,12 @@ string asset::to_string()const {
 asset asset::from_string(const string& from)
 {
    try {
-      string s = fc::trim(from);
+      string s = boost::algorithm::trim_copy(from);
 
       // Find space in order to split amount and symbol
       auto space_pos = s.find(' ');
       EOS_ASSERT((space_pos != string::npos), asset_type_exception, "Asset's amount and symbol should be separated with space");
-      auto symbol_str = fc::trim(s.substr(space_pos + 1));
+      auto symbol_str = boost::algorithm::trim_copy(s.substr(space_pos + 1));
       auto amount_str = s.substr(0, space_pos);
 
       // Ensure that if decimal point is used (.), decimal fraction is specified

libraries/chain/authorization_manager.cpp

@@ -134,6 +134,7 @@ namespace eosio { namespace chain {
                                                                       permission_name name,
                                                                       permission_id_type parent,
                                                                       const authority& auth,
+                                                                      bool is_trx_transient,
                                                                       time_point initial_creation_time
                                                                     )
    {
@@ -158,7 +159,7 @@ namespace eosio { namespace chain {
          p.last_updated = creation_time;
          p.auth         = auth;
 
-         if (auto dm_logger = _control.get_deep_mind_logger()) {
+         if (auto dm_logger = _control.get_deep_mind_logger(is_trx_transient)) {
             dm_logger->on_create_permission(p);
          }
       });
@@ -169,6 +170,7 @@ namespace eosio { namespace chain {
                                                                       permission_name name,
                                                                       permission_id_type parent,
                                                                       authority&& auth,
+                                                                      bool is_trx_transient,
                                                                       time_point initial_creation_time
                                                                     )
    {
@@ -193,20 +195,20 @@ namespace eosio { namespace chain {
          p.last_updated = creation_time;
          p.auth         = std::move(auth);
 
-         if (auto dm_logger = _control.get_deep_mind_logger()) {
+         if (auto dm_logger = _control.get_deep_mind_logger(is_trx_transient)) {
             dm_logger->on_create_permission(p);
          }
       });
       return perm;
    }
 
-   void authorization_manager::modify_permission( const permission_object& permission, const authority& auth ) {
+   void authorization_manager::modify_permission( const permission_object& permission, const authority& auth, bool is_trx_transient ) {
       for(const key_weight& k: auth.keys)
          EOS_ASSERT(k.key.which() < _db.get<protocol_state_object>().num_supported_key_types, unactivated_key_type,
            "Unactivated key type used when modifying permission");
 
       _db.modify( permission, [&](permission_object& po) {
-         auto dm_logger = _control.get_deep_mind_logger();
+         auto dm_logger = _control.get_deep_mind_logger(is_trx_transient);
 
          std::optional<permission_object> old_permission;
          if (dm_logger) {
@@ -216,13 +218,13 @@ namespace eosio { namespace chain {
          po.auth = auth;
          po.last_updated = _control.pending_block_time();
 
-         if (auto dm_logger = _control.get_deep_mind_logger()) {
+         if (auto dm_logger = _control.get_deep_mind_logger(is_trx_transient)) {
             dm_logger->on_modify_permission(*old_permission, po);
          }
       });
    }
 
-   void authorization_manager::remove_permission( const permission_object& permission ) {
+   void authorization_manager::remove_permission( const permission_object& permission, bool is_trx_transient ) {
       const auto& index = _db.template get_index<permission_index, by_parent>();
       auto range = index.equal_range(permission.id);
       EOS_ASSERT( range.first == range.second, action_validate_exception,
@@ -230,7 +232,7 @@ namespace eosio { namespace chain {
 
       _db.get_mutable_index<permission_usage_index>().remove_object( permission.usage_id._id );
 
-      if (auto dm_logger = _control.get_deep_mind_logger()) {
+      if (auto dm_logger = _control.get_deep_mind_logger(is_trx_transient)) {
          dm_logger->on_remove_permission(permission);
       }
 
@@ -557,8 +559,8 @@ namespace eosio { namespace chain {
 
       }
 
-      if( !allow_unused_keys  || check_but_dont_fail) {
-         EOS_ASSERT( checker.all_keys_used(), tx_irrelevant_sig,
+      if( !allow_unused_keys ) {
+         EOS_ASSERT( checker.all_keys_used() || check_but_dont_fail, tx_irrelevant_sig,
                      "transaction bears irrelevant signatures from these keys: ${keys}",
                      ("keys", checker.unused_keys()) );
       }

libraries/chain/eosio_contract.cpp

@@ -54,7 +54,7 @@ void validate_authority_precondition( const apply_context& context, const author
       }
    }
 
-   if( context.trx_context.enforce_whiteblacklist && context.control.is_producing_block() ) {
+   if( context.trx_context.enforce_whiteblacklist && context.control.is_speculative_block() ) {
       for( const auto& p : auth.keys ) {
          context.control.check_key_list( p.key );
       }
@@ -65,6 +65,7 @@ void validate_authority_precondition( const apply_context& context, const author
  *  This method is called assuming precondition_system_newaccount succeeds a
  */
 void apply_eosio_newaccount(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "newaccount not allowed in read-only transaction" );
    auto create = context.get_action().data_as<newaccount>();
    try {
    context.require_authorization(create.creator);
@@ -107,18 +108,18 @@ void apply_eosio_newaccount(apply_context& context) {
    }
 
    const auto& owner_permission  = authorization.create_permission( create.name, config::owner_name, 0,
-                                                                    std::move(create.owner) );
+                                                                    std::move(create.owner), context.trx_context.is_transient() );
    const auto& active_permission = authorization.create_permission( create.name, config::active_name, owner_permission.id,
-                                                                    std::move(create.active) );
+                                                                    std::move(create.active), context.trx_context.is_transient() );
 
-   context.control.get_mutable_resource_limits_manager().initialize_account(create.name);
+   context.control.get_mutable_resource_limits_manager().initialize_account(create.name, context.trx_context.is_transient());
 
    int64_t ram_delta = config::overhead_per_account_ram_bytes;
    ram_delta += 2*config::billable_size_v<permission_object>;
    ram_delta += owner_permission.auth.get_billable_size();
    ram_delta += active_permission.auth.get_billable_size();
 
-   if (auto dm_logger = context.control.get_deep_mind_logger()) {
+   if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
       dm_logger->on_ram_trace(RAM_EVENT_ID("${name}", ("name", create.name)), "account", "add", "newaccount");
    }
 
@@ -127,6 +128,7 @@ void apply_eosio_newaccount(apply_context& context) {
 } FC_CAPTURE_AND_RETHROW( (create) ) }
 
 void apply_eosio_setcode(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "setcode not allowed in read-only transaction" );
    auto& db = context.db;
    auto  act = context.get_action().data_as<setcode>();
    context.require_authorization(act.account);
@@ -158,7 +160,7 @@ void apply_eosio_setcode(apply_context& context) {
       old_size  = (int64_t)old_code_entry.code.size() * config::setcode_ram_bytes_multiplier;
       if( old_code_entry.code_ref_count == 1 ) {
          db.remove(old_code_entry);
-         context.control.get_wasm_interface().code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
+         context.control.code_block_num_last_used(account.code_hash, account.vm_type, account.vm_version, context.control.head_block_num() + 1);
       } else {
          db.modify(old_code_entry, [](code_object& o) {
             --o.code_ref_count;
@@ -194,7 +196,7 @@ void apply_eosio_setcode(apply_context& context) {
    });
 
    if (new_size != old_size) {
-      if (auto dm_logger = context.control.get_deep_mind_logger()) {
+      if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
          const char* operation = "update";
          if (old_size <= 0) {
             operation = "add";
@@ -210,6 +212,7 @@ void apply_eosio_setcode(apply_context& context) {
 }
 
 void apply_eosio_setabi(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "setabi ot allowed in read-only transaction" );
    auto& db  = context.db;
    auto  act = context.get_action().data_as<setabi>();
 
@@ -232,7 +235,7 @@ void apply_eosio_setabi(apply_context& context) {
    });
 
    if (new_size != old_size) {
-      if (auto dm_logger = context.control.get_deep_mind_logger()) {
+      if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
          const char* operation = "update";
          if (old_size <= 0) {
             operation = "add";
@@ -248,6 +251,7 @@ void apply_eosio_setabi(apply_context& context) {
 }
 
 void apply_eosio_updateauth(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "updateauth not allowed in read-only transaction" );
 
    auto update = context.get_action().data_as<updateauth>();
    context.require_authorization(update.account); // only here to mark the single authority on this action as used
@@ -297,21 +301,21 @@ void apply_eosio_updateauth(apply_context& context) {
 
       int64_t old_size = (int64_t)(config::billable_size_v<permission_object> + permission->auth.get_billable_size());
 
-      authorization.modify_permission( *permission, update.auth );
+      authorization.modify_permission( *permission, update.auth, context.trx_context.is_transient() );
 
       int64_t new_size = (int64_t)(config::billable_size_v<permission_object> + permission->auth.get_billable_size());
 
-      if (auto dm_logger = context.control.get_deep_mind_logger()) {
+      if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", permission->id)), "auth", "update", "updateauth_update");
       }
 
       context.add_ram_usage( permission->owner, new_size - old_size );
    } else {
-      const auto& p = authorization.create_permission( update.account, update.permission, parent_id, update.auth );
+      const auto& p = authorization.create_permission( update.account, update.permission, parent_id, update.auth, context.trx_context.is_transient() );
 
       int64_t new_size = (int64_t)(config::billable_size_v<permission_object> + p.auth.get_billable_size());
 
-      if (auto dm_logger = context.control.get_deep_mind_logger()) {
+      if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
          dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", p.id)), "auth", "add", "updateauth_create");
       }
 
@@ -322,6 +326,8 @@ void apply_eosio_updateauth(apply_context& context) {
 void apply_eosio_deleteauth(apply_context& context) {
 //   context.require_write_lock( config::eosio_auth_scope );
 
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "deleteauth not allowed in read-only transaction" );
+
    auto remove = context.get_action().data_as<deleteauth>();
    context.require_authorization(remove.account); // only here to mark the single authority on this action as used
 
@@ -344,11 +350,11 @@ void apply_eosio_deleteauth(apply_context& context) {
    const auto& permission = authorization.get_permission({remove.account, remove.permission});
    int64_t old_size = config::billable_size_v<permission_object> + permission.auth.get_billable_size();
 
-   if (auto dm_logger = context.control.get_deep_mind_logger()) {
+   if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
       dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", permission.id)), "auth", "remove", "deleteauth");
    }
 
-   authorization.remove_permission( permission );
+   authorization.remove_permission( permission, context.trx_context.is_transient() );
 
    context.add_ram_usage( remove.account, -old_size );
 
@@ -357,6 +363,8 @@ void apply_eosio_deleteauth(apply_context& context) {
 void apply_eosio_linkauth(apply_context& context) {
 //   context.require_write_lock( config::eosio_auth_scope );
 
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "linkauth not allowed in read-only transaction" );
+
    auto requirement = context.get_action().data_as<linkauth>();
    try {
       EOS_ASSERT(!requirement.requirement.empty(), action_validate_exception, "Required permission cannot be empty");
@@ -401,7 +409,7 @@ void apply_eosio_linkauth(apply_context& context) {
             link.required_permission = requirement.requirement;
          });
 
-         if (auto dm_logger = context.control.get_deep_mind_logger()) {
+         if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
             dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", l.id)), "auth_link", "add", "linkauth");
          }
 
@@ -417,6 +425,8 @@ void apply_eosio_linkauth(apply_context& context) {
 void apply_eosio_unlinkauth(apply_context& context) {
 //   context.require_write_lock( config::eosio_auth_scope );
 
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "unlinkauth not allowed in read-only transaction" );
+
    auto& db = context.db;
    auto unlink = context.get_action().data_as<unlinkauth>();
 
@@ -426,7 +436,7 @@ void apply_eosio_unlinkauth(apply_context& context) {
    auto link = db.find<permission_link_object, by_action_name>(link_key);
    EOS_ASSERT(link != nullptr, action_validate_exception, "Attempting to unlink authority, but no link found");
 
-   if (auto dm_logger = context.control.get_deep_mind_logger()) {
+   if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
       dm_logger->on_ram_trace(RAM_EVENT_ID("${id}", ("id", link->id)), "auth_link", "remove", "unlinkauth");
    }
 
@@ -439,6 +449,7 @@ void apply_eosio_unlinkauth(apply_context& context) {
 }
 
 void apply_eosio_canceldelay(apply_context& context) {
+   EOS_ASSERT( !context.trx_context.is_read_only(), action_validate_exception, "canceldelay not allowed in read-only transaction" );
    auto cancel = context.get_action().data_as<canceldelay>();
    context.require_authorization(cancel.canceling_auth.actor); // only here to mark the single authority on this action as used
 

libraries/chain/fork_database.cpp

@@ -9,6 +9,7 @@
 #include <boost/multi_index/composite_key.hpp>
 #include <fc/io/fstream.hpp>
 #include <fstream>
+#include <shared_mutex>
 
 namespace eosio { namespace chain {
    using boost::multi_index_container;
@@ -60,27 +61,44 @@ namespace eosio { namespace chain {
    }
 
    struct fork_database_impl {
-      fork_database_impl( fork_database& self, const fc::path& data_dir )
-      :self(self)
-      ,datadir(data_dir)
+      explicit fork_database_impl( const std::filesystem::path& data_dir )
+      :datadir(data_dir)
       {}
 
-      fork_database&        self;
+      std::shared_mutex     mtx;
       fork_multi_index_type index;
       block_state_ptr       root; // Only uses the block_header_state portion
       block_state_ptr       head;
-      fc::path              datadir;
+      std::filesystem::path datadir;
 
-      void add( const block_state_ptr& n,
-                bool ignore_duplicate, bool validate,
-                const std::function<void( block_timestamp_type,
-                                          const flat_set<digest_type>&,
-                                          const vector<digest_type>& )>& validator );
+      void open_impl( const std::function<void( block_timestamp_type,
+                                                const flat_set<digest_type>&,
+                                                const vector<digest_type>& )>& validator );
+      void close_impl();
+
+
+      block_header_state_ptr  get_block_header_impl( const block_id_type& id )const;
+      block_state_ptr         get_block_impl( const block_id_type& id )const;
+      void            reset_impl( const block_header_state& root_bhs );
+      void            rollback_head_to_root_impl();
+      void            advance_root_impl( const block_id_type& id );
+      void            remove_impl( const block_id_type& id );
+      branch_type     fetch_branch_impl( const block_id_type& h, uint32_t trim_after_block_num )const;
+      block_state_ptr search_on_branch_impl( const block_id_type& h, uint32_t block_num )const;
+      pair<branch_type, branch_type> fetch_branch_from_impl( const block_id_type& first,
+                                                             const block_id_type& second )const;
+      void mark_valid_impl( const block_state_ptr& h );
+
+      void add_impl( const block_state_ptr& n,
+                     bool ignore_duplicate, bool validate,
+                     const std::function<void( block_timestamp_type,
+                                               const flat_set<digest_type>&,
+                                               const vector<digest_type>& )>& validator );
    };
 
 
-   fork_database::fork_database( const fc::path& data_dir )
-   :my( new fork_database_impl( *this, data_dir ) )
+   fork_database::fork_database( const std::filesystem::path& data_dir )
+   :my( new fork_database_impl( data_dir ) )
    {}
 
 
@@ -88,11 +106,19 @@ namespace eosio { namespace chain {
                                                        const flat_set<digest_type>&,
                                                        const vector<digest_type>& )>& validator )
    {
-      if (!fc::is_directory(my->datadir))
-         fc::create_directories(my->datadir);
+      std::lock_guard g( my->mtx );
+      my->open_impl( validator );
+   }
 
-      auto fork_db_dat = my->datadir / config::forkdb_filename;
-      if( fc::exists( fork_db_dat ) ) {
+   void fork_database_impl::open_impl( const std::function<void( block_timestamp_type,
+                                                                 const flat_set<digest_type>&,
+                                                                 const vector<digest_type>& )>& validator )
+   {
+      if (!std::filesystem::is_directory(datadir))
+         std::filesystem::create_directories(datadir);
+
+      auto fork_db_dat = datadir / config::forkdb_filename;
+      if( std::filesystem::exists( fork_db_dat ) ) {
          try {
             string content;
             fc::read_file_contents( fork_db_dat, content );
@@ -102,29 +128,29 @@ namespace eosio { namespace chain {
             // validate totem
             uint32_t totem = 0;
             fc::raw::unpack( ds, totem );
-            EOS_ASSERT( totem == magic_number, fork_database_exception,
+            EOS_ASSERT( totem == fork_database::magic_number, fork_database_exception,
                         "Fork database file '${filename}' has unexpected magic number: ${actual_totem}. Expected ${expected_totem}",
-                        ("filename", fork_db_dat.generic_string())
+                        ("filename", fork_db_dat)
                         ("actual_totem", totem)
-                        ("expected_totem", magic_number)
+                        ("expected_totem", fork_database::magic_number)
             );
 
             // validate version
             uint32_t version = 0;
             fc::raw::unpack( ds, version );
-            EOS_ASSERT( version >= min_supported_version && version <= max_supported_version,
+            EOS_ASSERT( version >= fork_database::min_supported_version && version <= fork_database::max_supported_version,
                         fork_database_exception,
                        "Unsupported version of fork database file '${filename}'. "
                        "Fork database version is ${version} while code supports version(s) [${min},${max}]",
-                       ("filename", fork_db_dat.generic_string())
+                       ("filename", fork_db_dat)
                        ("version", version)
-                       ("min", min_supported_version)
-                       ("max", max_supported_version)
+                       ("min", fork_database::min_supported_version)
+                       ("max", fork_database::max_supported_version)
             );
 
             block_header_state bhs;
             fc::raw::unpack( ds, bhs );
-            reset( bhs );
+            reset_impl( bhs );
 
             unsigned_int size; fc::raw::unpack( ds, size );
             for( uint32_t i = 0, n = size.value; i < n; ++i ) {
@@ -132,55 +158,60 @@ namespace eosio { namespace chain {
                fc::raw::unpack( ds, s );
                // do not populate transaction_metadatas, they will be created as needed in apply_block with appropriate key recovery
                s.header_exts = s.block->validate_and_extract_header_extensions();
-               my->add( std::make_shared<block_state>( move( s ) ), false, true, validator );
+               add_impl( std::make_shared<block_state>( std::move( s ) ), false, true, validator );
             }
             block_id_type head_id;
             fc::raw::unpack( ds, head_id );
 
-            if( my->root->id == head_id ) {
-               my->head = my->root;
+            if( root->id == head_id ) {
+               head = root;
             } else {
-               my->head = get_block( head_id );
-               EOS_ASSERT( my->head, fork_database_exception,
+               head = get_block_impl( head_id );
+               EOS_ASSERT( head, fork_database_exception,
                            "could not find head while reconstructing fork database from file; '${filename}' is likely corrupted",
-                           ("filename", fork_db_dat.generic_string()) );
+                           ("filename", fork_db_dat) );
             }
 
-            auto candidate = my->index.get<by_lib_block_num>().begin();
-            if( candidate == my->index.get<by_lib_block_num>().end() || !(*candidate)->is_valid() ) {
-               EOS_ASSERT( my->head->id == my->root->id, fork_database_exception,
+            auto candidate = index.get<by_lib_block_num>().begin();
+            if( candidate == index.get<by_lib_block_num>().end() || !(*candidate)->is_valid() ) {
+               EOS_ASSERT( head->id == root->id, fork_database_exception,
                            "head not set to root despite no better option available; '${filename}' is likely corrupted",
-                           ("filename", fork_db_dat.generic_string()) );
+                           ("filename", fork_db_dat) );
             } else {
-               EOS_ASSERT( !first_preferred( **candidate, *my->head ), fork_database_exception,
+               EOS_ASSERT( !first_preferred( **candidate, *head ), fork_database_exception,
                            "head not set to best available option available; '${filename}' is likely corrupted",
-                           ("filename", fork_db_dat.generic_string()) );
+                           ("filename", fork_db_dat) );
             }
          } FC_CAPTURE_AND_RETHROW( (fork_db_dat) )
 
-         fc::remove( fork_db_dat );
+         std::filesystem::remove( fork_db_dat );
       }
    }
 
    void fork_database::close() {
-      auto fork_db_dat = my->datadir / config::forkdb_filename;
+      std::lock_guard g( my->mtx );
+      my->close_impl();
+   }
 
-      if( !my->root ) {
-         if( my->index.size() > 0 ) {
+   void fork_database_impl::close_impl() {
+      auto fork_db_dat = datadir / config::forkdb_filename;
+
+      if( !root ) {
+         if( index.size() > 0 ) {
             elog( "fork_database is in a bad state when closing; not writing out '${filename}'",
-                  ("filename", fork_db_dat.generic_string()) );
+                  ("filename", fork_db_dat) );
          }
          return;
       }
 
       std::ofstream out( fork_db_dat.generic_string().c_str(), std::ios::out | std::ios::binary | std::ofstream::trunc );
-      fc::raw::pack( out, magic_number );
-      fc::raw::pack( out, max_supported_version ); // write out current version which is always max_supported_version
-      fc::raw::pack( out, *static_cast<block_header_state*>(&*my->root) );
-      uint32_t num_blocks_in_fork_db = my->index.size();
+      fc::raw::pack( out, fork_database::magic_number );
+      fc::raw::pack( out, fork_database::max_supported_version ); // write out current version which is always max_supported_version
+      fc::raw::pack( out, *static_cast<block_header_state*>(&*root) );
+      uint32_t num_blocks_in_fork_db = index.size();
       fc::raw::pack( out, unsigned_int{num_blocks_in_fork_db} );
 
-      const auto& indx = my->index.get<by_lib_block_num>();
+      const auto& indx = index.get<by_lib_block_num>();
 
       auto unvalidated_itr = indx.rbegin();
       auto unvalidated_end = boost::make_reverse_iterator( indx.lower_bound( false ) );
@@ -215,30 +246,40 @@ namespace eosio { namespace chain {
          fc::raw::pack( out, *(*itr) );
       }
 
-      if( my->head ) {
-         fc::raw::pack( out, my->head->id );
+      if( head ) {
+         fc::raw::pack( out, head->id );
       } else {
          elog( "head not set in fork database; '${filename}' will be corrupted",
-               ("filename", fork_db_dat.generic_string()) );
+               ("filename", fork_db_dat) );
       }
 
-      my->index.clear();
+      index.clear();
    }
 
    fork_database::~fork_database() {
-      close();
+      my->close_impl();
    }
 
    void fork_database::reset( const block_header_state& root_bhs ) {
-      my->index.clear();
-      my->root = std::make_shared<block_state>();
-      static_cast<block_header_state&>(*my->root) = root_bhs;
-      my->root->validated = true;
-      my->head = my->root;
+      std::lock_guard g( my->mtx );
+      my->reset_impl(root_bhs);
+   }
+
+   void fork_database_impl::reset_impl( const block_header_state& root_bhs ) {
+      index.clear();
+      root = std::make_shared<block_state>();
+      static_cast<block_header_state&>(*root) = root_bhs;
+      root->validated = true;
+      head = root;
    }
 
    void fork_database::rollback_head_to_root() {
-      auto& by_id_idx = my->index.get<by_block_id>();
+      std::lock_guard g( my->mtx );
+      my->rollback_head_to_root_impl();
+   }
+
+   void fork_database_impl::rollback_head_to_root_impl() {
+      auto& by_id_idx = index.get<by_block_id>();
       auto itr = by_id_idx.begin();
       while (itr != by_id_idx.end()) {
          by_id_idx.modify( itr, [&]( block_state_ptr& bsp ) {
@@ -246,13 +287,18 @@ namespace eosio { namespace chain {
          } );
          ++itr;
       }
-      my->head = my->root;
+      head = root;
    }
 
    void fork_database::advance_root( const block_id_type& id ) {
-      EOS_ASSERT( my->root, fork_database_exception, "root not yet set" );
+      std::lock_guard g( my->mtx );
+      my->advance_root_impl( id );
+   }
 
-      auto new_root = get_block( id );
+   void fork_database_impl::advance_root_impl( const block_id_type& id ) {
+      EOS_ASSERT( root, fork_database_exception, "root not yet set" );
+
+      auto new_root = get_block_impl( id );
       EOS_ASSERT( new_root, fork_database_exception,
                   "cannot advance root to a block that does not exist in the fork database" );
       EOS_ASSERT( new_root->is_valid(), fork_database_exception,
@@ -262,48 +308,53 @@ namespace eosio { namespace chain {
       deque<block_id_type> blocks_to_remove;
       for( auto b = new_root; b; ) {
          blocks_to_remove.emplace_back( b->header.previous );
-         b = get_block( blocks_to_remove.back() );
-         EOS_ASSERT( b || blocks_to_remove.back() == my->root->id, fork_database_exception, "invariant violation: orphaned branch was present in forked database" );
+         b = get_block_impl( blocks_to_remove.back() );
+         EOS_ASSERT( b || blocks_to_remove.back() == root->id, fork_database_exception, "invariant violation: orphaned branch was present in forked database" );
       }
 
       // The new root block should be erased from the fork database index individually rather than with the remove method,
       // because we do not want the blocks branching off of it to be removed from the fork database.
-      my->index.erase( my->index.find( id ) );
+      index.erase( index.find( id ) );
 
       // The other blocks to be removed are removed using the remove method so that orphaned branches do not remain in the fork database.
       for( const auto& block_id : blocks_to_remove ) {
-         remove( block_id );
+         remove_impl( block_id );
       }
 
       // Even though fork database no longer needs block or trxs when a block state becomes a root of the tree,
       // avoid mutating the block state at all, for example clearing the block shared pointer, because other
       // parts of the code which run asynchronously may later expect it remain unmodified.
 
-      my->root = new_root;
+      root = new_root;
    }
 
    block_header_state_ptr fork_database::get_block_header( const block_id_type& id )const {
-      if( my->root->id == id ) {
-         return my->root;
+      std::shared_lock g( my->mtx );
+      return my->get_block_header_impl( id );
+   }
+
+   block_header_state_ptr fork_database_impl::get_block_header_impl( const block_id_type& id )const {
+      if( root->id == id ) {
+         return root;
       }
 
-      auto itr = my->index.find( id );
-      if( itr != my->index.end() )
+      auto itr = index.find( id );
+      if( itr != index.end() )
          return *itr;
 
       return block_header_state_ptr();
    }
 
-   void fork_database_impl::add( const block_state_ptr& n,
-                                 bool ignore_duplicate, bool validate,
-                                 const std::function<void( block_timestamp_type,
-                                                           const flat_set<digest_type>&,
-                                                           const vector<digest_type>& )>& validator )
+   void fork_database_impl::add_impl( const block_state_ptr& n,
+                                      bool ignore_duplicate, bool validate,
+                                      const std::function<void( block_timestamp_type,
+                                                                const flat_set<digest_type>&,
+                                                                const vector<digest_type>& )>& validator )
    {
       EOS_ASSERT( root, fork_database_exception, "root not yet set" );
       EOS_ASSERT( n, fork_database_exception, "attempt to add null block state" );
 
-      auto prev_bh = self.get_block_header( n->header.previous );
+      auto prev_bh = get_block_header_impl( n->header.previous );
 
       EOS_ASSERT( prev_bh, unlinkable_block_exception,
                   "unlinkable block", ("id", n->id)("previous", n->header.previous) );
@@ -332,19 +383,27 @@ namespace eosio { namespace chain {
    }
 
    void fork_database::add( const block_state_ptr& n, bool ignore_duplicate ) {
-      my->add( n, ignore_duplicate, false,
-               []( block_timestamp_type timestamp,
-                   const flat_set<digest_type>& cur_features,
-                   const vector<digest_type>& new_features )
-               {}
+      std::lock_guard g( my->mtx );
+      my->add_impl( n, ignore_duplicate, false,
+                    []( block_timestamp_type timestamp,
+                        const flat_set<digest_type>& cur_features,
+                        const vector<digest_type>& new_features )
+                    {}
       );
    }
 
-   const block_state_ptr& fork_database::root()const { return my->root; }
+   block_state_ptr fork_database::root()const {
+      std::shared_lock g( my->mtx );
+      return my->root;
+   }
 
-   const block_state_ptr& fork_database::head()const { return my->head; }
+   block_state_ptr fork_database::head()const {
+      std::shared_lock g( my->mtx );
+      return my->head;
+   }
 
    block_state_ptr fork_database::pending_head()const {
+      std::shared_lock g( my->mtx );
       const auto& indx = my->index.get<by_lib_block_num>();
 
       auto itr = indx.lower_bound( false );
@@ -357,8 +416,13 @@ namespace eosio { namespace chain {
    }
 
    branch_type fork_database::fetch_branch( const block_id_type& h, uint32_t trim_after_block_num )const {
+      std::shared_lock g( my->mtx );
+      return my->fetch_branch_impl( h, trim_after_block_num );
+   }
+
+   branch_type fork_database_impl::fetch_branch_impl( const block_id_type& h, uint32_t trim_after_block_num )const {
       branch_type result;
-      for( auto s = get_block(h); s; s = get_block( s->header.previous ) ) {
+      for( auto s = get_block_impl(h); s; s = get_block_impl( s->header.previous ) ) {
          if( s->block_num <= trim_after_block_num )
              result.push_back( s );
       }
@@ -367,7 +431,12 @@ namespace eosio { namespace chain {
    }
 
    block_state_ptr fork_database::search_on_branch( const block_id_type& h, uint32_t block_num )const {
-      for( auto s = get_block(h); s; s = get_block( s->header.previous ) ) {
+      std::shared_lock g( my->mtx );
+      return my->search_on_branch_impl( h, block_num );
+   }
+
+   block_state_ptr fork_database_impl::search_on_branch_impl( const block_id_type& h, uint32_t block_num )const {
+      for( auto s = get_block_impl(h); s; s = get_block_impl( s->header.previous ) ) {
          if( s->block_num == block_num )
              return s;
       }
@@ -381,9 +450,15 @@ namespace eosio { namespace chain {
     */
    pair< branch_type, branch_type >  fork_database::fetch_branch_from( const block_id_type& first,
                                                                        const block_id_type& second )const {
+      std::shared_lock g( my->mtx );
+      return my->fetch_branch_from_impl( first, second );
+   }
+
+   pair< branch_type, branch_type >  fork_database_impl::fetch_branch_from_impl( const block_id_type& first,
+                                                                                 const block_id_type& second )const {
       pair<branch_type,branch_type> result;
-      auto first_branch = (first == my->root->id) ? my->root : get_block(first);
-      auto second_branch = (second == my->root->id) ? my->root : get_block(second);
+      auto first_branch = (first == root->id) ? root : get_block_impl(first);
+      auto second_branch = (second == root->id) ? root : get_block_impl(second);
 
       EOS_ASSERT(first_branch, fork_db_block_not_found, "block ${id} does not exist", ("id", first));
       EOS_ASSERT(second_branch, fork_db_block_not_found, "block ${id} does not exist", ("id", second));
@@ -392,7 +467,7 @@ namespace eosio { namespace chain {
       {
          result.first.push_back(first_branch);
          const auto& prev = first_branch->header.previous;
-         first_branch = (prev == my->root->id) ? my->root : get_block( prev );
+         first_branch = (prev == root->id) ? root : get_block_impl( prev );
          EOS_ASSERT( first_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", prev)
@@ -403,7 +478,7 @@ namespace eosio { namespace chain {
       {
          result.second.push_back( second_branch );
          const auto& prev = second_branch->header.previous;
-         second_branch = (prev == my->root->id) ? my->root : get_block( prev );
+         second_branch = (prev == root->id) ? root : get_block_impl( prev );
          EOS_ASSERT( second_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", prev)
@@ -417,9 +492,9 @@ namespace eosio { namespace chain {
          result.first.push_back(first_branch);
          result.second.push_back(second_branch);
          const auto &first_prev = first_branch->header.previous;
-         first_branch = get_block( first_prev );
+         first_branch = get_block_impl( first_prev );
          const auto &second_prev = second_branch->header.previous;
-         second_branch = get_block( second_prev );
+         second_branch = get_block_impl( second_prev );
          EOS_ASSERT( first_branch, fork_db_block_not_found,
                      "block ${id} does not exist",
                      ("id", first_prev)
@@ -436,13 +511,18 @@ namespace eosio { namespace chain {
          result.second.push_back(second_branch);
       }
       return result;
-   } /// fetch_branch_from
+   } /// fetch_branch_from_impl
 
    /// remove all of the invalid forks built off of this id including this id
    void fork_database::remove( const block_id_type& id ) {
+      std::lock_guard g( my->mtx );
+      return my->remove_impl( id );
+   }
+
+   void fork_database_impl::remove_impl( const block_id_type& id ) {
       deque<block_id_type> remove_queue{id};
-      const auto& previdx = my->index.get<by_prev>();
-      const auto& head_id = my->head->id;
+      const auto& previdx = index.get<by_prev>();
+      const auto& head_id = head->id;
 
       for( uint32_t i = 0; i < remove_queue.size(); ++i ) {
          EOS_ASSERT( remove_queue[i] != head_id, fork_database_exception,
@@ -456,16 +536,19 @@ namespace eosio { namespace chain {
       }
 
       for( const auto& block_id : remove_queue ) {
-         auto itr = my->index.find( block_id );
-         if( itr != my->index.end() )
-            my->index.erase(itr);
+         index.erase( block_id );
       }
    }
 
    void fork_database::mark_valid( const block_state_ptr& h ) {
+      std::lock_guard g( my->mtx );
+      my->mark_valid_impl( h );
+   }
+
+   void fork_database_impl::mark_valid_impl( const block_state_ptr& h ) {
       if( h->validated ) return;
 
-      auto& by_id_idx = my->index.get<by_block_id>();
+      auto& by_id_idx = index.get<by_block_id>();
 
       auto itr = by_id_idx.find( h->id );
       EOS_ASSERT( itr != by_id_idx.end(), fork_database_exception,
@@ -476,15 +559,20 @@ namespace eosio { namespace chain {
          bsp->validated = true;
       } );
 
-      auto candidate = my->index.get<by_lib_block_num>().begin();
-      if( first_preferred( **candidate, *my->head ) ) {
-         my->head = *candidate;
+      auto candidate = index.get<by_lib_block_num>().begin();
+      if( first_preferred( **candidate, *head ) ) {
+         head = *candidate;
       }
    }
 
-   block_state_ptr   fork_database::get_block(const block_id_type& id)const {
-      auto itr = my->index.find( id );
-      if( itr != my->index.end() )
+   block_state_ptr fork_database::get_block(const block_id_type& id)const {
+      std::shared_lock g( my->mtx );
+      return my->get_block_impl(id);
+   }
+
+   block_state_ptr fork_database_impl::get_block_impl(const block_id_type& id)const {
+      auto itr = index.find( id );
+      if( itr != index.end() )
          return *itr;
       return block_state_ptr();
    }

libraries/chain/include/eosio/chain/abi_serializer.hpp

@@ -6,8 +6,9 @@
 #include <utility>
 #include <fc/variant_object.hpp>
 #include <fc/scoped_exit.hpp>
+#include <fc/time.hpp>
 
-namespace eosio { namespace chain {
+namespace eosio::chain {
 
 using std::map;
 using std::string;
@@ -23,6 +24,7 @@ namespace impl {
    struct abi_traverse_context_with_path;
    struct binary_to_variant_context;
    struct variant_to_binary_context;
+   struct action_data_to_variant_context;
 }
 
 /**
@@ -35,10 +37,10 @@ struct abi_serializer {
    using yield_function_t = fc::optional_delegate<void(size_t)>;
 
    abi_serializer(){ configure_built_in_types(); }
-   abi_serializer( const abi_def& abi, const yield_function_t& yield );
+   abi_serializer( abi_def abi, const yield_function_t& yield );
    [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
    abi_serializer( const abi_def& abi, const fc::microseconds& max_serialization_time );
-   void set_abi( const abi_def& abi, const yield_function_t& yield );
+   void set_abi( abi_def abi, const yield_function_t& yield );
    [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
    void set_abi(const abi_def& abi, const fc::microseconds& max_serialization_time);
 
@@ -48,7 +50,6 @@ struct abi_serializer {
    bool      is_szarray(const std::string_view& type)const;
    bool      is_optional(const std::string_view& type)const;
    bool      is_type( const std::string_view& type, const yield_function_t& yield )const;
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
    bool      is_type(const std::string_view& type, const fc::microseconds& max_serialization_time)const;
    bool      is_builtin_type(const std::string_view& type)const;
    bool      is_integer(const std::string_view& type) const;
@@ -67,33 +68,29 @@ struct abi_serializer {
    std::optional<string>  get_error_message( uint64_t error_code )const;
 
    fc::variant binary_to_variant( const std::string_view& type, const bytes& binary, const yield_function_t& yield, bool short_path = false )const;
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   fc::variant binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const;
+   fc::variant binary_to_variant( const std::string_view& type, const bytes& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const;
    fc::variant binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const yield_function_t& yield, bool short_path = false )const;
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   fc::variant binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const;
+   fc::variant binary_to_variant( const std::string_view& type, fc::datastream<const char*>& binary, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const;
 
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   bytes       variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path = false )const;
+   bytes       variant_to_binary( const std::string_view& type, const fc::variant& var, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const;
    bytes       variant_to_binary( const std::string_view& type, const fc::variant& var, const yield_function_t& yield, bool short_path = false )const;
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   void        variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const fc::microseconds& max_serialization_time, bool short_path = false )const;
+   void        variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const fc::microseconds& max_action_data_serialization_time, bool short_path = false )const;
    void        variant_to_binary( const std::string_view& type, const fc::variant& var, fc::datastream<char*>& ds, const yield_function_t& yield, bool short_path = false )const;
 
    template<typename T, typename Resolver>
-   static void to_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield );
+   static void to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield );
    template<typename T, typename Resolver>
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   static void to_variant( const T& o, fc::variant& vo, Resolver resolver, const fc::microseconds& max_serialization_time );
+   static void to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time );
 
    template<typename T, typename Resolver>
-   static void to_log_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield );
+   static void to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield );
+   template<typename T, typename Resolver>
+   static void to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time );
 
    template<typename T, typename Resolver>
-   static void from_variant( const fc::variant& v, T& o, Resolver resolver, const yield_function_t& yield );
+   static void from_variant( const fc::variant& v, T& o, const Resolver& resolver, const yield_function_t& yield );
    template<typename T, typename Resolver>
-   [[deprecated("use the overload with yield_function_t[=create_yield_function(max_serialization_time)]")]]
-   static void from_variant( const fc::variant& v, T& o, Resolver resolver, const fc::microseconds& max_serialization_time );
+   static void from_variant( const fc::variant& v, T& o, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time );
 
    template<typename Vec>
    static bool is_empty_abi(const Vec& abi_vec)
@@ -120,14 +117,10 @@ struct abi_serializer {
    static constexpr size_t max_recursion_depth = 32; // arbitrary depth to prevent infinite recursion
 
    // create standard yield function that checks for max_serialization_time and max_recursion_depth.
-   // now() deadline caputered at time of this call
+   // restricts serialization time from creation of yield function until serialization is complete.
+   // now() deadline captured at time of this call
    static yield_function_t create_yield_function(const fc::microseconds& max_serialization_time) {
-      fc::time_point deadline = fc::time_point::now();
-      if( max_serialization_time > fc::microseconds::maximum() - deadline.time_since_epoch() ) {
-         deadline = fc::time_point::maximum();
-      } else {
-         deadline += max_serialization_time;
-      }
+      fc::time_point deadline = fc::time_point::now().safe_add(max_serialization_time);
       return [max_serialization_time, deadline](size_t recursion_depth) {
          EOS_ASSERT( recursion_depth < max_recursion_depth, abi_recursion_depth_exception,
                      "recursive definition, max_recursion_depth ${r} ", ("r", max_recursion_depth) );
@@ -137,6 +130,13 @@ struct abi_serializer {
       };
    }
 
+   static yield_function_t create_depth_yield_function() {
+      return [](size_t recursion_depth) {
+         EOS_ASSERT( recursion_depth < max_recursion_depth, abi_recursion_depth_exception,
+                     "recursive definition, max_recursion_depth ${r} ", ("r", max_recursion_depth) );
+      };
+   }
+
 private:
 
    map<type_name, type_name, std::less<>>     typedefs;
@@ -172,8 +172,9 @@ private:
 namespace impl {
    const static size_t hex_log_max_size = 64;
    struct abi_traverse_context {
-      explicit abi_traverse_context( abi_serializer::yield_function_t yield )
+      abi_traverse_context( abi_serializer::yield_function_t yield, fc::microseconds max_action_data_serialization )
       : yield(std::move( yield ))
+      , max_action_serialization_time(max_action_data_serialization)
       {
       }
 
@@ -187,7 +188,9 @@ namespace impl {
 
    protected:
       abi_serializer::yield_function_t  yield;
-      size_t                            recursion_depth = 0;
+      // if set then restricts each individual action data serialization
+      fc::microseconds                  max_action_serialization_time;
+      size_t                            recursion_depth = 1;
       bool                              log = false;
    };
 
@@ -226,8 +229,8 @@ namespace impl {
    using path_item = std::variant<empty_path_item, array_index_path_item, field_path_item, variant_path_item>;
 
    struct abi_traverse_context_with_path : public abi_traverse_context {
-      abi_traverse_context_with_path( const abi_serializer& abis, abi_serializer::yield_function_t yield, const std::string_view& type )
-      : abi_traverse_context( std::move( yield ) ), abis(abis)
+      abi_traverse_context_with_path( const abi_serializer& abis, abi_serializer::yield_function_t yield, fc::microseconds max_action_data_serialization_time, const std::string_view& type )
+      : abi_traverse_context( std::move( yield ), max_action_data_serialization_time ), abis(abis)
       {
          set_path_root(type);
       }
@@ -263,6 +266,22 @@ namespace impl {
       using abi_traverse_context_with_path::abi_traverse_context_with_path;
    };
 
+   struct action_data_to_variant_context : public binary_to_variant_context {
+      action_data_to_variant_context( const abi_serializer& abis, const abi_traverse_context& ctx, const std::string_view& type )
+            : binary_to_variant_context(abis, ctx, type)
+      {
+         short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place
+         if (max_action_serialization_time.count() > 0) {
+            fc::time_point deadline = fc::time_point::now().safe_add(max_action_serialization_time);
+            yield = [deadline, y=yield, max=max_action_serialization_time](size_t depth) {
+               y(depth); // call provided yield that might include an overall time limit or not
+               EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception,
+                           "serialization action data time limit ${t}us exceeded", ("t", max) );
+            };
+         }
+      }
+   };
+
    struct variant_to_binary_context : public abi_traverse_context_with_path {
       using abi_traverse_context_with_path::abi_traverse_context_with_path;
 
@@ -335,7 +354,7 @@ namespace impl {
        * and can be degraded to the normal ::to_variant(...) processing
        */
       template<typename M, typename Resolver, not_require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const M& v, Resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver&, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          mvo(name,v);
@@ -346,14 +365,14 @@ namespace impl {
        * for these types we create new ABI aware visitors
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const M& v, Resolver resolver, abi_traverse_context& ctx );
+      static void add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver& resolver, abi_traverse_context& ctx );
 
       /**
        * template which overloads add for vectors of types which contain ABI information in their trees
        * for these members we call ::add in order to trigger further processing
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const vector<M>& v, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const vector<M>& v, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          vector<fc::variant> array;
@@ -372,7 +391,7 @@ namespace impl {
       * for these members we call ::add in order to trigger further processing
       */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const deque<M>& v, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const deque<M>& v, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          deque<fc::variant> array;
@@ -389,7 +408,7 @@ namespace impl {
        * for these members we call ::add in order to trigger further processing
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void add( mutable_variant_object &mvo, const char* name, const std::shared_ptr<M>& v, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const std::shared_ptr<M>& v, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          if( !v ) return;
@@ -402,10 +421,10 @@ namespace impl {
       struct add_static_variant
       {
          mutable_variant_object& obj_mvo;
-         Resolver& resolver;
+         const Resolver& resolver;
          abi_traverse_context& ctx;
 
-         add_static_variant( mutable_variant_object& o, Resolver& r, abi_traverse_context& ctx )
+         add_static_variant( mutable_variant_object& o, const Resolver& r, abi_traverse_context& ctx )
                :obj_mvo(o), resolver(r), ctx(ctx) {}
 
          typedef void result_type;
@@ -416,7 +435,7 @@ namespace impl {
       };
 
       template<typename Resolver, typename... Args>
-      static void add( mutable_variant_object &mvo, const char* name, const std::variant<Args...>& v, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &mvo, const char* name, const std::variant<Args...>& v, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          mutable_variant_object obj_mvo;
@@ -426,7 +445,7 @@ namespace impl {
       }
 
       template<typename Resolver>
-      static bool add_special_logging( mutable_variant_object& mvo, const char* name, const action& act, Resolver& resolver, abi_traverse_context& ctx ) {
+      static bool add_special_logging( mutable_variant_object& mvo, const char* name, const action& act, const Resolver& resolver, abi_traverse_context& ctx ) {
          if( !ctx.is_logging() ) return false;
 
          try {
@@ -455,7 +474,7 @@ namespace impl {
        * @return
        */
       template<typename Resolver>
-      static void add( mutable_variant_object &out, const char* name, const action& act, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &out, const char* name, const action& act, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<action>::total_member_count == 4);
          auto h = ctx.enter_scope();
@@ -485,16 +504,16 @@ namespace impl {
          };
 
          try {
-            auto abi = resolver(act.account);
-            if (abi) {
-               auto type = abi->get_action_type(act.name);
+            auto abi_optional = resolver(act.account);
+            if (abi_optional) {
+               const abi_serializer& abi = *abi_optional;
+               auto type = abi.get_action_type(act.name);
                if (!type.empty()) {
                   try {
-                     binary_to_variant_context _ctx(*abi, ctx, type);
-                     _ctx.short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place
-                     mvo( "data", abi->_binary_to_variant( type, act.data, _ctx ));
+                     action_data_to_variant_context _ctx(abi, ctx, type);
+                     mvo( "data", abi._binary_to_variant( type, act.data, _ctx ));
                   } catch(...) {
-                     // any failure to serialize data, then leave as not serailzed
+                     // any failure to serialize data, then leave as not serialized
                      set_hex_data(mvo, "data", act.data);
                   }
                } else {
@@ -520,7 +539,7 @@ namespace impl {
        * @return
        */
       template<typename Resolver>
-      static void add( mutable_variant_object& out, const char* name, const action_trace& act_trace, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object& out, const char* name, const action_trace& act_trace, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<action_trace>::total_member_count == 17);
          auto h = ctx.enter_scope();
@@ -546,13 +565,13 @@ namespace impl {
          mvo("return_value_hex_data", act_trace.return_value);
          auto act = act_trace.act;
          try {
-            auto abi = resolver(act.account);
-            if (abi) {
-               auto type = abi->get_action_result_type(act.name);
+            auto abi_optional = resolver(act.account);
+            if (abi_optional) {
+               const abi_serializer& abi = *abi_optional;
+               auto type = abi.get_action_result_type(act.name);
                if (!type.empty()) {
-                  binary_to_variant_context _ctx(*abi, ctx, type);
-                  _ctx.short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place
-                  mvo( "return_value_data", abi->_binary_to_variant( type, act_trace.return_value, _ctx ));
+                  action_data_to_variant_context _ctx(abi, ctx, type);
+                  mvo( "return_value_data", abi._binary_to_variant( type, act_trace.return_value, _ctx ));
                }
             }
          } catch(...) {}
@@ -569,7 +588,7 @@ namespace impl {
        * @return
        */
       template<typename Resolver>
-      static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<packed_transaction>::total_member_count == 4);
          auto h = ctx.enter_scope();
@@ -593,7 +612,7 @@ namespace impl {
        * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of trx.transaction_extensions
        */
       template<typename Resolver>
-      static void add( mutable_variant_object &out, const char* name, const transaction& trx, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &out, const char* name, const transaction& trx, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<transaction>::total_member_count == 9);
          auto h = ctx.enter_scope();
@@ -624,7 +643,7 @@ namespace impl {
        * block.header_extensions and block.block_extensions
        */
       template<typename Resolver>
-      static void add( mutable_variant_object &out, const char* name, const signed_block& block, Resolver resolver, abi_traverse_context& ctx )
+      static void add( mutable_variant_object &out, const char* name, const signed_block& block, const Resolver& resolver, abi_traverse_context& ctx )
       {
          static_assert(fc::reflector<signed_block>::total_member_count == 12);
          auto h = ctx.enter_scope();
@@ -648,7 +667,7 @@ namespace impl {
             for (auto feature : new_protocol_features) {
                mutable_variant_object feature_mvo;
                add(feature_mvo, "feature_digest", feature, resolver, ctx);
-               pf_array.push_back(feature_mvo);
+               pf_array.push_back(std::move(feature_mvo));
             }
             mvo("new_protocol_features", pf_array);
          }
@@ -678,13 +697,13 @@ namespace impl {
     * this will degrade to the common fc::to_variant as soon as the type no longer contains
     * ABI related info
     *
-    * @tparam Reslover - callable with the signature (const name& code_account) -> std::optional<abi_def>
+    * @tparam Resolver - callable with the signature (const name& code_account) -> std::optional<abi_def>
     */
    template<typename T, typename Resolver>
    class abi_to_variant_visitor
    {
       public:
-         abi_to_variant_visitor( mutable_variant_object& _mvo, const T& _val, Resolver _resolver, abi_traverse_context& _ctx )
+         abi_to_variant_visitor( mutable_variant_object& _mvo, const T& _val, const Resolver& _resolver, abi_traverse_context& _ctx )
          :_vo(_mvo)
          ,_val(_val)
          ,_resolver(_resolver)
@@ -707,7 +726,7 @@ namespace impl {
       private:
          mutable_variant_object& _vo;
          const T& _val;
-         Resolver _resolver;
+         const Resolver& _resolver;
          abi_traverse_context& _ctx;
    };
 
@@ -717,7 +736,7 @@ namespace impl {
        * and can be degraded to the normal ::from_variant(...) processing
        */
       template<typename M, typename Resolver, not_require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, M& o, Resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, M& o, const Resolver&, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          from_variant(v, o);
@@ -728,14 +747,14 @@ namespace impl {
        * for these types we create new ABI aware visitors
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, M& o, Resolver resolver, abi_traverse_context& ctx );
+      static void extract( const fc::variant& v, M& o, const Resolver& resolver, abi_traverse_context& ctx );
 
       /**
        * template which overloads extract for vectors of types which contain ABI information in their trees
        * for these members we call ::extract in order to trigger further processing
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, vector<M>& o, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, vector<M>& o, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variants& array = v.get_array();
@@ -753,7 +772,7 @@ namespace impl {
   * for these members we call ::extract in order to trigger further processing
   */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, deque<M>& o, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, deque<M>& o, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variants& array = v.get_array();
@@ -771,7 +790,7 @@ namespace impl {
        * for these members we call ::extract in order to trigger further processing
        */
       template<typename M, typename Resolver, require_abi_t<M> = 1>
-      static void extract( const fc::variant& v, std::shared_ptr<M>& o, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, std::shared_ptr<M>& o, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variant_object& vo = v.get_object();
@@ -786,7 +805,7 @@ namespace impl {
        * exploded and processed explicitly
        */
       template<typename Resolver>
-      static void extract( const fc::variant& v, action& act, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, action& act, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variant_object& vo = v.get_object();
@@ -806,13 +825,14 @@ namespace impl {
                from_variant(data, act.data);
                valid_empty_data = act.data.empty();
             } else if ( data.is_object() ) {
-               auto abi = resolver(act.account);
-               if (abi) {
-                  auto type = abi->get_action_type(act.name);
+               auto abi_optional = resolver(act.account);
+               if (abi_optional) {
+                  const abi_serializer& abi = *abi_optional;
+                  auto type = abi.get_action_type(act.name);
                   if (!type.empty()) {
-                     variant_to_binary_context _ctx(*abi, ctx, type);
+                     variant_to_binary_context _ctx(abi, ctx, type);
                      _ctx.short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place
-                     act.data = std::move( abi->_variant_to_binary( type, data, _ctx ));
+                     act.data = abi._variant_to_binary( type, data, _ctx );
                      valid_empty_data = act.data.empty();
                   }
                }
@@ -833,7 +853,7 @@ namespace impl {
       }
 
       template<typename Resolver>
-      static void extract( const fc::variant& v, packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx )
+      static void extract( const fc::variant& v, packed_transaction& ptrx, const Resolver& resolver, abi_traverse_context& ctx )
       {
          auto h = ctx.enter_scope();
          const variant_object& vo = v.get_object();
@@ -890,7 +910,7 @@ namespace impl {
    class abi_from_variant_visitor : public reflector_init_visitor<T>
    {
       public:
-         abi_from_variant_visitor( const variant_object& _vo, T& v, Resolver _resolver, abi_traverse_context& _ctx )
+         abi_from_variant_visitor( const variant_object& _vo, T& v, const Resolver& _resolver, abi_traverse_context& _ctx )
          : reflector_init_visitor<T>(v)
          ,_vo(_vo)
          ,_resolver(_resolver)
@@ -914,12 +934,12 @@ namespace impl {
 
       private:
          const variant_object& _vo;
-         Resolver _resolver;
+         const Resolver& _resolver;
          abi_traverse_context& _ctx;
    };
 
    template<typename M, typename Resolver, require_abi_t<M>>
-   void abi_to_variant::add( mutable_variant_object &mvo, const char* name, const M& v, Resolver resolver, abi_traverse_context& ctx )
+   void abi_to_variant::add( mutable_variant_object &mvo, const char* name, const M& v, const Resolver& resolver, abi_traverse_context& ctx )
    {
       auto h = ctx.enter_scope();
       mutable_variant_object member_mvo;
@@ -928,7 +948,7 @@ namespace impl {
    }
 
    template<typename M, typename Resolver, require_abi_t<M>>
-   void abi_from_variant::extract( const fc::variant& v, M& o, Resolver resolver, abi_traverse_context& ctx )
+   void abi_from_variant::extract( const fc::variant& v, M& o, const Resolver& resolver, abi_traverse_context& ctx )
    {
       auto h = ctx.enter_scope();
       const variant_object& vo = v.get_object();
@@ -937,38 +957,158 @@ namespace impl {
 } /// namespace eosio::chain::impl
 
 template<typename T, typename Resolver>
-void abi_serializer::to_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield ) try {
+void abi_serializer::to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield ) try {
    mutable_variant_object mvo;
-   impl::abi_traverse_context ctx( yield );
+   impl::abi_traverse_context ctx( yield, fc::microseconds{} );
    impl::abi_to_variant::add(mvo, "_", o, resolver, ctx);
    vo = std::move(mvo["_"]);
 } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: ${type}", ("type", boost::core::demangle( typeid(o).name() ) ))
 
 template<typename T, typename Resolver>
-void abi_serializer::to_variant( const T& o, fc::variant& vo, Resolver resolver, const fc::microseconds& max_serialization_time ) {
-   to_variant( o, vo, resolver, create_yield_function(max_serialization_time) );
-}
+void abi_serializer::to_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try {
+   mutable_variant_object mvo;
+   impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time );
+   impl::abi_to_variant::add(mvo, "_", o, resolver, ctx);
+   vo = std::move(mvo["_"]);
+} FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: ${type}", ("type", boost::core::demangle( typeid(o).name() ) ))
 
 template<typename T, typename Resolver>
-void abi_serializer::to_log_variant( const T& o, fc::variant& vo, Resolver resolver, const yield_function_t& yield ) try {
+void abi_serializer::to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const yield_function_t& yield ) try {
     mutable_variant_object mvo;
-    impl::abi_traverse_context ctx( yield );
+    impl::abi_traverse_context ctx( yield, fc::microseconds{} );
     ctx.logging();
     impl::abi_to_variant::add(mvo, "_", o, resolver, ctx);
     vo = std::move(mvo["_"]);
 } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: ${type}", ("type", boost::core::demangle( typeid(o).name() ) ))
 
+template<typename T, typename Resolver>
+void abi_serializer::to_log_variant( const T& o, fc::variant& vo, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try {
+   mutable_variant_object mvo;
+   impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time );
+   ctx.logging();
+   impl::abi_to_variant::add(mvo, "_", o, resolver, ctx);
+   vo = std::move(mvo["_"]);
+} FC_RETHROW_EXCEPTIONS(error, "Failed to serialize: ${type}", ("type", boost::core::demangle( typeid(o).name() ) ))
 
 template<typename T, typename Resolver>
-void abi_serializer::from_variant( const fc::variant& v, T& o, Resolver resolver, const yield_function_t& yield ) try {
-   impl::abi_traverse_context ctx( yield );
+void abi_serializer::from_variant( const fc::variant& v, T& o, const Resolver& resolver, const yield_function_t& yield ) try {
+   impl::abi_traverse_context ctx( yield, fc::microseconds{} );
    impl::abi_from_variant::extract(v, o, resolver, ctx);
 } FC_RETHROW_EXCEPTIONS(error, "Failed to deserialize variant", ("variant",v))
 
 template<typename T, typename Resolver>
-void abi_serializer::from_variant( const fc::variant& v, T& o, Resolver resolver, const fc::microseconds& max_serialization_time ) {
-   from_variant( v, o, resolver, create_yield_function(max_serialization_time) );
-}
+void abi_serializer::from_variant( const fc::variant& v, T& o, const Resolver& resolver, const fc::microseconds& max_action_data_serialization_time ) try {
+   impl::abi_traverse_context ctx( create_depth_yield_function(), max_action_data_serialization_time );
+   impl::abi_from_variant::extract(v, o, resolver, ctx);
+} FC_RETHROW_EXCEPTIONS(error, "Failed to deserialize variant", ("variant",v))
 
+using abi_serializer_cache_t = std::unordered_map<account_name, std::optional<abi_serializer>>;
+using resolver_fn_t = std::function<std::optional<abi_serializer>(const account_name& name)>;
+   
+class abi_resolver {
+public:
+   explicit abi_resolver(abi_serializer_cache_t&& abi_serializers) :
+      abi_serializers(std::move(abi_serializers))
+   {}
 
-} } // eosio::chain
+   std::optional<std::reference_wrapper<const abi_serializer>> operator()(const account_name& account) const {
+      auto it = abi_serializers.find(account);
+      if (it != abi_serializers.end() && it->second)
+         return std::reference_wrapper<const abi_serializer>(*it->second);
+      return {};
+   };
+
+private:
+   abi_serializer_cache_t abi_serializers;
+};
+
+class abi_serializer_cache_builder {
+public:
+   explicit abi_serializer_cache_builder(resolver_fn_t resolver) :
+      resolver_(std::move(resolver))
+   {
+   }
+
+   abi_serializer_cache_builder(const abi_serializer_cache_builder&) = delete;
+
+   abi_serializer_cache_builder&& add_serializers(const chain::signed_block_ptr& block) && {
+      for( const auto& receipt: block->transactions ) {
+         if( std::holds_alternative<chain::packed_transaction>( receipt.trx ) ) {
+            const auto& pt = std::get<chain::packed_transaction>( receipt.trx );
+            const auto& t = pt.get_transaction();
+            for( const auto& a: t.actions )
+               add_to_cache( a );
+            for( const auto& a: t.context_free_actions )
+               add_to_cache( a );
+         }
+      }
+      return std::move(*this);
+   }
+
+   abi_serializer_cache_builder&& add_serializers(const transaction_trace_ptr& trace_ptr) && {
+      for( const auto& trace: trace_ptr->action_traces ) {
+         add_to_cache(trace.act);
+      }
+      return std::move(*this);
+   }
+
+   abi_serializer_cache_t&& get() && {
+      return std::move(abi_serializers);
+   }
+
+private:
+   void add_to_cache(const chain::action& a) {
+      auto it = abi_serializers.find( a.account );
+      if( it == abi_serializers.end() ) {
+         try {
+            abi_serializers.emplace_hint( it, a.account, resolver_( a.account ) );
+         } catch( ... ) {
+            // keep behavior of not throwing on invalid abi, will result in hex data
+         }
+      }
+   }
+
+   resolver_fn_t resolver_;
+   abi_serializer_cache_t abi_serializers;
+};
+
+/*
+ * This is equivalent to a resolver, except that everytime the abi_serializer for an account 
+ * is retrieved, it is stored in an unordered_map, so we won't waste time retrieving it again.
+ * This is handy when parsing packed_transactions received in a fc::variant.
+ */
+class caching_resolver {
+public:
+   explicit caching_resolver(resolver_fn_t resolver) :
+      resolver_(std::move(resolver))
+   {
+   }
+
+   // make it non-copiable (we should only move it for performance reasons)
+   caching_resolver(const caching_resolver&) = delete;
+   caching_resolver& operator=(const caching_resolver&) = delete;
+
+   std::optional<std::reference_wrapper<const abi_serializer>> operator()(const account_name& account) const {
+      auto it = abi_serializers.find(account);
+      if (it != abi_serializers.end()) {
+         if (it->second)
+            return *it->second;
+         return {};
+      }
+      auto serializer = resolver_(account);
+      auto& dest = abi_serializers[account]; // add entry regardless
+      if (serializer) {
+         // we got a serializer, so move it into the cache
+         dest = abi_serializer_cache_t::mapped_type{std::move(*serializer)};
+         return *dest; // and return a reference to it
+      }
+      return {}; 
+   };
+
+private:
+   const resolver_fn_t resolver_;
+   mutable abi_serializer_cache_t abi_serializers;
+};
+      
+
+} // eosio::chain

libraries/chain/include/eosio/chain/apply_context.hpp

@@ -1,6 +1,7 @@
 #pragma once
 #include <eosio/chain/controller.hpp>
 #include <eosio/chain/transaction.hpp>
+#include <eosio/chain/transaction_context.hpp>
 #include <eosio/chain/contract_table_objects.hpp>
 #include <eosio/chain/deep_mind.hpp>
 #include <fc/utility.hpp>
@@ -13,7 +14,6 @@ namespace chainbase { class database; }
 namespace eosio { namespace chain {
 
 class controller;
-class transaction_context;
 
 class apply_context {
    private:
@@ -177,6 +177,7 @@ class apply_context {
             int store( uint64_t scope, uint64_t table, const account_name& payer,
                        uint64_t id, secondary_key_proxy_const_type value )
             {
+               EOS_ASSERT( !context.trx_context.is_read_only(), table_operation_not_permitted, "cannot store a db record when executing a readonly transaction" );
                EOS_ASSERT( payer != account_name(), invalid_table_payer, "must specify a valid account to pay for new record" );
 
 //               context.require_write_lock( scope );
@@ -193,7 +194,7 @@ class apply_context {
                context.db.modify( tab, [&]( auto& t ) {
                  ++t.count;
 
-                  if (auto dm_logger = context.control.get_deep_mind_logger()) {
+                  if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
                      std::string event_id = RAM_EVENT_ID("${code}:${scope}:${table}:${index_name}",
                         ("code", t.code)
                         ("scope", t.scope)
@@ -211,12 +212,13 @@ class apply_context {
             }
 
             void remove( int iterator ) {
+               EOS_ASSERT( !context.trx_context.is_read_only(), table_operation_not_permitted, "cannot remove a db record when executing a readonly transaction" );
                const auto& obj = itr_cache.get( iterator );
 
                const auto& table_obj = itr_cache.get_table( obj.t_id );
                EOS_ASSERT( table_obj.code == context.receiver, table_access_violation, "db access violation" );
 
-               if (auto dm_logger = context.control.get_deep_mind_logger()) {
+               if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient())) {
                   std::string event_id = RAM_EVENT_ID("${code}:${scope}:${table}:${index_name}",
                      ("code", table_obj.code)
                      ("scope", table_obj.scope)
@@ -243,6 +245,7 @@ class apply_context {
             }
 
             void update( int iterator, account_name payer, secondary_key_proxy_const_type secondary ) {
+               EOS_ASSERT( !context.trx_context.is_read_only(), table_operation_not_permitted, "cannot update a db record when executing a readonly transaction" );
                const auto& obj = itr_cache.get( iterator );
 
                const auto& table_obj = itr_cache.get_table( obj.t_id );
@@ -255,7 +258,7 @@ class apply_context {
                int64_t billing_size =  config::billable_size_v<ObjectType>;
 
                std::string event_id;
-               if (context.control.get_deep_mind_logger() != nullptr) {
+               if (context.control.get_deep_mind_logger(context.trx_context.is_transient()) != nullptr) {
                   event_id = RAM_EVENT_ID("${code}:${scope}:${table}:${index_name}",
                      ("code", table_obj.code)
                      ("scope", table_obj.scope)
@@ -265,12 +268,12 @@ class apply_context {
                }
 
                if( obj.payer != payer ) {
-                  if (auto dm_logger = context.control.get_deep_mind_logger())
+                  if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient()))
                   {
                      dm_logger->on_ram_trace(std::string(event_id), "secondary_index", "remove", "secondary_index_remove");
                   }
                   context.update_db_usage( obj.payer, -(billing_size) );
-                  if (auto dm_logger = context.control.get_deep_mind_logger())
+                  if (auto dm_logger = context.control.get_deep_mind_logger(context.trx_context.is_transient()))
                   {
                      dm_logger->on_ram_trace(std::move(event_id), "secondary_index", "add", "secondary_index_update_add_new_payer");
                   }
@@ -595,6 +598,9 @@ class apply_context {
 
       action_name get_sender() const;
 
+      bool is_applying_block() const { return trx_context.explicit_billed_cpu_time; }
+      bool should_use_eos_vm_oc()const;
+
    /// Fields:
    public:
 

libraries/chain/include/eosio/chain/authority.hpp

@@ -188,7 +188,7 @@ struct authority {
    }
 
    authority( uint32_t t, vector<key_weight> k, vector<permission_level_weight> p = {}, vector<wait_weight> w = {} )
-   :threshold(t),keys(move(k)),accounts(move(p)),waits(move(w)){}
+   :threshold(t),keys(std::move(k)),accounts(std::move(p)),waits(std::move(w)){}
    authority(){}
 
    uint32_t                          threshold = 0;

libraries/chain/include/eosio/chain/authorization_manager.hpp

@@ -31,6 +31,7 @@ namespace eosio { namespace chain {
                                                      permission_name name,
                                                      permission_id_type parent,
                                                      const authority& auth,
+                                                     bool is_trx_transient,
                                                      time_point initial_creation_time = time_point()
                                                    );
 
@@ -38,12 +39,13 @@ namespace eosio { namespace chain {
                                                      permission_name name,
                                                      permission_id_type parent,
                                                      authority&& auth,
+                                                     bool is_trx_transient,
                                                      time_point initial_creation_time = time_point()
                                                    );
 
-         void modify_permission( const permission_object& permission, const authority& auth );
+         void modify_permission( const permission_object& permission, const authority& auth, bool is_trx_transient );
 
-         void remove_permission( const permission_object& permission );
+         void remove_permission( const permission_object& permission, bool is_trx_transient );
 
          void update_permission_usage( const permission_object& permission );
 

libraries/chain/include/eosio/chain/block_log.hpp

@@ -2,10 +2,11 @@
 #include <fc/filesystem.hpp>
 #include <eosio/chain/block.hpp>
 #include <eosio/chain/genesis_state.hpp>
+#include <eosio/chain/block_log_config.hpp>
 
 namespace eosio { namespace chain {
 
-   namespace detail { class block_log_impl; }
+   namespace detail { struct block_log_impl; }
 
    /* The block log is an external append only log of the blocks with a header. Blocks should only
     * be written to the log after they irreverisble as the log is append only. The log is a doubly
@@ -34,18 +35,15 @@ namespace eosio { namespace chain {
     * An optional "pruned" mode can be activated which stores a 4 byte trailer on the log file indicating
     * how many blocks at the end of the log are valid. Any earlier blocks in the log are assumed destroyed
     * and unreadable due to reclamation for purposes of saving space.
+    *
+    * Object thread-safe. Not safe to have multiple block_log objects to same data_dir.
     */
 
-   struct block_log_prune_config {
-      uint32_t                prune_blocks;                  //number of blocks to prune to when doing a prune
-      size_t                  prune_threshold = 4*1024*1024; //(approximately) how many bytes need to be added before a prune is performed
-      std::optional<size_t>   vacuum_on_close;               //when set, a vacuum is performed on dtor if log contains less than this many live bytes
-   };
 
    class block_log {
       public:
-         block_log(const fc::path& data_dir, std::optional<block_log_prune_config> prune_config);
-         block_log(block_log&& other);
+         explicit block_log(const std::filesystem::path& data_dir, const block_log_config& config = block_log_config{});
+         block_log(block_log&& other) noexcept;
          ~block_log();
 
          void append(const signed_block_ptr& b, const block_id_type& id);
@@ -54,12 +52,11 @@ namespace eosio { namespace chain {
          void flush();
          void reset( const genesis_state& gs, const signed_block_ptr& genesis_block );
          void reset( const chain_id_type& chain_id, uint32_t first_block_num );
-         void remove(); // remove blocks.log and blocks.index
 
-         signed_block_ptr read_block(uint64_t file_pos)const;
-         void             read_block_header(block_header& bh, uint64_t file_pos)const;
          signed_block_ptr read_block_by_num(uint32_t block_num)const;
+         std::optional<signed_block_header> read_block_header_by_num(uint32_t block_num)const;
          block_id_type    read_block_id_by_num(uint32_t block_num)const;
+
          signed_block_ptr read_block_by_id(const block_id_type& id)const {
             return read_block_by_num(block_header::num_from_id(id));
          }
@@ -67,10 +64,11 @@ namespace eosio { namespace chain {
          /**
           * Return offset of block in file, or block_log::npos if it does not exist.
           */
-         uint64_t get_block_pos(uint32_t block_num) const;
-         signed_block_ptr        read_head()const;
-         const signed_block_ptr& head()const;
-         const block_id_type&    head_id()const;
+
+         signed_block_ptr read_head()const; //use blocklog
+         signed_block_ptr head()const;
+         std::optional<block_id_type> head_id()const;
+
          uint32_t                first_block_num() const;
 
          static const uint64_t npos = std::numeric_limits<uint64_t>::max();
@@ -78,13 +76,25 @@ namespace eosio { namespace chain {
          static const uint32_t min_supported_version;
          static const uint32_t max_supported_version;
 
-         static fc::path repair_log( const fc::path& data_dir, uint32_t truncate_at_block = 0, const char* reversible_block_dir_name="" );
+         /**
+          * All static methods expected to be called on quiescent block log
+          */
 
-         static std::optional<genesis_state> extract_genesis_state( const fc::path& data_dir );
+         static std::filesystem::path repair_log( const std::filesystem::path& data_dir, uint32_t truncate_at_block = 0, const char* reversible_block_dir_name="" );
 
-         static chain_id_type extract_chain_id( const fc::path& data_dir );
+         using chain_context = std::variant<genesis_state, chain_id_type>;
+         static std::optional<chain_context> extract_chain_context(const std::filesystem::path& data_dir,
+                                                                   const std::filesystem::path& retained_dir);
 
-         static void construct_index(const fc::path& block_file_name, const fc::path& index_file_name);
+         static std::optional<genesis_state>
+         extract_genesis_state(const std::filesystem::path& data_dir,
+                               const std::filesystem::path& retained_dir = std::filesystem::path{});
+
+         static std::optional<chain_id_type>
+         extract_chain_id(const std::filesystem::path& data_dir,
+                          const std::filesystem::path& retained_dir = std::filesystem::path{});
+
+         static void construct_index(const std::filesystem::path& block_file_name, const std::filesystem::path& index_file_name);
 
          static bool contains_genesis_state(uint32_t version, uint32_t first_block_num);
 
@@ -92,39 +102,26 @@ namespace eosio { namespace chain {
 
          static bool is_supported_version(uint32_t version);
 
-         static bool is_pruned_log(const fc::path& data_dir);
+         static bool is_pruned_log(const std::filesystem::path& data_dir);
 
-         static bool extract_block_range(const fc::path& block_dir, const fc::path&output_dir, block_num_type& start, block_num_type& end, bool rename_input=false);
+         static void extract_block_range(const std::filesystem::path& block_dir, const std::filesystem::path&output_dir, block_num_type start, block_num_type end);
 
+         static bool trim_blocklog_front(const std::filesystem::path& block_dir, const std::filesystem::path& temp_dir, uint32_t truncate_at_block);
+         static int  trim_blocklog_end(const std::filesystem::path& block_dir, uint32_t n);
+
+         // used for unit test to generate older version blocklog
+         static void set_initial_version(uint32_t);
+         uint32_t    version() const;
+         uint64_t get_block_pos(uint32_t block_num) const;
+
+         /**
+          * @param n Only test 1 block out of every n blocks. If n is 0, the interval is adjusted so that at most 8 blocks are tested.
+          */
+         static void smoke_test(const std::filesystem::path& block_dir, uint32_t n);
+
+         static void split_blocklog(const std::filesystem::path& block_dir, const std::filesystem::path& dest_dir, uint32_t stride);
+         static void merge_blocklogs(const std::filesystem::path& block_dir, const std::filesystem::path& dest_dir);
    private:
-         void open(const fc::path& data_dir);
-         void construct_index();
-
          std::unique_ptr<detail::block_log_impl> my;
    };
-
-//to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed
-//   block_timestamp_type timestamp;                  //bytes 0:3
-//   account_name         producer;                   //bytes 4:11
-//   uint16_t             confirmed;                  //bytes 12:13
-//   block_id_type        previous;                   //bytes 14:45, low 4 bytes is big endian block number of previous block
-
-   struct trim_data {            //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test()
-      trim_data(fc::path block_dir);
-      ~trim_data();
-      uint64_t block_index(uint32_t n) const;
-      uint64_t block_pos(uint32_t n);
-      fc::path block_file_name, index_file_name;        //full pathname for blocks.log and blocks.index
-      uint32_t version = 0;                              //blocklog version
-      uint32_t first_block = 0;                          //first block in blocks.log
-      uint32_t last_block = 0;                          //last block in blocks.log
-      FILE* blk_in = nullptr;                            //C style files for reading blocks.log and blocks.index
-      FILE* ind_in = nullptr;                            //C style files for reading blocks.log and blocks.index
-      //we use low level file IO because it is distinctly faster than C++ filebuf or iostream
-      uint64_t first_block_pos = 0;                      //file position in blocks.log for the first block in the log
-      genesis_state gs;
-      chain_id_type chain_id;
-
-      static constexpr int blknum_offset{14};            //offset from start of block to 4 byte block number, valid for the only allowed versions
-   };
 } }

libraries/chain/include/eosio/chain/block_log_config.hpp

@@ -0,0 +1,29 @@
+#pragma once
+#include <variant>
+
+namespace eosio { namespace chain {
+
+
+   struct basic_blocklog_config {};
+
+   struct empty_blocklog_config {};
+
+   struct partitioned_blocklog_config {
+      std::filesystem::path retained_dir;
+      std::filesystem::path archive_dir;
+      uint32_t              stride             = UINT32_MAX;
+      uint32_t              max_retained_files = UINT32_MAX;
+   };
+
+   struct prune_blocklog_config {
+      uint32_t prune_blocks; // number of blocks to prune to when doing a prune
+      size_t   prune_threshold =
+            4 * 1024 * 1024; //(approximately) how many bytes need to be added before a prune is performed
+      std::optional<size_t>
+            vacuum_on_close; // when set, a vacuum is performed on dtor if log contains less than this many live bytes
+   };
+
+   using block_log_config =
+         std::variant<basic_blocklog_config, empty_blocklog_config, partitioned_blocklog_config, prune_blocklog_config>;
+
+}} // namespace eosio::chain

libraries/chain/include/eosio/chain/chain_id_type.hpp

@@ -37,6 +37,10 @@ namespace chain {
 
       void reflector_init()const;
 
+      static chain_id_type empty_chain_id() {
+         return {};
+      }
+
       private:
          chain_id_type() = default;
 

libraries/chain/include/eosio/chain/config.hpp

@@ -76,7 +76,7 @@ const static uint32_t   default_max_inline_action_size               = 512 * 102
 const static uint16_t   default_max_inline_action_depth              = 4;
 const static uint16_t   default_max_auth_depth                       = 6;
 const static uint32_t   default_sig_cpu_bill_pct                     = 50 * percent_1; // billable percentage of signature recovery
-const static uint32_t   default_block_cpu_effort_pct                 = 80 * percent_1; // percentage of block time used for producing block
+const static uint32_t   default_block_cpu_effort_pct                 = 90 * percent_1; // percentage of block time used for producing block
 const static uint16_t   default_controller_thread_pool_size          = 2;
 const static uint32_t   default_max_variable_signature_length        = 16384u;
 const static uint32_t   default_max_nonprivileged_inline_action_size = 4 * 1024; // 4 KB

libraries/chain/include/eosio/chain/controller.hpp

@@ -39,6 +39,7 @@ namespace eosio { namespace chain {
    class permission_object;
    class account_object;
    class deep_mind_handler;
+   class subjective_billing;
    using resource_limits::resource_limits_manager;
    using apply_handler = std::function<void(apply_context&)>;
    using forked_branch_callback = std::function<void(const branch_type&)>;
@@ -48,10 +49,9 @@ namespace eosio { namespace chain {
    class fork_database;
 
    enum class db_read_mode {
-      SPECULATIVE,
       HEAD,
-      READ_ONLY,
-      IRREVERSIBLE
+      IRREVERSIBLE,
+      SPECULATIVE
    };
 
    enum class validation_mode {
@@ -70,7 +70,7 @@ namespace eosio { namespace chain {
             flat_set< pair<account_name, action_name> > action_blacklist;
             flat_set<public_key_type> key_blacklist;
             path                     blocks_dir             =  chain::config::default_blocks_dir_name;
-            std::optional<block_log_prune_config>  prune_config;
+            block_log_config         blog;
             path                     state_dir              =  chain::config::default_state_dir_name;
             uint64_t                 state_size             =  chain::config::default_state_size;
             uint64_t                 state_guard_size       =  chain::config::default_state_guard_size;
@@ -84,15 +84,15 @@ namespace eosio { namespace chain {
             bool                     allow_ram_billing_in_notify = false;
             uint32_t                 maximum_variable_signature_length = chain::config::default_max_variable_signature_length;
             bool                     disable_all_subjective_mitigations = false; //< for developer & testing purposes, can be configured using `disable-all-subjective-mitigations` when `EOSIO_DEVELOPER` build option is provided
-            uint32_t                 terminate_at_block     = 0; //< primarily for testing purposes
+            uint32_t                 terminate_at_block     = 0;
             bool                     integrity_hash_on_start= false;
             bool                     integrity_hash_on_stop = false;
 
             wasm_interface::vm_type  wasm_runtime = chain::config::default_wasm_runtime;
             eosvmoc::config          eosvmoc_config;
-            bool                     eosvmoc_tierup         = false;
+            wasm_interface::vm_oc_enable eosvmoc_tierup     = wasm_interface::vm_oc_enable::oc_auto;
 
-            db_read_mode             read_mode              = db_read_mode::SPECULATIVE;
+            db_read_mode             read_mode              = db_read_mode::HEAD;
             validation_mode          block_validation_mode  = validation_mode::FULL;
 
             pinnable_mapped_file::map_mode db_map_mode      = pinnable_mapped_file::map_mode::mapped;
@@ -108,7 +108,8 @@ namespace eosio { namespace chain {
             irreversible = 0, ///< this block has already been applied before by this node and is considered irreversible
             validated   = 1, ///< this is a complete block signed by a valid producer and has been previously applied by this node and therefore validated but it is not yet irreversible
             complete   = 2, ///< this is a complete block signed by a valid producer but is not yet irreversible nor has it yet been applied by this node
-            incomplete  = 3, ///< this is an incomplete block (either being produced by a producer or speculatively produced by a node)
+            incomplete  = 3, ///< this is an incomplete block being produced by a producer
+            ephemeral = 4  ///< this is an incomplete block created for speculative execution of trxs, will always be aborted
          };
 
          controller( const config& cfg, const chain_id_type& chain_id );
@@ -120,27 +121,19 @@ namespace eosio { namespace chain {
          void startup( std::function<void()> shutdown, std::function<bool()> check_shutdown, const genesis_state& genesis);
          void startup( std::function<void()> shutdown, std::function<bool()> check_shutdown);
 
-         void preactivate_feature( const digest_type& feature_digest );
+         void preactivate_feature( const digest_type& feature_digest, bool is_trx_transient );
 
          vector<digest_type> get_preactivated_protocol_features()const;
 
          void validate_protocol_features( const vector<digest_type>& features_to_activate )const;
 
          /**
-          *  Starts a new pending block session upon which new transactions can
-          *  be pushed.
-          *
-          *  Will only activate protocol features that have been pre-activated.
-          */
-         void start_block( block_timestamp_type time = block_timestamp_type(), uint16_t confirm_block_count = 0 );
-
-         /**
-          * Starts a new pending block session upon which new transactions can
-          * be pushed.
+          * Starts a new pending block session upon which new transactions can be pushed.
           */
          void start_block( block_timestamp_type time,
                            uint16_t confirm_block_count,
                            const vector<digest_type>& new_protocol_feature_activations,
+                           block_status bs,
                            const fc::time_point& deadline = fc::time_point::maximum() );
 
          /**
@@ -175,16 +168,19 @@ namespace eosio { namespace chain {
          void sign_block( const signer_callback_type& signer_callback );
          void commit_block();
 
+         // thread-safe
          std::future<block_state_ptr> create_block_state_future( const block_id_type& id, const signed_block_ptr& b );
+         // thread-safe
+         block_state_ptr create_block_state( const block_id_type& id, const signed_block_ptr& b ) const;
 
          /**
           * @param br returns statistics for block
-          * @param block_state_future provide from call to create_block_state_future
+          * @param bsp block to push
           * @param cb calls cb with forked applied transactions for each forked block
           * @param trx_lookup user provided lookup function for externally cached transaction_metadata
           */
          void push_block( block_report& br,
-                          std::future<block_state_ptr>& block_state_future,
+                          const block_state_ptr& bsp,
                           const forked_branch_callback& cb,
                           const trx_meta_cache_lookup& trx_lookup );
 
@@ -202,6 +198,8 @@ namespace eosio { namespace chain {
          const authorization_manager&          get_authorization_manager()const;
          authorization_manager&                get_mutable_authorization_manager();
          const protocol_feature_manager&       get_protocol_feature_manager()const;
+         const subjective_billing&             get_subjective_billing()const;
+         subjective_billing&                   get_mutable_subjective_billing();
          uint32_t                              get_max_nonprivileged_inline_action_size()const;
 
          const flat_set<account_name>&   get_actor_whitelist() const;
@@ -227,15 +225,9 @@ namespace eosio { namespace chain {
 
          uint32_t             fork_db_head_block_num()const;
          block_id_type        fork_db_head_block_id()const;
-         time_point           fork_db_head_block_time()const;
-         account_name         fork_db_head_block_producer()const;
-
-         uint32_t             fork_db_pending_head_block_num()const;
-         block_id_type        fork_db_pending_head_block_id()const;
-         time_point           fork_db_pending_head_block_time()const;
-         account_name         fork_db_pending_head_block_producer()const;
 
          time_point                     pending_block_time()const;
+         block_timestamp_type           pending_block_timestamp()const;
          account_name                   pending_block_producer()const;
          const block_signing_authority& pending_block_signing_authority()const;
          std::optional<block_id_type>   pending_producer_block_id()const;
@@ -249,12 +241,19 @@ namespace eosio { namespace chain {
          block_id_type last_irreversible_block_id() const;
          time_point last_irreversible_block_time() const;
 
+         // thread-safe
          signed_block_ptr fetch_block_by_number( uint32_t block_num )const;
-         signed_block_ptr fetch_block_by_id( block_id_type id )const;
-
+         // thread-safe
+         signed_block_ptr fetch_block_by_id( const block_id_type& id )const;
+         // thread-safe
+         std::optional<signed_block_header> fetch_block_header_by_number( uint32_t block_num )const;
+         // thread-safe
+         std::optional<signed_block_header> fetch_block_header_by_id( const block_id_type& id )const;
+         // return block_state from forkdb, thread-safe
          block_state_ptr fetch_block_state_by_number( uint32_t block_num )const;
+         // return block_state from forkdb, thread-safe
          block_state_ptr fetch_block_state_by_id( block_id_type id )const;
-
+         // thread-safe
          block_id_type get_block_id_for_num( uint32_t block_num )const;
 
          sha256 calculate_integrity_hash();
@@ -266,7 +265,7 @@ namespace eosio { namespace chain {
          void check_action_list( account_name code, action_name action )const;
          void check_key_list( const public_key_type& key )const;
          bool is_building_block()const;
-         bool is_producing_block()const;
+         bool is_speculative_block()const;
 
          bool is_ram_billing_in_notify_allowed()const;
 
@@ -316,13 +315,16 @@ namespace eosio { namespace chain {
          void add_to_ram_correction( account_name account, uint64_t ram_bytes );
          bool all_subjective_mitigations_disabled()const;
 
-         deep_mind_handler* get_deep_mind_logger() const;
+         deep_mind_handler* get_deep_mind_logger(bool is_trx_transient) const;
          void enable_deep_mind( deep_mind_handler* logger );
          uint32_t earliest_available_block_num() const;
 
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
          vm::wasm_allocator&  get_wasm_allocator();
 #endif
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+         bool is_eos_vm_oc_enabled() const;
+#endif
 
          static std::optional<uint64_t> convert_exception_to_error_code( const fc::exception& e );
 
@@ -348,27 +350,6 @@ namespace eosio { namespace chain {
          const apply_handler* find_apply_handler( account_name contract, scope_name scope, action_name act )const;
          wasm_interface& get_wasm_interface();
 
-
-         std::optional<abi_serializer> get_abi_serializer( account_name n, const abi_serializer::yield_function_t& yield )const {
-            if( n.good() ) {
-               try {
-                  const auto& a = get_account( n );
-                  abi_def abi;
-                  if( abi_serializer::to_abi( a.abi, abi ))
-                     return abi_serializer( abi, yield );
-               } FC_CAPTURE_AND_LOG((n))
-            }
-            return std::optional<abi_serializer>();
-         }
-
-         template<typename T>
-         fc::variant to_variant_with_abi( const T& obj, const abi_serializer::yield_function_t& yield )const {
-            fc::variant pretty_output;
-            abi_serializer::to_variant( obj, pretty_output,
-                                        [&]( account_name n ){ return get_abi_serializer( n, yield ); }, yield );
-            return pretty_output;
-         }
-
       static chain_id_type extract_chain_id(snapshot_reader& snapshot);
 
       static std::optional<chain_id_type> extract_chain_id_from_db( const path& state_dir );
@@ -376,6 +357,17 @@ namespace eosio { namespace chain {
       void replace_producer_keys( const public_key_type& key );
       void replace_account_keys( name account, name permission, const public_key_type& key );
 
+      void set_producer_node(bool is_producer_node);
+      bool is_producer_node()const;
+
+      void set_db_read_only_mode();
+      void unset_db_read_only_mode();
+      void init_thread_local_data();
+      void set_to_write_window();
+      void set_to_read_window();
+      bool is_write_window() const;
+      void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num);
+
       private:
          friend class apply_context;
          friend class transaction_context;

libraries/chain/include/eosio/chain/database_utils.hpp

@@ -170,7 +170,7 @@ namespace fc {
 
    inline
    void from_variant( const variant& v, eosio::chain::shared_string& s ) {
-      string _s;
+      std::string _s;
       from_variant(v, _s);
       s = eosio::chain::shared_string(_s.begin(), _s.end(), s.get_allocator());
    }
@@ -182,7 +182,7 @@ namespace fc {
 
    inline
    void from_variant( const variant& v, eosio::chain::shared_blob& b ) {
-      string _s = base64_decode(v.as_string());
+      std::string _s = base64_decode(v.as_string());
       b = eosio::chain::shared_blob(_s.begin(), _s.end(), b.get_allocator());
    }
 

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -87,6 +87,26 @@
       NEXT(e.dynamic_copy_exception());\
    }
 
+/**
+ * Capture all exceptions and return return_type which is constructible from a fc::exception_ptr
+ */
+#define CATCH_AND_RETURN(return_type)\
+   catch ( const fc::exception& err ) {\
+      return return_type(err.dynamic_copy_exception());\
+   } catch ( const std::exception& e ) {\
+      fc::exception fce( \
+         FC_LOG_MESSAGE( warn, "rethrow ${what}: ", ("what",e.what())),\
+         fc::std_exception_code,\
+         BOOST_CORE_TYPEID(e).name(),\
+         e.what() ) ;\
+      return return_type(fce.dynamic_copy_exception());\
+   } catch( ... ) {\
+      fc::unhandled_exception e(\
+         FC_LOG_MESSAGE(warn, "rethrow"),\
+         std::current_exception());\
+      return return_type(e.dynamic_copy_exception());\
+   }
+
 #define EOS_RECODE_EXC( cause_type, effect_type ) \
    catch( const cause_type& e ) \
    { throw( effect_type( e.what(), e.get_log() ) ); }
@@ -361,6 +381,10 @@ namespace eosio { namespace chain {
                                     3080007, "Transaction exceeded the current greylisted account network usage limit" )
       FC_DECLARE_DERIVED_EXCEPTION( greylist_cpu_usage_exceeded, resource_exhausted_exception,
                                     3080008, "Transaction exceeded the current greylisted account CPU usage limit" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_temporary_failure, resource_exhausted_exception,
+                                    3080009, "Read-only transaction eos-vm-oc compile temporary failure" )
+      FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_permanent_failure, resource_exhausted_exception,
+                                    3080010, "Read-only transaction eos-vm-oc compile permanent failure" )
 
       FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception,
                                     3081001, "Transaction reached the deadline set due to leeway on account CPU limits" )
@@ -573,6 +597,14 @@ namespace eosio { namespace chain {
                                     3170011, "The signer returned no valid block signatures" )
       FC_DECLARE_DERIVED_EXCEPTION( unsupported_multiple_block_signatures,  producer_exception,
                                     3170012, "The signer returned multiple signatures but that is not supported" )
+      FC_DECLARE_DERIVED_EXCEPTION( duplicate_snapshot_request,  producer_exception,
+                                    3170013, "Snapshot has been already scheduled with specified parameters" )
+      FC_DECLARE_DERIVED_EXCEPTION( snapshot_request_not_found,  producer_exception,
+                                    3170014, "Snapshot request not found" )
+      FC_DECLARE_DERIVED_EXCEPTION( invalid_snapshot_request,  producer_exception,
+                                    3170015, "Invalid snapshot request" )
+      FC_DECLARE_DERIVED_EXCEPTION( snapshot_execution_exception,  producer_exception,
+                                    3170016, "Snapshot execution exception" )
 
    FC_DECLARE_DERIVED_EXCEPTION( reversible_blocks_exception,           chain_exception,
                                  3180000, "Reversible Blocks exception" )

libraries/chain/include/eosio/chain/fork_database.hpp

@@ -16,11 +16,13 @@ namespace eosio { namespace chain {
     * database tracks the longest chain and the last irreversible block number. All
     * blocks older than the last irreversible block are freed after emitting the
     * irreversible signal.
+    *
+    * An internal mutex is used to provide thread-safety.
     */
    class fork_database {
       public:
 
-         explicit fork_database( const fc::path& data_dir );
+         explicit fork_database( const std::filesystem::path& data_dir );
          ~fork_database();
 
          void open( const std::function<void( block_timestamp_type,
@@ -55,9 +57,9 @@ namespace eosio { namespace chain {
 
          void            remove( const block_id_type& id );
 
-         const block_state_ptr& root()const;
-         const block_state_ptr& head()const;
-         block_state_ptr        pending_head()const;
+         block_state_ptr root()const;
+         block_state_ptr head()const;
+         block_state_ptr pending_head()const;
 
          /**
           *  Returns the sequence of block states resulting from trimming the branch from the

libraries/chain/include/eosio/chain/log_catalog.hpp

@@ -0,0 +1,278 @@
+#pragma once
+#include <fc/io/cfile.hpp>
+#include <fc/io/datastream.hpp>
+#include <filesystem>
+#include <regex>
+#include <map>
+
+namespace eosio {
+namespace chain {
+
+
+template <typename Lambda>
+void for_each_file_in_dir_matches(const std::filesystem::path& dir, std::string_view pattern, Lambda&& lambda) {
+   const std::regex        my_filter(pattern.begin(), pattern.size());
+   std::smatch             what;
+   std::filesystem::directory_iterator end_itr; // Default ctor yields past-the-end
+   for (std::filesystem::directory_iterator p(dir); p != end_itr; ++p) {
+      // Skip if not a file
+      if (!std::filesystem::is_regular_file(p->status()))
+         continue;
+      auto name = p->path().filename().string();
+      // skip if it does not match the pattern
+      if (!std::regex_match(name, what, my_filter))
+         continue;
+      lambda(p->path());
+   }
+}
+
+struct null_verifier {
+   template <typename LogData>
+   void verify(const LogData&, const std::filesystem::path&) {}
+};
+
+template <typename LogData, typename LogIndex, typename LogVerifier = null_verifier>
+struct log_catalog {
+   using block_num_t = uint32_t;
+
+   struct mapped_type {
+      block_num_t           last_block_num = 0;
+      std::filesystem::path filename_base;
+   };
+   using collection_t              = std::map<block_num_t, mapped_type>;
+   using size_type                 = typename collection_t::size_type;
+   static constexpr size_type npos = std::numeric_limits<size_type>::max();
+
+   std::filesystem::path retained_dir;
+   std::filesystem::path archive_dir;
+   size_type             max_retained_files = std::numeric_limits<size_type>::max();
+   collection_t          collection;
+   size_type             active_index = npos;
+   LogData               log_data;
+   LogIndex              log_index;
+   LogVerifier           verifier;
+
+   bool empty() const { return collection.empty(); }
+
+   block_num_t first_block_num() const {
+      if (empty())
+         return std::numeric_limits<block_num_t>::max();
+      return collection.begin()->first;
+   }
+
+   block_num_t last_block_num() const {
+      if (empty())
+         return std::numeric_limits<block_num_t>::min();
+      return collection.rbegin()->second.last_block_num;
+   }
+
+   static std::filesystem::path make_absolute_dir(const std::filesystem::path& base_dir, std::filesystem::path new_dir) {
+      if (new_dir.is_relative())
+         new_dir = base_dir / new_dir;
+
+      if (!std::filesystem::is_directory(new_dir))
+         std::filesystem::create_directories(new_dir);
+
+      return new_dir;
+   }
+
+   void open(const std::filesystem::path& log_dir, const std::filesystem::path& retained_path, const std::filesystem::path& archive_path, const char* name,
+             const char* suffix_pattern = R"(-\d+-\d+\.log)") {
+
+      retained_dir = make_absolute_dir(log_dir, retained_path.empty() ? log_dir : retained_path);
+      if (!archive_path.empty()) {
+         archive_dir = make_absolute_dir(log_dir, archive_path);
+      }
+
+      std::string pattern = std::string(name) + suffix_pattern;
+      for_each_file_in_dir_matches(retained_dir, pattern, [this](std::filesystem::path path) {
+         auto log_path               = path;
+         const auto& index_path      = path.replace_extension("index");
+         auto path_without_extension = log_path.parent_path() / log_path.stem().string();
+
+         LogData log(log_path);
+
+         verifier.verify(log, log_path);
+
+         // check if index file matches the log file
+         if (!index_matches_data(index_path, log)) {
+            ilog("Recreating index for: ${i}", ("i", index_path.string()));
+            log.construct_index( index_path );
+         }
+
+         auto existing_itr = collection.find(log.first_block_num());
+         if (existing_itr != collection.end()) {
+            if (log.last_block_num() <= existing_itr->second.last_block_num) {
+               wlog("${log_path} contains the overlapping range with ${existing_path}.log, dropping ${log_path} "
+                    "from catalog",
+                    ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string()));
+               return;
+            } else {
+               wlog(
+                   "${log_path} contains the overlapping range with ${existing_path}.log, droping ${existing_path}.log "
+                   "from catelog",
+                   ("log_path", log_path.string())("existing_path", existing_itr->second.filename_base.string()));
+            }
+         }
+
+         collection.insert_or_assign(log.first_block_num(), mapped_type{log.last_block_num(), std::move(path_without_extension)});
+      });
+   }
+
+   bool index_matches_data(const std::filesystem::path& index_path, LogData& log) const {
+      if (!std::filesystem::exists(index_path))
+         return false;
+
+      LogIndex log_i;
+      log_i.open(index_path);
+
+      if (log_i.num_blocks() != log.num_blocks())
+         return false;
+
+      return log_i.back() == log.last_block_position();
+   }
+
+   std::optional<uint64_t> get_block_position(uint32_t block_num) {
+      try {
+         if (active_index != npos) {
+            auto active_item = std::next(collection.begin(), active_index);
+            if (active_item->first <= block_num && block_num <= active_item->second.last_block_num) {
+               return log_index.nth_block_position(block_num - log_data.first_block_num());
+            }
+         }
+         if (block_num < first_block_num())
+            return {};
+
+         auto it = --collection.upper_bound(block_num);
+
+         if (block_num <= it->second.last_block_num) {
+            auto name = it->second.filename_base;
+            log_data.open(name.replace_extension("log"));
+            log_index.open(name.replace_extension("index"));
+            active_index = std::distance(collection.begin(), it);
+            return log_index.nth_block_position(block_num - log_data.first_block_num());
+         }
+         return {};
+      } catch (...) {
+         active_index = npos;
+         return {};
+      }
+   }
+
+   fc::datastream<fc::cfile>* ro_stream_for_block(uint32_t block_num) {
+      auto pos = get_block_position(block_num);
+      if (pos) {
+         return &log_data.ro_stream_at(*pos);
+      }
+      return nullptr;
+   }
+
+   template <typename ...Rest>
+   auto ro_stream_for_block(uint32_t block_num, Rest&& ...rest) -> std::optional<decltype( std::declval<LogData>().ro_stream_at(0, std::forward<Rest&&>(rest)...))> {
+      auto pos = get_block_position(block_num);
+      if (pos) {
+         return log_data.ro_stream_at(*pos, std::forward<Rest&&>(rest)...);
+      }
+      return {};
+   }
+
+   std::optional<block_id_type> id_for_block(uint32_t block_num) {
+      auto pos = get_block_position(block_num);
+      if (pos) {
+         return log_data.block_id_at(*pos);
+      }
+      return {};
+   }
+
+   static void rename_if_not_exists(std::filesystem::path old_name, std::filesystem::path new_name) {
+      if (!std::filesystem::exists(new_name)) {
+         std::filesystem::rename(old_name, new_name);
+      } else {
+         std::filesystem::remove(old_name);
+         wlog("${new_name} already exists, just removing ${old_name}",
+              ("old_name", old_name.string())("new_name", new_name.string()));
+      }
+   }
+
+   static void rename_bundle(std::filesystem::path orig_path, std::filesystem::path new_path) {
+      rename_if_not_exists(orig_path.replace_extension(".log"), new_path.replace_extension(".log"));
+      rename_if_not_exists(orig_path.replace_extension(".index"), new_path.replace_extension(".index"));
+   }
+
+   /// Add a new entry into the catalog.
+   ///
+   /// Notice that \c start_block_num must be monotonically increasing between the invocations of this function
+   /// so that the new entry would be inserted at the 'end' of the map; otherwise, \c active_index would be
+   /// invalidated and the mapping between the log data their block range would be wrong. This function is only used
+   /// during the splitting of block log. Using this function for other purpose should make sure if the monotonically
+   /// increasing block num guarantee can be met.
+   void add(uint32_t start_block_num, uint32_t end_block_num, const std::filesystem::path& dir, const char* name) {
+
+      const int bufsize = 64;
+      char      buf[bufsize];
+      snprintf(buf, bufsize, "%s-%u-%u", name, start_block_num, end_block_num);
+      std::filesystem::path new_path = retained_dir / buf;
+      rename_bundle(dir / name, new_path);
+      size_type items_to_erase = 0;
+      collection.emplace(start_block_num, mapped_type{end_block_num, std::move(new_path)});
+      if (collection.size() >= max_retained_files) {
+         items_to_erase =
+            max_retained_files > 0 ? collection.size() - max_retained_files : collection.size();
+         auto last = std::next( collection.begin(), items_to_erase);
+
+         for (auto it = collection.begin(); it != last; ++it) {
+            auto orig_name = it->second.filename_base;
+            if (archive_dir.empty()) {
+               // delete the old files when no backup dir is specified
+               std::filesystem::remove(orig_name.replace_extension("log"));
+               std::filesystem::remove(orig_name.replace_extension("index"));
+            } else {
+               // move the archive dir
+               rename_bundle(orig_name, archive_dir / orig_name.filename());
+            }
+         }
+         collection.erase(collection.begin(), last);
+         active_index = active_index == npos || active_index < items_to_erase
+                        ? npos
+                        : active_index - items_to_erase;
+      }
+   }
+
+   /// Truncate the catalog so that the log/index bundle containing the block with \c block_num
+   /// would be rename to \c new_name; the log/index bundles with blocks strictly higher
+   /// than \c block_num would be deleted, and all the renamed/removed entries would be erased
+   /// from the catalog.
+   ///
+   /// \return if nonzero, it's the starting block number for the log/index bundle being renamed.
+   uint32_t truncate(uint32_t block_num, std::filesystem::path new_name) {
+      if (collection.empty())
+         return 0;
+
+      auto remove_files = [](typename collection_t::const_reference v) {
+         auto name = v.second.filename_base;
+         std::filesystem::remove(name.replace_extension("log"));
+         std::filesystem::remove(name.replace_extension("index"));
+      };
+
+      active_index = npos;
+      auto it = collection.upper_bound(block_num);
+
+      if (it == collection.begin() || block_num > std::prev(it)->second.last_block_num) {
+         std::for_each(it, collection.end(), remove_files);
+         collection.erase(it, collection.end());
+         return 0;
+      } else {
+         auto truncate_it = --it;
+         auto name        = truncate_it->second.filename_base;
+         std::filesystem::rename(name.replace_extension("log"), new_name.replace_extension("log"));
+         std::filesystem::rename(name.replace_extension("index"), new_name.replace_extension("index"));
+         std::for_each(std::next(truncate_it), collection.end(), remove_files);
+         auto result = truncate_it->first;
+         collection.erase(truncate_it, collection.end());
+         return result;
+      }
+   }
+};
+
+} // namespace chain
+} // namespace eosio

libraries/chain/include/eosio/chain/log_data_base.hpp

@@ -0,0 +1,49 @@
+#pragma once
+#include <fc/io/cfile.hpp>
+
+namespace eosio {
+namespace chain {
+
+template <typename T>
+T read_data_at(fc::datastream<fc::cfile>& file, std::size_t offset) {
+   file.seek(offset);
+   T value;
+   fc::raw::unpack(file, value);
+   return value;
+}
+
+template <typename Derived>
+class log_data_base {
+ protected:
+   fc::datastream<fc::cfile> file;
+
+   Derived* self() { return static_cast<Derived*>(this); }
+   const Derived* self() const { return static_cast<const Derived*>(this); }
+
+ public:
+
+   log_data_base() = default;
+
+   void    close() { file.close(); }
+   bool    is_open() const { return file.is_open(); }
+
+   uint64_t    size() const { return self()->size(); }
+
+   uint32_t    last_block_num() { return self()->block_num_at(last_block_position()); }
+   uint64_t    last_block_position() {
+      uint32_t offset = sizeof(uint64_t);
+      if (self()->is_currently_pruned())
+         offset += sizeof(uint32_t);
+      return read_data_at<uint64_t>(file, size() - offset);
+   }
+
+   uint32_t num_blocks() {
+      if (self()->first_block_position() == size())
+         return 0;
+      else if (self()->is_currently_pruned())
+         return read_data_at<uint32_t>(file, size() - sizeof(uint32_t));
+      return last_block_num() - self()->first_block_num() + 1;
+   }
+};
+} // namespace chain
+} // namespace eosio

libraries/chain/include/eosio/chain/log_index.hpp

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <fc/io/cfile.hpp>
+
+namespace eosio {
+namespace chain {
+/// copy up to n bytes from the current position of src to dest
+void copy_file_content(fc::cfile& src, fc::cfile& dest, uint64_t n = UINT64_MAX);
+
+template <typename Exception>
+class log_index {
+   fc::cfile file_;
+   std::size_t num_blocks_ = 0;
+ public:
+   log_index() = default;
+   log_index(const std::filesystem::path& path) {
+      open(path);
+   }
+
+   void open(const std::filesystem::path& path) {
+      if (file_.is_open())
+         file_.close();
+      file_.set_file_path(path);
+      file_.open("rb");
+      file_.seek_end(0);
+      num_blocks_ = file_.tellp()/ sizeof(uint64_t);
+      EOS_ASSERT(file_.tellp() % sizeof(uint64_t) == 0, Exception,
+                 "The size of ${file} is not a multiple of sizeof(uint64_t)", ("file", path));
+   }
+
+   bool is_open() const { return file_.is_open(); }
+
+   uint64_t back() { return nth_block_position(num_blocks()-1); }
+   uint32_t num_blocks() const { return num_blocks_; }
+   uint64_t nth_block_position(uint32_t n) {
+      file_.seek(n*sizeof(uint64_t));
+      uint64_t r;
+      file_.read((char*)&r, sizeof(r));
+      return r;
+   }
+
+   void copy_to(fc::cfile& dest, uint64_t nbytes) {
+      file_.seek(0);
+      copy_file_content(file_, dest, nbytes);
+   }
+
+};
+
+} // namespace chain
+} // namespace eosio

libraries/chain/include/eosio/chain/name.hpp

@@ -75,6 +75,41 @@ namespace eosio::chain {
       friend constexpr bool operator != ( const name& a, uint64_t b ) { return a.value != b; }
 
       constexpr explicit operator bool()const { return value != 0; }
+
+      /**
+       *  Returns the prefix.
+       *  for exmaple:
+       *    "eosio.any" -> "eosio"
+       *    "eosio" -> "eosio"
+       */
+      constexpr name prefix() const {
+         uint64_t result                 = value;
+         bool     not_dot_character_seen = false;
+         uint64_t mask                   = 0xFull;
+
+         // Get characters one-by-one in name in order from right to left
+         for (int32_t offset = 0; offset <= 59;) {
+            auto c = (value >> offset) & mask;
+
+            if (!c) {                        // if this character is a dot
+               if (not_dot_character_seen) { // we found the rightmost dot character
+                  result = (value >> offset) << offset;
+                  break;
+               }
+            } else {
+               not_dot_character_seen = true;
+            }
+
+            if (offset == 0) {
+               offset += 4;
+               mask = 0x1Full;
+            } else {
+               offset += 5;
+            }
+         }
+
+         return name{ result };
+      }
    };
 
    // Each char of the string is encoded into 5-bit chunk and left-shifted

libraries/chain/include/eosio/chain/pending_snapshot.hpp

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <eosio/chain/controller.hpp>
+#include <eosio/chain/types.hpp>
+
+#include <string>
+
+namespace eosio::chain {
+
+namespace fs = std::filesystem;
+
+template<typename T>
+class pending_snapshot {
+public:
+   using next_t = eosio::chain::next_function<T>;
+
+   pending_snapshot(const chain::block_id_type& block_id, const next_t& next, std::string pending_path, std::string final_path)
+       : block_id(block_id), next(next), pending_path(std::move(pending_path)), final_path(std::move(final_path)) {}
+
+   uint32_t get_height() const {
+      return chain::block_header::num_from_id(block_id);
+   }
+
+   static fs::path get_final_path(const chain::block_id_type& block_id, const fs::path& snapshots_dir) {
+      return snapshots_dir / fc::format_string("snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id));
+   }
+
+   static fs::path get_pending_path(const chain::block_id_type& block_id, const fs::path& snapshots_dir) {
+      return snapshots_dir / fc::format_string(".pending-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id));
+   }
+
+   static fs::path get_temp_path(const chain::block_id_type& block_id, const fs::path& snapshots_dir) {
+      return snapshots_dir / fc::format_string(".incomplete-snapshot-${id}.bin", fc::mutable_variant_object()("id", block_id));
+   }
+
+   T finalize(const chain::controller& chain) const {
+      auto block_ptr = chain.fetch_block_by_id(block_id);
+      auto in_chain = (bool) block_ptr;
+      std::error_code ec;
+
+      if(!in_chain) {
+         fs::remove(fs::path(pending_path), ec);
+         EOS_THROW(chain::snapshot_finalization_exception,
+                   "Snapshotted block was forked out of the chain.  ID: ${block_id}",
+                   ("block_id", block_id));
+      }
+
+      fs::rename(fs::path(pending_path), fs::path(final_path), ec);
+      EOS_ASSERT(!ec, chain::snapshot_finalization_exception,
+                 "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}",
+                 ("bn", get_height())("ec", ec.value())("message", ec.message()));
+
+      return {block_id, block_ptr->block_num(), block_ptr->timestamp, chain::chain_snapshot_header::current_version, final_path};
+   }
+
+   chain::block_id_type block_id;
+   next_t next;
+   std::string pending_path;
+   std::string final_path;
+};
+}// namespace eosio::chain

libraries/chain/include/eosio/chain/producer_schedule.hpp

@@ -335,4 +335,3 @@ FC_REFLECT_DERIVED( eosio::chain::producer_schedule_change_extension, (eosio::ch
 FC_REFLECT( eosio::chain::shared_block_signing_authority_v0, (threshold)(keys))
 FC_REFLECT( eosio::chain::shared_producer_authority, (producer_name)(authority) )
 FC_REFLECT( eosio::chain::shared_producer_authority_schedule, (version)(producers) )
-

libraries/chain/include/eosio/chain/protocol_feature_manager.hpp

@@ -264,7 +264,7 @@ protected:
 class protocol_feature_manager {
 public:
 
-   protocol_feature_manager( protocol_feature_set&& pfs, std::function<deep_mind_handler*()> get_deep_mind_logger );
+   protocol_feature_manager( protocol_feature_set&& pfs, std::function<deep_mind_handler*(bool is_trx_transient)> get_deep_mind_logger );
 
    class const_iterator {
    public:
@@ -393,9 +393,12 @@ protected:
    bool                                   _initialized = false;
 
 private:
-   std::function<deep_mind_handler*()>           _get_deep_mind_logger;
+   std::function<deep_mind_handler*(bool is_trx_transient)> _get_deep_mind_logger;
 };
 
+std::optional<builtin_protocol_feature> read_builtin_protocol_feature( const std::filesystem::path& p  );
+protocol_feature_set initialize_protocol_features( const std::filesystem::path& p, bool populate_missing_builtins = true );
+
 } } // namespace eosio::chain
 
 FC_REFLECT(eosio::chain::protocol_feature_subjective_restrictions,

libraries/chain/include/eosio/chain/resource_limits.hpp

@@ -64,7 +64,7 @@ namespace eosio { namespace chain {
    class resource_limits_manager {
       public:
 
-         explicit resource_limits_manager(chainbase::database& db, std::function<deep_mind_handler*()> get_deep_mind_logger)
+         explicit resource_limits_manager(chainbase::database& db, std::function<deep_mind_handler*(bool is_trx_transient)> get_deep_mind_logger)
          :_db(db),_get_deep_mind_logger(get_deep_mind_logger)
          {
          }
@@ -74,17 +74,17 @@ namespace eosio { namespace chain {
          void add_to_snapshot( const snapshot_writer_ptr& snapshot ) const;
          void read_from_snapshot( const snapshot_reader_ptr& snapshot );
 
-         void initialize_account( const account_name& account );
+         void initialize_account( const account_name& account, bool is_trx_transient );
          void set_block_parameters( const elastic_limit_parameters& cpu_limit_parameters, const elastic_limit_parameters& net_limit_parameters );
 
          void update_account_usage( const flat_set<account_name>& accounts, uint32_t ordinal );
-         void add_transaction_usage( const flat_set<account_name>& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t ordinal );
+         void add_transaction_usage( const flat_set<account_name>& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t ordinal, bool is_trx_transient = false );
 
-         void add_pending_ram_usage( const account_name account, int64_t ram_delta );
+         void add_pending_ram_usage( const account_name account, int64_t ram_delta, bool is_trx_transient = false );
          void verify_account_ram_usage( const account_name accunt )const;
 
          /// set_account_limits returns true if new ram_bytes limit is more restrictive than the previously set one
-         bool set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight);
+         bool set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight, bool is_trx_transient);
          void get_account_limits( const account_name& account, int64_t& ram_bytes, int64_t& net_weight, int64_t& cpu_weight) const;
 
          bool is_unlimited_cpu( const account_name& account ) const;
@@ -114,7 +114,7 @@ namespace eosio { namespace chain {
 
       private:
          chainbase::database&         _db;
-         std::function<deep_mind_handler*()> _get_deep_mind_logger;
+         std::function<deep_mind_handler*(bool is_trx_transient)> _get_deep_mind_logger;
    };
 } } } /// eosio::chain
 

libraries/chain/include/eosio/chain/snapshot.hpp

@@ -376,7 +376,7 @@ namespace eosio { namespace chain {
 
    class istream_json_snapshot_reader : public snapshot_reader {
       public:
-         explicit istream_json_snapshot_reader(const fc::path& p);
+         explicit istream_json_snapshot_reader(const std::filesystem::path& p);
          ~istream_json_snapshot_reader();
 
          void validate() const override;

libraries/chain/include/eosio/chain/snapshot_scheduler.hpp

@@ -0,0 +1,223 @@
+#pragma once
+
+#include <eosio/chain/pending_snapshot.hpp>
+
+#include <eosio/chain/block_state.hpp>
+#include <eosio/chain/config.hpp>
+#include <eosio/chain/exceptions.hpp>
+#include <eosio/chain/resource_limits.hpp>
+#include <eosio/chain/resource_limits_private.hpp>
+#include <eosio/chain/transaction.hpp>
+#include <eosio/chain/types.hpp>
+
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/random_access_index.hpp>
+#include <boost/multi_index_container.hpp>
+
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/property_tree/json_parser.hpp>
+#include <boost/property_tree/ptree.hpp>
+
+#include <limits>
+
+namespace eosio::chain {
+
+namespace bmi = boost::multi_index;
+namespace fs = std::filesystem;
+
+class snapshot_scheduler {
+public:
+   struct snapshot_information {
+      chain::block_id_type head_block_id;
+      uint32_t head_block_num;
+      fc::time_point head_block_time;
+      uint32_t version;
+      std::string snapshot_name;
+   };
+
+   struct snapshot_request_information {
+      uint32_t block_spacing = 0;
+      uint32_t start_block_num = 0;
+      uint32_t end_block_num = std::numeric_limits<uint32_t>::max();
+      std::string snapshot_description = "";
+   };
+
+   // this struct used to hold request params in api call
+   // it is differentiate between 0 and empty values
+   struct snapshot_request_params {
+      std::optional<uint32_t> block_spacing;
+      std::optional<uint32_t> start_block_num;
+      std::optional<uint32_t> end_block_num;
+      std::optional<std::string> snapshot_description;
+   };
+
+   struct snapshot_request_id_information {
+      uint32_t snapshot_request_id = 0;
+   };
+
+   struct snapshot_schedule_result : public snapshot_request_id_information, public snapshot_request_information {
+   };
+
+   struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information {
+      std::vector<snapshot_information> pending_snapshots;
+   };
+
+   struct get_snapshot_requests_result {
+      std::vector<snapshot_schedule_information> snapshot_requests;
+   };
+
+   template<typename T>
+   using next_function = eosio::chain::next_function<T>;
+
+   struct by_height;
+
+   using pending_snapshot_index = bmi::multi_index_container<
+         pending_snapshot<snapshot_information>,
+         indexed_by<
+               bmi::hashed_unique<tag<by_id>, BOOST_MULTI_INDEX_MEMBER(pending_snapshot<snapshot_information>, block_id_type, block_id)>,
+               bmi::ordered_non_unique<tag<by_height>, BOOST_MULTI_INDEX_CONST_MEM_FUN(pending_snapshot<snapshot_information>, uint32_t, get_height)>>>;
+
+   class snapshot_db_json {
+   public:
+      snapshot_db_json() = default;
+      ~snapshot_db_json() = default;
+
+      void set_path(std::filesystem::path path) {
+         db_path = std::move(path);
+      }
+
+      std::filesystem::path get_json_path() const {
+         return db_path / "snapshot-schedule.json";
+      }
+
+      const snapshot_db_json& operator>>(std::vector<snapshot_schedule_information>& sr) {
+         boost::property_tree::ptree root;
+
+         try {
+            std::ifstream file(get_json_path().string());
+            file.exceptions(std::istream::failbit | std::istream::eofbit);
+            boost::property_tree::read_json(file, root);
+
+            // parse ptree
+            for(boost::property_tree::ptree::value_type& req: root.get_child("snapshot_requests")) {
+               snapshot_schedule_information ssi;
+               ssi.snapshot_request_id = req.second.get<uint32_t>("snapshot_request_id");
+               ssi.snapshot_description = req.second.get<std::string>("snapshot_description");
+               ssi.block_spacing = req.second.get<uint32_t>("block_spacing");
+               ssi.start_block_num = req.second.get<uint32_t>("start_block_num");
+               ssi.end_block_num = req.second.get<uint32_t>("end_block_num");
+               sr.push_back(ssi);
+            }
+         } catch(std::ifstream::failure& e) {
+            elog("unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}",
+                 ("jsonpath", get_json_path().string())("details", e.what()));
+         }
+
+         return *this;
+      }
+
+      const snapshot_db_json& operator<<(const std::vector<snapshot_schedule_information>& sr) const {
+         boost::property_tree::ptree root;
+         boost::property_tree::ptree node_srs;
+
+         for(const auto& key: sr) {
+            boost::property_tree::ptree node;
+            node.put("snapshot_request_id", key.snapshot_request_id);
+            node.put("snapshot_description", key.snapshot_description);
+            node.put("block_spacing", key.block_spacing);
+            node.put("start_block_num", key.start_block_num);
+            node.put("end_block_num", key.end_block_num);
+            node_srs.push_back(std::make_pair("", node));
+         }
+
+         root.push_back(std::make_pair("snapshot_requests", node_srs));
+
+         try {
+            std::ofstream file(get_json_path().string());
+            file.exceptions(std::istream::failbit | std::istream::eofbit);
+            boost::property_tree::write_json(file, root);
+         } catch(std::ofstream::failure& e) {
+            elog("unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}",
+                 ("jsonpath", get_json_path().string())("details", e.what()));
+         }
+
+         return *this;
+      }
+
+   private:
+      fs::path db_path;
+   };
+
+private:
+   struct by_snapshot_id;
+   struct by_snapshot_value;
+   struct as_vector;
+
+   using snapshot_requests = bmi::multi_index_container<
+         snapshot_schedule_information,
+         indexed_by<
+               bmi::hashed_unique<tag<by_snapshot_id>, BOOST_MULTI_INDEX_MEMBER(snapshot_request_id_information, uint32_t, snapshot_request_id)>,
+               bmi::random_access<tag<as_vector>>,
+               bmi::ordered_unique<tag<by_snapshot_value>,
+                                   composite_key<snapshot_schedule_information,
+                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, block_spacing),
+                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, start_block_num),
+                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, end_block_num)>>>>;
+   snapshot_requests _snapshot_requests;
+   snapshot_db_json _snapshot_db;
+   pending_snapshot_index _pending_snapshot_index;
+
+   uint32_t _snapshot_id = 0;
+   uint32_t _inflight_sid = 0;
+
+   // path to write the snapshots to
+   fs::path _snapshots_dir;
+
+   void x_serialize() {
+      auto& vec = _snapshot_requests.get<as_vector>();
+      std::vector<snapshot_schedule_information> sr(vec.begin(), vec.end());
+      _snapshot_db << sr;
+   };
+
+public:
+   snapshot_scheduler() = default;
+
+   // snapshot scheduler listener
+   void on_start_block(uint32_t height, chain::controller& chain);
+
+   // to promote pending snapshots
+   void on_irreversible_block(const signed_block_ptr& lib, const chain::controller& chain);
+
+   // snapshot scheduler handlers
+   snapshot_schedule_result schedule_snapshot(const snapshot_request_information& sri);
+   snapshot_schedule_result unschedule_snapshot(uint32_t sri);
+   get_snapshot_requests_result get_snapshot_requests();
+
+   // initialize with storage
+   void set_db_path(fs::path db_path);
+
+   // set snapshot path
+   void set_snapshots_path(fs::path sn_path);
+
+   // add pending snapshot info to inflight snapshot request
+   void add_pending_snapshot_info(const snapshot_information& si);
+
+   // execute snapshot
+   void execute_snapshot(uint32_t srid, chain::controller& chain);
+
+   // former producer_plugin snapshot fn
+   void create_snapshot(next_function<snapshot_information> next, chain::controller& chain, std::function<void(void)> predicate);
+};
+
+
+}// namespace eosio::chain
+
+FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_information, (head_block_id) (head_block_num) (head_block_time) (version) (snapshot_name))
+FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_information, (block_spacing) (start_block_num) (end_block_num) (snapshot_description))
+FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_params, (block_spacing) (start_block_num) (end_block_num) (snapshot_description))
+FC_REFLECT(eosio::chain::snapshot_scheduler::snapshot_request_id_information, (snapshot_request_id))
+FC_REFLECT(eosio::chain::snapshot_scheduler::get_snapshot_requests_result, (snapshot_requests))
+FC_REFLECT_DERIVED(eosio::chain::snapshot_scheduler::snapshot_schedule_information, (eosio::chain::snapshot_scheduler::snapshot_request_id_information)(eosio::chain::snapshot_scheduler::snapshot_request_information), (pending_snapshots))
+FC_REFLECT_DERIVED(eosio::chain::snapshot_scheduler::snapshot_schedule_result, (eosio::chain::snapshot_scheduler::snapshot_request_id_information)(eosio::chain::snapshot_scheduler::snapshot_request_information), )

libraries/chain/include/eosio/chain/subjective_billing.hpp

@@ -8,28 +8,23 @@
 #include <eosio/chain/resource_limits_private.hpp>
 #include <eosio/chain/config.hpp>
 
+#include <fc/time.hpp>
+
 #include <boost/multi_index_container.hpp>
 #include <boost/multi_index/member.hpp>
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
 
-namespace eosio {
-
-namespace bmi = boost::multi_index;
-using chain::transaction_id_type;
-using chain::account_name;
-using chain::block_state_ptr;
-using chain::packed_transaction;
-namespace config = chain::config;
+namespace eosio::chain {
 
 class subjective_billing {
 private:
 
    struct trx_cache_entry {
-      transaction_id_type     trx_id;
-      account_name            account;
-      int64_t                 subjective_cpu_bill;
-      fc::time_point          expiry;
+      chain::transaction_id_type trx_id;
+      chain::account_name        account;
+      int64_t                    subjective_cpu_bill = 0;
+      fc::time_point             expiry;
    };
    struct by_id;
    struct by_expiry;
@@ -37,7 +32,7 @@ private:
    using trx_cache_index = bmi::multi_index_container<
          trx_cache_entry,
          indexed_by<
-               bmi::hashed_unique<tag<by_id>, BOOST_MULTI_INDEX_MEMBER( trx_cache_entry, transaction_id_type, trx_id ) >,
+               bmi::hashed_unique<tag<by_id>, BOOST_MULTI_INDEX_MEMBER( trx_cache_entry, chain::transaction_id_type, trx_id ) >,
                ordered_non_unique<tag<by_expiry>, BOOST_MULTI_INDEX_MEMBER( trx_cache_entry, fc::time_point, expiry ) >
          >
    >;
@@ -45,26 +40,24 @@ private:
    using decaying_accumulator = chain::resource_limits::impl::exponential_decay_accumulator<>;
 
    struct subjective_billing_info {
-      uint64_t              pending_cpu_us;        // tracked cpu us for transactions that may still succeed in a block
+      uint64_t              pending_cpu_us = 0;    // tracked cpu us for transactions that may still succeed in a block
       decaying_accumulator  expired_accumulator;   // accumulator used to account for transactions that have expired
 
-      bool empty(uint32_t time_ordinal, uint32_t expired_accumulator_average_window) {
+      bool empty(uint32_t time_ordinal, uint32_t expired_accumulator_average_window) const {
          return pending_cpu_us == 0 && expired_accumulator.value_at(time_ordinal, expired_accumulator_average_window) == 0;
       }
    };
 
-   using account_subjective_bill_cache = std::map<account_name, subjective_billing_info>;
-   using block_subjective_bill_cache = std::map<account_name, uint64_t>;
+   using account_subjective_bill_cache = std::map<chain::account_name, subjective_billing_info>;
 
    bool                                      _disabled = false;
    trx_cache_index                           _trx_cache_index;
    account_subjective_bill_cache             _account_subjective_bill_cache;
-   block_subjective_bill_cache               _block_subjective_bill_cache;
    std::set<chain::account_name>             _disabled_accounts;
-   uint32_t                                  _expired_accumulator_average_window = config::account_cpu_usage_average_window_ms / subjective_time_interval_ms;
+   uint32_t                                  _expired_accumulator_average_window = chain::config::account_cpu_usage_average_window_ms / subjective_time_interval_ms;
 
 private:
-   uint32_t time_ordinal_for( const fc::time_point& t ) const {
+   static uint32_t time_ordinal_for( const fc::time_point& t ) {
       auto ordinal = t.time_since_epoch().count() / (1000U * (uint64_t)subjective_time_interval_ms);
       EOS_ASSERT(ordinal <= std::numeric_limits<uint32_t>::max(), chain::tx_resource_exhaustion, "overflow of quantized time in subjective billing");
       return ordinal;
@@ -88,11 +81,11 @@ private:
       }
    }
 
-   void remove_subjective_billing( const block_state_ptr& bsp, uint32_t time_ordinal ) {
+   void remove_subjective_billing( const chain::block_state_ptr& bsp, uint32_t time_ordinal ) {
       if( !_trx_cache_index.empty() ) {
          for( const auto& receipt : bsp->block->transactions ) {
-            if( std::holds_alternative<packed_transaction>(receipt.trx) ) {
-               const auto& pt = std::get<packed_transaction>(receipt.trx);
+            if( std::holds_alternative<chain::packed_transaction>(receipt.trx) ) {
+               const auto& pt = std::get<chain::packed_transaction>(receipt.trx);
                remove_subjective_billing( pt.id(), time_ordinal );
             }
          }
@@ -101,8 +94,8 @@ private:
 
 public: // public for tests
    static constexpr uint32_t subjective_time_interval_ms = 5'000;
-
-   void remove_subjective_billing( const transaction_id_type& trx_id, uint32_t time_ordinal ) {
+   size_t get_account_cache_size() const {return _account_subjective_bill_cache.size();}
+   void remove_subjective_billing( const chain::transaction_id_type& trx_id, uint32_t time_ordinal ) {
       auto& idx = _trx_cache_index.get<by_id>();
       auto itr = idx.find( trx_id );
       if( itr != idx.end() ) {
@@ -114,11 +107,10 @@ public: // public for tests
 public:
    void disable() { _disabled = true; }
    void disable_account( chain::account_name a ) { _disabled_accounts.emplace( a ); }
-   bool is_account_disabled(const account_name& a ) const { return _disabled || _disabled_accounts.count( a ); }
+   bool is_account_disabled(const chain::account_name& a ) const { return _disabled || _disabled_accounts.count( a ); }
 
-   /// @param in_pending_block pass true if pt's bill time is accounted for in the pending block
-   void subjective_bill( const transaction_id_type& id, const fc::time_point& expire, const account_name& first_auth,
-                         const fc::microseconds& elapsed, bool in_pending_block )
+   void subjective_bill( const chain::transaction_id_type& id, fc::time_point_sec expire,
+                         const chain::account_name& first_auth, const fc::microseconds& elapsed )
    {
       if( !_disabled && !_disabled_accounts.count( first_auth ) ) {
          int64_t bill = std::max<int64_t>( 0, elapsed.count() );
@@ -126,17 +118,14 @@ public:
                trx_cache_entry{id,
                                first_auth,
                                bill,
-                               expire} );
+                               expire.to_time_point()} );
          if( p.second ) {
             _account_subjective_bill_cache[first_auth].pending_cpu_us += bill;
-            if( in_pending_block ) {
-               _block_subjective_bill_cache[first_auth] += bill;
-            }
          }
       }
    }
 
-   void subjective_bill_failure( const account_name& first_auth, const fc::microseconds& elapsed, const fc::time_point& now )
+   void subjective_bill_failure( const chain::account_name& first_auth, const fc::microseconds& elapsed, const fc::time_point& now )
    {
       if( !_disabled && !_disabled_accounts.count( first_auth ) ) {
          int64_t bill = std::max<int64_t>( 0, elapsed.count() );
@@ -145,7 +134,7 @@ public:
       }
    }
 
-   int64_t get_subjective_bill( const account_name& first_auth, const fc::time_point& now ) const {
+   int64_t get_subjective_bill( const chain::account_name& first_auth, const fc::time_point& now ) const {
       if( _disabled || _disabled_accounts.count( first_auth ) ) return 0;
       const auto time_ordinal = time_ordinal_for(now);
       const subjective_billing_info* sub_bill_info = nullptr;
@@ -153,34 +142,28 @@ public:
       if( aitr != _account_subjective_bill_cache.end() ) {
          sub_bill_info = &aitr->second;
       }
-      uint64_t in_block_pending_cpu_us = 0;
-      auto bitr = _block_subjective_bill_cache.find( first_auth );
-      if( bitr != _block_subjective_bill_cache.end() ) {
-         in_block_pending_cpu_us = bitr->second;
-      }
 
       if (sub_bill_info) {
-         EOS_ASSERT(sub_bill_info->pending_cpu_us >= in_block_pending_cpu_us, chain::tx_resource_exhaustion, "Logic error subjective billing ${a}", ("a", first_auth) );
-         int64_t sub_bill = sub_bill_info->pending_cpu_us - in_block_pending_cpu_us + sub_bill_info->expired_accumulator.value_at(time_ordinal, _expired_accumulator_average_window );
+         int64_t sub_bill = sub_bill_info->pending_cpu_us + sub_bill_info->expired_accumulator.value_at(time_ordinal, _expired_accumulator_average_window );
          return sub_bill;
       } else {
          return 0;
       }
    }
 
-   void abort_block() {
-      _block_subjective_bill_cache.clear();
-   }
-
-   void on_block( fc::logger& log, const block_state_ptr& bsp, const fc::time_point& now ) {
+   void on_block( fc::logger& log, const chain::block_state_ptr& bsp, const fc::time_point& now ) {
       if( bsp == nullptr || _disabled ) return;
       const auto time_ordinal = time_ordinal_for(now);
       const auto orig_count = _account_subjective_bill_cache.size();
       remove_subjective_billing( bsp, time_ordinal );
-      fc_dlog( log, "Subjective billed accounts ${n} removed ${r}", ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
+      if (orig_count > 0) {
+         fc_dlog( log, "Subjective billed accounts ${n} removed ${r}",
+                  ("n", orig_count)("r", orig_count - _account_subjective_bill_cache.size()) );
+      }
    }
 
-   bool remove_expired( fc::logger& log, const fc::time_point& pending_block_time, const fc::time_point& now, const fc::time_point& deadline ) {
+   template <typename Yield>
+   bool remove_expired( fc::logger& log, const fc::time_point& pending_block_time, const fc::time_point& now, Yield&& yield ) {
       bool exhausted = false;
       auto& idx = _trx_cache_index.get<by_expiry>();
       if( !idx.empty() ) {
@@ -189,7 +172,7 @@ public:
          uint32_t num_expired = 0;
 
          while( !idx.empty() ) {
-            if( deadline <= fc::time_point::now() ) {
+            if( yield() ) {
                exhausted = true;
                break;
             }
@@ -216,4 +199,4 @@ public:
    }
 };
 
-} //eosio
+} //eosio::chain

libraries/chain/include/eosio/chain/symbol.hpp

@@ -1,12 +1,12 @@
 #pragma once
 #include <fc/exception/exception.hpp>
+#include <eosio/chain/exceptions.hpp>
 #include <eosio/chain/types.hpp>
 #include <eosio/chain/core_symbol.hpp>
 #include <string>
 #include <functional>
 
-namespace eosio {
-   namespace chain {
+namespace eosio::chain {
 
       /**
          class symbol represents a token and contains precision and name.
@@ -65,20 +65,7 @@ namespace eosio {
             explicit symbol(uint64_t v = CORE_SYMBOL): m_value(v) {
                EOS_ASSERT(valid(), symbol_type_exception, "invalid symbol: ${name}", ("name",name()));
             }
-            static symbol from_string(const string& from)
-            {
-               try {
-                  string s = fc::trim(from);
-                  EOS_ASSERT(!s.empty(), symbol_type_exception, "creating symbol from empty string");
-                  auto comma_pos = s.find(',');
-                  EOS_ASSERT(comma_pos != string::npos, symbol_type_exception, "missing comma in symbol");
-                  auto prec_part = s.substr(0, comma_pos);
-                  uint8_t p = fc::to_int64(prec_part);
-                  string name_part = s.substr(comma_pos + 1);
-                  EOS_ASSERT( p <= max_precision, symbol_type_exception, "precision ${p} should be <= 18", ("p", p));
-                  return symbol(string_to_symbol(p, name_part.c_str()));
-               } FC_CAPTURE_LOG_AND_RETHROW((from))
-            }
+            static symbol from_string(const string& from);
             uint64_t value() const { return m_value; }
             bool valid() const
             {
@@ -184,8 +171,7 @@ namespace eosio {
       {
          return std::tie(lhs.sym, lhs.contract) > std::tie(rhs.sym, rhs.contract);
       }
-   } // namespace chain
-} // namespace eosio
+} // namespace eosio::chain
 
 namespace fc {
    inline void to_variant(const eosio::chain::symbol& var, fc::variant& vo) { vo = var.to_string(); }

libraries/chain/include/eosio/chain/thread_utils.hpp

@@ -1,46 +1,166 @@
 #pragma once
 
+#include <eosio/chain/name.hpp>
+#include <fc/exception/exception.hpp>
+#include <fc/log/logger_config.hpp>
 #include <boost/asio/io_context.hpp>
-#include <boost/asio/thread_pool.hpp>
 #include <boost/asio/post.hpp>
 #include <future>
 #include <memory>
 #include <optional>
+#include <thread>
 
 namespace eosio { namespace chain {
 
    /**
-    * Wrapper class for boost asio thread pool and io_context run.
+    * Wrapper class for thread pool of boost asio io_context run.
     * Also names threads so that tools like htop can see thread name.
+    * Example: named_thread_pool<struct net> thread_pool;
+    *      or: struct net{}; named_thread_pool<net> thread_pool;
+    * @param NamePrefixTag is a type name appended with -## of thread.
+    *                    A short NamePrefixTag type name (6 chars or under) is recommended as console_appender uses
+    *                    9 chars for the thread name.
     */
+   template<typename NamePrefixTag>
    class named_thread_pool {
    public:
-      // name_prefix is name appended with -## of thread.
-      // short name_prefix (6 chars or under) is recommended as console_appender uses 9 chars for thread name
-      named_thread_pool( std::string name_prefix, size_t num_threads );
+      using on_except_t = std::function<void(const fc::exception& e)>;
+      using init_t = std::function<void()>;
 
-      // calls stop()
-      ~named_thread_pool();
+      named_thread_pool() = default;
+
+      ~named_thread_pool(){
+         stop();
+      }
 
       boost::asio::io_context& get_executor() { return _ioc; }
 
-      // destroy work guard, stop io_context, join thread_pool, and stop thread_pool
-      void stop();
+      /// Spawn threads, can be re-started after stop().
+      /// Assumes start()/stop() called from the same thread or externally protected.
+      /// Blocks until all threads are created and completed their init function, or an exception is thrown
+      ///  during thread startup or an init function. Exceptions thrown during these stages are rethrown from start()
+      ///  but some threads might still have been started. Calling stop() after such a failure is safe.
+      /// @param num_threads is number of threads spawned
+      /// @param on_except is the function to call if io_context throws an exception, is called from thread pool thread.
+      ///                  if an empty function then logs and rethrows exception on thread which will terminate. Not called
+      ///                  for exceptions during the init function (such exceptions are rethrown from start())
+      /// @param init is an optional function to call at startup to initialize any data.
+      /// @throw assert_exception if already started and not stopped.
+      void start( size_t num_threads, on_except_t on_except, init_t init = {} ) {
+         FC_ASSERT( !_ioc_work, "Thread pool already started" );
+         _ioc_work.emplace( boost::asio::make_work_guard( _ioc ) );
+         _ioc.restart();
+         _thread_pool.reserve( num_threads );
+
+         std::promise<void> start_complete;
+         std::atomic<uint32_t> threads_remaining = num_threads;
+         std::exception_ptr pending_exception;
+         std::mutex pending_exception_mutex;
+
+         try {
+            for( size_t i = 0; i < num_threads; ++i ) {
+               _thread_pool.emplace_back( std::thread( &named_thread_pool::run_thread, this, i, on_except, init, std::ref(start_complete),
+                                                       std::ref(threads_remaining), std::ref(pending_exception), std::ref(pending_exception_mutex) ) );
+            }
+         }
+         catch( ... ) {
+            /// only an exception from std::thread's ctor should end up here. shut down all threads to ensure no
+            ///  potential access to the promise, atomic, etc above performed after throwing out of start
+            stop();
+            throw;
+         }
+         start_complete.get_future().get();
+      }
+
+      /// destroy work guard, stop io_context, join thread_pool
+      void stop() {
+         _ioc_work.reset();
+         _ioc.stop();
+         for( auto& t : _thread_pool ) {
+            t.join();
+         }
+         _thread_pool.clear();
+      }
+
+   private:
+      void run_thread( size_t i, const on_except_t& on_except, const init_t& init, std::promise<void>& start_complete,
+                       std::atomic<uint32_t>& threads_remaining, std::exception_ptr& pending_exception, std::mutex& pending_exception_mutex ) {
+
+         std::string tn;
+
+         auto decrement_remaining = [&]() {
+            if( !--threads_remaining ) {
+               if( pending_exception )
+                  start_complete.set_exception( pending_exception );
+               else
+                  start_complete.set_value();
+            }
+         };
+
+         try {
+            try {
+               tn = boost::core::demangle(typeid(this).name());
+               auto offset = tn.rfind("::");
+               if (offset != std::string::npos)
+                  tn.erase(0, offset+2);
+               tn = tn.substr(0, tn.find('>')) + "-" + std::to_string( i );
+               fc::set_thread_name( tn );
+               if ( init )
+                  init();
+            } FC_LOG_AND_RETHROW()
+         }
+         catch( ... ) {
+            std::lock_guard<std::mutex> l( pending_exception_mutex );
+            pending_exception = std::current_exception();
+            decrement_remaining();
+            return;
+         }
+
+         decrement_remaining();
+
+         try {
+            _ioc.run();
+         } catch( const fc::exception& e ) {
+            if( on_except ) {
+               on_except( e );
+            } else {
+               elog( "Exiting thread ${t} on exception: ${e}", ("t", tn)("e", e.to_detail_string()) );
+               throw;
+            }
+         } catch( const std::exception& e ) {
+            fc::std_exception_wrapper se( FC_LOG_MESSAGE( warn, "${what}: ", ("what", e.what()) ),
+                                          std::current_exception(), BOOST_CORE_TYPEID( e ).name(), e.what() );
+            if( on_except ) {
+               on_except( se );
+            } else {
+               elog( "Exiting thread ${t} on exception: ${e}", ("t", tn)("e", se.to_detail_string()) );
+               throw;
+            }
+         } catch( ... ) {
+            if( on_except ) {
+               fc::unhandled_exception ue( FC_LOG_MESSAGE( warn, "unknown exception" ), std::current_exception() );
+               on_except( ue );
+            } else {
+               elog( "Exiting thread ${t} on unknown exception", ("t", tn) );
+               throw;
+            }
+         }
+      }
 
    private:
       using ioc_work_t = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
 
-      boost::asio::thread_pool       _thread_pool;
       boost::asio::io_context        _ioc;
+      std::vector<std::thread>       _thread_pool;
       std::optional<ioc_work_t>      _ioc_work;
    };
 
 
-   // async on thread_pool and return future
+   // async on io_context and return future
    template<typename F>
-   auto async_thread_pool( boost::asio::io_context& thread_pool, F&& f ) {
+   auto post_async_task( boost::asio::io_context& ioc, F&& f ) {
       auto task = std::make_shared<std::packaged_task<decltype( f() )()>>( std::forward<F>( f ) );
-      boost::asio::post( thread_pool, [task]() { (*task)(); } );
+      boost::asio::post( ioc, [task]() { (*task)(); } );
       return task->get_future();
    }
 

libraries/chain/include/eosio/chain/transaction_context.hpp

@@ -37,9 +37,10 @@ namespace eosio { namespace chain {
 
          transaction_context( controller& c,
                               const packed_transaction& t,
+                              const transaction_id_type& trx_id, // trx_id diff than t.id() before replace_deferred
                               transaction_checktime_timer&& timer,
                               fc::time_point start = fc::time_point::now(),
-                              bool read_only=false);
+                              transaction_metadata::trx_type type = transaction_metadata::trx_type::input);
          ~transaction_context();
 
          void init_for_implicit_trx( uint64_t initial_net_usage = 0 );
@@ -83,6 +84,10 @@ namespace eosio { namespace chain {
 
          void validate_referenced_accounts( const transaction& trx, bool enforce_actor_whitelist_blacklist )const;
 
+         bool is_dry_run()const { return trx_type == transaction_metadata::trx_type::dry_run; };
+         bool is_read_only()const { return trx_type == transaction_metadata::trx_type::read_only; };
+         bool is_transient()const { return trx_type == transaction_metadata::trx_type::read_only || trx_type == transaction_metadata::trx_type::dry_run; };
+
       private:
 
          friend struct controller_impl;
@@ -116,11 +121,14 @@ namespace eosio { namespace chain {
 
          void disallow_transaction_extensions( const char* error_msg )const;
 
+         std::string get_tx_cpu_usage_exceeded_reason_msg(fc::microseconds& limit) const;
+
       /// Fields:
       public:
 
          controller&                                 control;
          const packed_transaction&                   packed_trx;
+         const transaction_id_type&                  id;
          std::optional<chainbase::database::session> undo_session;
          transaction_trace_ptr                       trace;
          fc::time_point                              start;
@@ -148,9 +156,9 @@ namespace eosio { namespace chain {
 
          transaction_checktime_timer   transaction_timer;
 
-         const bool                    is_read_only;
    private:
          bool                          is_initialized = false;
+         transaction_metadata::trx_type trx_type;
 
          uint64_t                      net_limit = 0;
          bool                          net_limit_due_to_block = true;
@@ -169,6 +177,15 @@ namespace eosio { namespace chain {
          int64_t                       billing_timer_exception_code = block_cpu_usage_exceeded::code_value;
          fc::time_point                pseudo_start;
          fc::microseconds              billed_time;
+
+         enum class tx_cpu_usage_exceeded_reason {
+            account_cpu_limit, // includes subjective billing
+            on_chain_consensus_max_transaction_cpu_usage,
+            user_specified_trx_max_cpu_usage_ms,
+            node_configured_max_transaction_time,
+            speculative_executed_adjusted_max_transaction_time // prev_billed_cpu_time_us > 0
+         };
+         tx_cpu_usage_exceeded_reason  tx_cpu_usage_reason = tx_cpu_usage_exceeded_reason::account_cpu_limit;
    };
 
 } }

libraries/chain/include/eosio/chain/transaction_metadata.hpp

@@ -24,6 +24,7 @@ class transaction_metadata {
          input,
          implicit,
          scheduled,
+         dry_run,
          read_only
       };
 
@@ -31,11 +32,9 @@ class transaction_metadata {
       const packed_transaction_ptr                               _packed_trx;
       const fc::microseconds                                     _sig_cpu_usage;
       const flat_set<public_key_type>                            _recovered_pub_keys;
+      const trx_type                                             _trx_type;
 
    public:
-      const bool                                                 implicit;
-      const bool                                                 scheduled;
-      const bool                                                 read_only;
       bool                                                       accepted = false;       // not thread safe
       uint32_t                                                   billed_cpu_time_us = 0; // not thread safe
 
@@ -52,13 +51,11 @@ class transaction_metadata {
       // creation of tranaction_metadata restricted to start_recover_keys and create_no_recover_keys below, public for make_shared
       explicit transaction_metadata( const private_type& pt, packed_transaction_ptr ptrx,
                                      fc::microseconds sig_cpu_usage, flat_set<public_key_type> recovered_pub_keys,
-                                     bool _implicit = false, bool _scheduled = false, bool _read_only = false)
+                                     trx_type t = trx_type::input )
          : _packed_trx( std::move( ptrx ) )
          , _sig_cpu_usage( sig_cpu_usage )
          , _recovered_pub_keys( std::move( recovered_pub_keys ) )
-         , implicit( _implicit )
-         , scheduled( _scheduled )
-         , read_only( _read_only) {
+         , _trx_type( t ) {
       }
 
       transaction_metadata() = delete;
@@ -73,6 +70,12 @@ class transaction_metadata {
       fc::microseconds signature_cpu_usage()const { return _sig_cpu_usage; }
       const flat_set<public_key_type>& recovered_keys()const { return _recovered_pub_keys; }
       size_t get_estimated_size() const;
+      trx_type get_trx_type() const { return _trx_type; };
+      bool implicit() const { return _trx_type == trx_type::implicit; };
+      bool scheduled() const { return _trx_type == trx_type::scheduled; };
+      bool is_dry_run() const { return _trx_type == trx_type::dry_run; };
+      bool is_read_only() const { return _trx_type == trx_type::read_only; };
+      bool is_transient() const { return _trx_type == trx_type::read_only || _trx_type == trx_type::dry_run; };
 
       /// Thread safe.
       /// @returns transaction_metadata_ptr or exception via future
@@ -85,7 +88,7 @@ class transaction_metadata {
       static transaction_metadata_ptr
       create_no_recover_keys( packed_transaction_ptr trx, trx_type t ) {
          return std::make_shared<transaction_metadata>( private_type(), std::move(trx),
-               fc::microseconds(), flat_set<public_key_type>(), t == trx_type::implicit, t == trx_type::scheduled, t==trx_type::read_only );
+               fc::microseconds(), flat_set<public_key_type>(), t );
       }
 
 };

libraries/chain/include/eosio/chain/types.hpp

@@ -8,7 +8,6 @@
 #include <fc/io/varint.hpp>
 #include <fc/io/enum_type.hpp>
 #include <fc/crypto/sha224.hpp>
-#include <fc/safe.hpp>
 #include <fc/container/flat.hpp>
 #include <fc/string.hpp>
 #include <fc/io/raw.hpp>
@@ -43,7 +42,7 @@
 
 #define _V(n, v)  fc::mutable_variant_object(n, v)
 
-namespace eosio { namespace chain {
+namespace eosio::chain {
    using                               std::map;
    using                               std::vector;
    using                               std::unordered_map;
@@ -61,7 +60,7 @@ namespace eosio { namespace chain {
    using                               std::to_string;
    using                               std::all_of;
 
-   using                               fc::path;
+   using                               std::filesystem::path;
    using                               fc::variant_object;
    using                               fc::variant;
    using                               fc::enum_type;
@@ -69,7 +68,6 @@ namespace eosio { namespace chain {
    using                               fc::signed_int;
    using                               fc::time_point_sec;
    using                               fc::time_point;
-   using                               fc::safe;
    using                               fc::flat_map;
    using                               fc::flat_multimap;
    using                               fc::flat_set;
@@ -78,15 +76,12 @@ namespace eosio { namespace chain {
    using public_key_type  = fc::crypto::public_key;
    using private_key_type = fc::crypto::private_key;
    using signature_type   = fc::crypto::signature;
-#if BOOST_VERSION >= 107100
-      // configurable boost deque performs much better than std::deque in our use cases
-      using block_1024_option_t = boost::container::deque_options< boost::container::block_size<1024u> >::type;
-      template<typename T>
-      using deque = boost::container::deque< T, void, block_1024_option_t >;
-#else
-      template<typename T>
-    using deque = std::deque<T>;
-#endif
+   
+   // configurable boost deque (for boost >= 1.71) performs much better than std::deque in our use cases
+   using block_1024_option_t = boost::container::deque_options< boost::container::block_size<1024u> >::type;
+   template<typename T>
+   using deque = boost::container::deque< T, void, block_1024_option_t >;
+   
    struct void_t{};
 
    using chainbase::allocator;
@@ -375,7 +370,7 @@ namespace eosio { namespace chain {
    }
 
    template<typename E, typename F>
-   static inline auto has_field( F flags, E field )
+   static constexpr auto has_field( F flags, E field )
    -> std::enable_if_t< std::is_integral<F>::value && std::is_unsigned<F>::value &&
                         std::is_enum<E>::value && std::is_same< F, std::underlying_type_t<E> >::value, bool>
    {
@@ -383,7 +378,7 @@ namespace eosio { namespace chain {
    }
 
    template<typename E, typename F>
-   static inline auto set_field( F flags, E field, bool value = true )
+   static constexpr auto set_field( F flags, E field, bool value = true )
    -> std::enable_if_t< std::is_integral<F>::value && std::is_unsigned<F>::value &&
                         std::is_enum<E>::value && std::is_same< F, std::underlying_type_t<E> >::value, F >
    {
@@ -396,7 +391,24 @@ namespace eosio { namespace chain {
    template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
    template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 
-} }  // eosio::chain
+   // next_function is a function passed to an API (like send_transaction) and which is called at the end of
+   // the API processing on the main thread. The type T is a description of the API result that can be
+   // serialized as output.
+   // The function accepts a variant which can contain an exception_ptr (if an exception occured while
+   // processing the API) or the result T.
+   // The third option is a function which can be executed in a multithreaded context (likely on the
+   // http_plugin thread pool) and which completes the API processing and returns the result T.
+   // -------------------------------------------------------------------------------------------------------
+   template<typename T>
+   using t_or_exception = std::variant<T, fc::exception_ptr>;
+
+   template<typename T>
+   using next_function_variant = std::variant<fc::exception_ptr, T, std::function<t_or_exception<T>()>>;
+
+   template<typename T>
+   using next_function = std::function<void(const next_function_variant<T>&)>;
+
+}  // eosio::chain
 
 namespace chainbase {
    // chainbase::shared_cow_string

libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp

@@ -21,14 +21,13 @@ using namespace boost::multi_index;
 
 enum class trx_enum_type {
    unknown = 0,
-   persisted = 1,
-   forked = 2,
-   aborted = 3,
-   incoming_persisted = 4,
-   incoming = 5 // incoming_end() needs to be updated if this changes
+   forked = 1,
+   aborted = 2,
+   incoming_api = 3,
+   incoming_p2p = 4 // incoming_end() needs to be updated if this changes
 };
 
-using next_func_t = std::function<void(const std::variant<fc::exception_ptr, transaction_trace_ptr>&)>;
+using next_func_t = next_function<transaction_trace_ptr>;
 
 struct unapplied_transaction {
    const transaction_metadata_ptr trx_meta;
@@ -38,16 +37,10 @@ struct unapplied_transaction {
 
    const transaction_id_type& id()const { return trx_meta->id(); }
    fc::time_point_sec expiration()const { return trx_meta->packed_trx()->expiration(); }
-
-   unapplied_transaction(const unapplied_transaction&) = delete;
-   unapplied_transaction() = delete;
-   unapplied_transaction& operator=(const unapplied_transaction&) = delete;
-   unapplied_transaction(unapplied_transaction&&) = default;
 };
 
 /**
- * Track unapplied transactions for persisted, forked blocks, and aborted blocks.
- * Persisted are first so that they can be applied in each block until expired.
+ * Track unapplied transactions for incoming, forked blocks, and aborted blocks.
  */
 class unapplied_transaction_queue {
 private:
@@ -96,15 +89,15 @@ public:
       return itr->trx_meta;
    }
 
-   template <typename Func>
-   bool clear_expired( const time_point& pending_block_time, const time_point& deadline, Func&& callback ) {
+   template <typename Yield, typename Callback>
+   bool clear_expired( const time_point& pending_block_time, Yield&& yield, Callback&& callback ) {
       auto& persisted_by_expiry = queue.get<by_expiry>();
       while( !persisted_by_expiry.empty() ) {
          const auto& itr = persisted_by_expiry.begin();
-         if( itr->expiration() > pending_block_time ) {
+         if( itr->expiration().to_time_point() > pending_block_time ) {
             break;
          }
-         if( deadline <= fc::time_point::now() ) {
+         if( yield() ) {
             return false;
          }
          callback( itr->trx_meta->packed_trx(), itr->trx_type );
@@ -159,26 +152,11 @@ public:
       }
    }
 
-   void add_persisted( const transaction_metadata_ptr& trx ) {
-      auto itr = queue.get<by_trx_id>().find( trx->id() );
-      if( itr == queue.get<by_trx_id>().end() ) {
-         auto insert_itr = queue.insert( { trx, trx_enum_type::persisted } );
-         if( insert_itr.second ) added( insert_itr.first );
-      } else if( itr->trx_type != trx_enum_type::persisted ) {
-         if (itr->trx_type == trx_enum_type::incoming || itr->trx_type == trx_enum_type::incoming_persisted)
-            --incoming_count;
-         queue.get<by_trx_id>().modify( itr, [](auto& un){
-            un.trx_type = trx_enum_type::persisted;
-            un.next = nullptr; // persisted already have ack'ed initial trace
-         } );
-      }
-   }
-
-   void add_incoming( const transaction_metadata_ptr& trx, bool persist_until_expired, bool return_failure_trace, next_func_t next ) {
+   void add_incoming( const transaction_metadata_ptr& trx, bool api_trx, bool return_failure_trace, next_func_t next ) {
       auto itr = queue.get<by_trx_id>().find( trx->id() );
       if( itr == queue.get<by_trx_id>().end() ) {
          auto insert_itr = queue.insert(
-               { trx, persist_until_expired ? trx_enum_type::incoming_persisted : trx_enum_type::incoming, return_failure_trace, std::move( next ) } );
+               { trx, api_trx ? trx_enum_type::incoming_api : trx_enum_type::incoming_p2p, return_failure_trace, std::move( next ) } );
          if( insert_itr.second ) added( insert_itr.first );
       } else {
          if( itr->trx_meta == trx ) return; // same trx meta pointer
@@ -194,14 +172,11 @@ public:
    iterator begin() { return queue.get<by_type>().begin(); }
    iterator end() { return queue.get<by_type>().end(); }
 
-   // persisted, forked, aborted
+   // forked, aborted
    iterator unapplied_begin() { return queue.get<by_type>().begin(); }
    iterator unapplied_end() { return queue.get<by_type>().upper_bound( trx_enum_type::aborted ); }
 
-   iterator persisted_begin() { return queue.get<by_type>().lower_bound( trx_enum_type::persisted ); }
-   iterator persisted_end() { return queue.get<by_type>().upper_bound( trx_enum_type::persisted ); }
-
-   iterator incoming_begin() { return queue.get<by_type>().lower_bound( trx_enum_type::incoming_persisted ); }
+   iterator incoming_begin() { return queue.get<by_type>().lower_bound( trx_enum_type::incoming_api ); }
    iterator incoming_end() { return queue.get<by_type>().end(); } // if changed to upper_bound, verify usage performance
 
    iterator lower_bound( const transaction_id_type& id ) {
@@ -220,7 +195,7 @@ private:
    template<typename Itr>
    void added( Itr itr ) {
       auto size = calc_size( itr->trx_meta );
-      if( itr->trx_type == trx_enum_type::incoming || itr->trx_type == trx_enum_type::incoming_persisted ) {
+      if( itr->trx_type == trx_enum_type::incoming_p2p || itr->trx_type == trx_enum_type::incoming_api ) {
          ++incoming_count;
          EOS_ASSERT( size_in_bytes + size < max_transaction_queue_size, tx_resource_exhaustion,
                      "Transaction ${id}, size ${s} bytes would exceed configured "
@@ -233,7 +208,7 @@ private:
 
    template<typename Itr>
    void removed( Itr itr ) {
-      if( itr->trx_type == trx_enum_type::incoming || itr->trx_type == trx_enum_type::incoming_persisted ) {
+      if( itr->trx_type == trx_enum_type::incoming_p2p || itr->trx_type == trx_enum_type::incoming_api ) {
          --incoming_count;
       }
       size_in_bytes -= calc_size( itr->trx_meta );

libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp

@@ -168,121 +168,121 @@ namespace eosio { namespace chain { namespace wasm_injections {
    constexpr const char* inject_which_op( uint16_t opcode ) {
       switch ( opcode ) {
          case wasm_ops::f32_add_code:
-            return u8"_eosio_f32_add";
+            return "_eosio_f32_add";
          case wasm_ops::f32_sub_code:
-            return u8"_eosio_f32_sub";
+            return "_eosio_f32_sub";
          case wasm_ops::f32_mul_code:
-            return u8"_eosio_f32_mul";
+            return "_eosio_f32_mul";
          case wasm_ops::f32_div_code:
-            return u8"_eosio_f32_div";
+            return "_eosio_f32_div";
          case wasm_ops::f32_min_code:
-            return u8"_eosio_f32_min";
+            return "_eosio_f32_min";
          case wasm_ops::f32_max_code:
-            return u8"_eosio_f32_max";
+            return "_eosio_f32_max";
          case wasm_ops::f32_copysign_code:
-            return u8"_eosio_f32_copysign";
+            return "_eosio_f32_copysign";
          case wasm_ops::f32_abs_code:
-            return u8"_eosio_f32_abs";
+            return "_eosio_f32_abs";
          case wasm_ops::f32_neg_code:
-            return u8"_eosio_f32_neg";
+            return "_eosio_f32_neg";
          case wasm_ops::f32_sqrt_code:
-            return u8"_eosio_f32_sqrt";
+            return "_eosio_f32_sqrt";
          case wasm_ops::f32_ceil_code:
-            return u8"_eosio_f32_ceil";
+            return "_eosio_f32_ceil";
          case wasm_ops::f32_floor_code:
-            return u8"_eosio_f32_floor";
+            return "_eosio_f32_floor";
          case wasm_ops::f32_trunc_code:
-            return u8"_eosio_f32_trunc";
+            return "_eosio_f32_trunc";
          case wasm_ops::f32_nearest_code:
-            return u8"_eosio_f32_nearest";
+            return "_eosio_f32_nearest";
          case wasm_ops::f32_eq_code:
-            return u8"_eosio_f32_eq";
+            return "_eosio_f32_eq";
          case wasm_ops::f32_ne_code:
-            return u8"_eosio_f32_ne";
+            return "_eosio_f32_ne";
          case wasm_ops::f32_lt_code:
-            return u8"_eosio_f32_lt";
+            return "_eosio_f32_lt";
          case wasm_ops::f32_le_code:
-            return u8"_eosio_f32_le";
+            return "_eosio_f32_le";
          case wasm_ops::f32_gt_code:
-            return u8"_eosio_f32_gt";
+            return "_eosio_f32_gt";
          case wasm_ops::f32_ge_code:
-            return u8"_eosio_f32_ge";
+            return "_eosio_f32_ge";
          case wasm_ops::f64_add_code:
-            return u8"_eosio_f64_add";
+            return "_eosio_f64_add";
          case wasm_ops::f64_sub_code:
-            return u8"_eosio_f64_sub";
+            return "_eosio_f64_sub";
          case wasm_ops::f64_mul_code:
-            return u8"_eosio_f64_mul";
+            return "_eosio_f64_mul";
          case wasm_ops::f64_div_code:
-            return u8"_eosio_f64_div";
+            return "_eosio_f64_div";
          case wasm_ops::f64_min_code:
-            return u8"_eosio_f64_min";
+            return "_eosio_f64_min";
          case wasm_ops::f64_max_code:
-            return u8"_eosio_f64_max";
+            return "_eosio_f64_max";
          case wasm_ops::f64_copysign_code:
-            return u8"_eosio_f64_copysign";
+            return "_eosio_f64_copysign";
          case wasm_ops::f64_abs_code:
-            return u8"_eosio_f64_abs";
+            return "_eosio_f64_abs";
          case wasm_ops::f64_neg_code:
-            return u8"_eosio_f64_neg";
+            return "_eosio_f64_neg";
          case wasm_ops::f64_sqrt_code:
-            return u8"_eosio_f64_sqrt";
+            return "_eosio_f64_sqrt";
          case wasm_ops::f64_ceil_code:
-            return u8"_eosio_f64_ceil";
+            return "_eosio_f64_ceil";
          case wasm_ops::f64_floor_code:
-            return u8"_eosio_f64_floor";
+            return "_eosio_f64_floor";
          case wasm_ops::f64_trunc_code:
-            return u8"_eosio_f64_trunc";
+            return "_eosio_f64_trunc";
          case wasm_ops::f64_nearest_code:
-            return u8"_eosio_f64_nearest";
+            return "_eosio_f64_nearest";
          case wasm_ops::f64_eq_code:
-            return u8"_eosio_f64_eq";
+            return "_eosio_f64_eq";
          case wasm_ops::f64_ne_code:
-            return u8"_eosio_f64_ne";
+            return "_eosio_f64_ne";
          case wasm_ops::f64_lt_code:
-            return u8"_eosio_f64_lt";
+            return "_eosio_f64_lt";
          case wasm_ops::f64_le_code:
-            return u8"_eosio_f64_le";
+            return "_eosio_f64_le";
          case wasm_ops::f64_gt_code:
-            return u8"_eosio_f64_gt";
+            return "_eosio_f64_gt";
          case wasm_ops::f64_ge_code:
-            return u8"_eosio_f64_ge";
+            return "_eosio_f64_ge";
          case wasm_ops::f64_promote_f32_code:
-            return u8"_eosio_f32_promote";
+            return "_eosio_f32_promote";
          case wasm_ops::f32_demote_f64_code:
-            return u8"_eosio_f64_demote";
+            return "_eosio_f64_demote";
          case wasm_ops::i32_trunc_u_f32_code:
-            return u8"_eosio_f32_trunc_i32u";
+            return "_eosio_f32_trunc_i32u";
          case wasm_ops::i32_trunc_s_f32_code:
-            return u8"_eosio_f32_trunc_i32s";
+            return "_eosio_f32_trunc_i32s";
          case wasm_ops::i32_trunc_u_f64_code:
-            return u8"_eosio_f64_trunc_i32u";
+            return "_eosio_f64_trunc_i32u";
          case wasm_ops::i32_trunc_s_f64_code:
-            return u8"_eosio_f64_trunc_i32s";
+            return "_eosio_f64_trunc_i32s";
          case wasm_ops::i64_trunc_u_f32_code:
-            return u8"_eosio_f32_trunc_i64u";
+            return "_eosio_f32_trunc_i64u";
          case wasm_ops::i64_trunc_s_f32_code:
-            return u8"_eosio_f32_trunc_i64s";
+            return "_eosio_f32_trunc_i64s";
          case wasm_ops::i64_trunc_u_f64_code:
-            return u8"_eosio_f64_trunc_i64u";
+            return "_eosio_f64_trunc_i64u";
          case wasm_ops::i64_trunc_s_f64_code:
-            return u8"_eosio_f64_trunc_i64s";
+            return "_eosio_f64_trunc_i64s";
          case wasm_ops::f32_convert_s_i32_code:
-            return u8"_eosio_i32_to_f32";
+            return "_eosio_i32_to_f32";
          case wasm_ops::f32_convert_u_i32_code:
-            return u8"_eosio_ui32_to_f32";
+            return "_eosio_ui32_to_f32";
          case wasm_ops::f32_convert_s_i64_code:
-            return u8"_eosio_i64_f32";
+            return "_eosio_i64_f32";
          case wasm_ops::f32_convert_u_i64_code:
-            return u8"_eosio_ui64_to_f32";
+            return "_eosio_ui64_to_f32";
          case wasm_ops::f64_convert_s_i32_code:
-            return u8"_eosio_i32_to_f64";
+            return "_eosio_i32_to_f64";
          case wasm_ops::f64_convert_u_i32_code:
-            return u8"_eosio_ui32_to_f64";
+            return "_eosio_ui32_to_f64";
          case wasm_ops::f64_convert_s_i64_code:
-            return u8"_eosio_i64_to_f64";
+            return "_eosio_i64_to_f64";
          case wasm_ops::f64_convert_u_i64_code:
-            return u8"_eosio_ui64_to_f64";
+            return "_eosio_ui64_to_f64";
 
          default:
             FC_THROW_EXCEPTION( eosio::chain::wasm_execution_error, "Error, unknown opcode in injection ${op}", ("op", opcode));
@@ -483,7 +483,7 @@ namespace eosio { namespace chain { namespace wasm_injections {
       static void init() {}
       static void accept( wasm_ops::instr* inst, wasm_ops::visitor_arg& arg ) {
          int32_t idx;
-         injector_utils::add_import<ResultType::f64, ValueType::f32>( *(arg.module), u8"_eosio_f32_promote", idx );
+         injector_utils::add_import<ResultType::f64, ValueType::f32>( *(arg.module), "_eosio_f32_promote", idx );
          wasm_ops::op_types<>::call_t f32promote;
          f32promote.field = idx;
          f32promote.pack(arg.new_code);
@@ -496,7 +496,7 @@ namespace eosio { namespace chain { namespace wasm_injections {
       static void init() {}
       static void accept( wasm_ops::instr* inst, wasm_ops::visitor_arg& arg ) {
          int32_t idx;
-         injector_utils::add_import<ResultType::f32, ValueType::f64>( *(arg.module), u8"_eosio_f64_demote", idx );
+         injector_utils::add_import<ResultType::f32, ValueType::f64>( *(arg.module), "_eosio_f64_demote", idx );
          wasm_ops::op_types<>::call_t f32promote;
          f32promote.field = idx;
          f32promote.pack(arg.new_code);

libraries/chain/include/eosio/chain/wasm_eosio_validation.hpp

@@ -332,7 +332,7 @@ namespace eosio { namespace chain { namespace wasm_validations {
       public:
          wasm_binary_validation( const eosio::chain::controller& control, IR::Module& mod ) : _module( &mod ) {
             // initialize validators here
-            nested_validator::init(!control.is_producing_block());
+            nested_validator::init(!control.is_speculative_block());
          }
 
          void validate() {

libraries/chain/include/eosio/chain/wasm_interface.hpp

@@ -4,8 +4,6 @@
 #include <eosio/chain/whitelisted_intrinsics.hpp>
 #include <eosio/chain/exceptions.hpp>
 #include <functional>
-#include "Runtime/Linker.h"
-#include "Runtime/Runtime.h"
 
 namespace eosio { namespace chain {
 
@@ -42,9 +40,18 @@ namespace eosio { namespace chain {
              }
          }
 
-         wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile);
+         enum class vm_oc_enable {
+            oc_auto,
+            oc_all,
+            oc_none
+         };
+
+         wasm_interface(vm_type vm, vm_oc_enable eosvmoc_tierup, const chainbase::database& d, const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile);
          ~wasm_interface();
 
+         // initialize exec per thread
+         void init_thread_local_data();
+
          //call before dtor to skip what can be minutes of dtor overhead with some runtimes; can cause leaks
          void indicate_shutting_down();
 
@@ -60,8 +67,8 @@ namespace eosio { namespace chain {
          //Calls apply or error on a given code
          void apply(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, apply_context& context);
 
-         //Immediately exits currently running wasm. UB is called when no wasm running
-         void exit();
+         //Returns true if the code is cached
+         bool is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const;
 
          // If substitute_apply is set, then apply calls it before doing anything else. If substitute_apply returns true,
          // then apply returns immediately.
@@ -75,6 +82,16 @@ namespace eosio { namespace chain {
 
 namespace eosio{ namespace chain {
    std::istream& operator>>(std::istream& in, wasm_interface::vm_type& runtime);
+   inline std::ostream& operator<<(std::ostream& os, wasm_interface::vm_oc_enable t) {
+      if (t == wasm_interface::vm_oc_enable::oc_auto) {
+         os << "auto";
+      } else if (t == wasm_interface::vm_oc_enable::oc_all) {
+         os << "all";
+      } else if (t == wasm_interface::vm_oc_enable::oc_none) {
+         os << "none";
+      }
+      return os;
+   }
 }}
 
 FC_REFLECT_ENUM( eosio::chain::wasm_interface::vm_type, (eos_vm)(eos_vm_jit)(eos_vm_oc) )

libraries/chain/include/eosio/chain/wasm_interface_collection.hpp

@@ -0,0 +1,85 @@
+#pragma once
+#include <eosio/chain/wasm_interface.hpp>
+
+namespace eosio::chain {
+
+   /**
+    * @class wasm_interface_collection manages the active wasm_interface to use for execution.
+    */
+   class wasm_interface_collection {
+      public:
+         wasm_interface_collection(wasm_interface::vm_type vm, wasm_interface::vm_oc_enable eosvmoc_tierup,
+                                   const chainbase::database& d, const std::filesystem::path& data_dir,
+                                   const eosvmoc::config& eosvmoc_config, bool profile)
+            : main_thread_id(std::this_thread::get_id())
+            , wasm_runtime(vm)
+            , eosvmoc_tierup(eosvmoc_tierup)
+            , wasmif(vm, eosvmoc_tierup, d, data_dir, eosvmoc_config, profile)
+         {}
+
+         wasm_interface& get_wasm_interface() {
+            if (is_on_main_thread()
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+                || is_eos_vm_oc_enabled()
+#endif
+            )
+               return wasmif;
+            return *threaded_wasmifs[std::this_thread::get_id()];
+         }
+
+         // update current lib of all wasm interfaces
+         void current_lib(const uint32_t lib) {
+            // producer_plugin has already asserted irreversible_block signal is called in write window
+            wasmif.current_lib(lib);
+            for (auto& w: threaded_wasmifs) {
+               w.second->current_lib(lib);
+            }
+         }
+
+         // only called from non-main threads (read-only trx execution threads) when producer_plugin starts them
+         void init_thread_local_data(const chainbase::database& d, const std::filesystem::path& data_dir,
+                                     const eosvmoc::config& eosvmoc_config, bool profile) {
+            EOS_ASSERT(!is_on_main_thread(), misc_exception, "init_thread_local_data called on the main thread");
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+            if (is_eos_vm_oc_enabled()) {
+               // EOSVMOC needs further initialization of its thread local data
+               wasmif.init_thread_local_data();
+            } else
+#endif
+            {
+               std::lock_guard g(threaded_wasmifs_mtx);
+               // Non-EOSVMOC needs a wasmif per thread
+               threaded_wasmifs[std::this_thread::get_id()] = std::make_unique<wasm_interface>(wasm_runtime, eosvmoc_tierup, d, data_dir, eosvmoc_config, profile);
+            }
+         }
+
+#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
+         bool is_eos_vm_oc_enabled() const {
+            return ((eosvmoc_tierup != wasm_interface::vm_oc_enable::oc_none) || wasm_runtime == wasm_interface::vm_type::eos_vm_oc);
+         }
+#endif
+
+         void code_block_num_last_used(const digest_type& code_hash, uint8_t vm_type, uint8_t vm_version, uint32_t block_num) {
+            // The caller of this function apply_eosio_setcode has already asserted that
+            // the transaction is not a read-only trx, which implies we are
+            // in write window. Safe to call threaded_wasmifs's code_block_num_last_used
+            wasmif.code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+            for (auto& w: threaded_wasmifs) {
+               w.second->code_block_num_last_used(code_hash, vm_type, vm_version, block_num);
+            }
+         }
+
+      private:
+         bool is_on_main_thread() { return main_thread_id == std::this_thread::get_id(); };
+
+      private:
+         const std::thread::id main_thread_id;
+         const wasm_interface::vm_type wasm_runtime;
+         const wasm_interface::vm_oc_enable eosvmoc_tierup;
+
+         wasm_interface wasmif;  // used by main thread and all threads for EOSVMOC
+         std::mutex threaded_wasmifs_mtx;
+         std::unordered_map<std::thread::id, std::unique_ptr<wasm_interface>> threaded_wasmifs; // one for each read-only thread, used by eos-vm and eos-vm-jit
+   };
+
+} // eosio::chain

libraries/chain/include/eosio/chain/wasm_interface_private.hpp

@@ -15,7 +15,6 @@
 #include <fc/scoped_exit.hpp>
 
 #include "IR/Module.h"
-#include "Runtime/Intrinsics.h"
 #include "Platform/Platform.h"
 #include "WAST/WAST.h"
 #include "IR/Validate.h"
@@ -26,7 +25,6 @@
 using namespace fc;
 using namespace eosio::chain::webassembly;
 using namespace IR;
-using namespace Runtime;
 
 using boost::multi_index_container;
 
@@ -37,28 +35,41 @@ namespace eosio { namespace chain {
    struct wasm_interface_impl {
       struct wasm_cache_entry {
          digest_type                                          code_hash;
-         uint32_t                                             first_block_num_used;
          uint32_t                                             last_block_num_used;
          std::unique_ptr<wasm_instantiated_module_interface>  module;
          uint8_t                                              vm_type = 0;
          uint8_t                                              vm_version = 0;
       };
       struct by_hash;
-      struct by_first_block_num;
       struct by_last_block_num;
 
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       struct eosvmoc_tier {
-         eosvmoc_tier(const boost::filesystem::path& d, const eosvmoc::config& c, const chainbase::database& db)
-          : cc(d, c, db), exec(cc),
-            mem(wasm_constraints::maximum_linear_memory/wasm_constraints::wasm_page_size) {}
+         eosvmoc_tier(const std::filesystem::path& d, const eosvmoc::config& c, const chainbase::database& db)
+          : cc(d, c, db) {
+             // construct exec for the main thread
+             init_thread_local_data();
+          }
+
+         // Support multi-threaded execution.
+         void init_thread_local_data() {
+            exec = std::make_unique<eosvmoc::executor>(cc);
+         }
+
          eosvmoc::code_cache_async cc;
-         eosvmoc::executor exec;
-         eosvmoc::memory mem;
+
+         // Each thread requires its own exec and mem. Defined in wasm_interface.cpp
+         thread_local static std::unique_ptr<eosvmoc::executor> exec;
+         thread_local static eosvmoc::memory mem;
       };
 #endif
 
-      wasm_interface_impl(wasm_interface::vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile) : db(d), wasm_runtime_time(vm) {
+      wasm_interface_impl(wasm_interface::vm_type vm, wasm_interface::vm_oc_enable eosvmoc_tierup, const chainbase::database& d,
+                          const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, bool profile)
+         : db(d)
+         , wasm_runtime_time(vm)
+         , eosvmoc_tierup(eosvmoc_tierup)
+      {
 #ifdef EOSIO_EOS_VM_RUNTIME_ENABLED
          if(vm == wasm_interface::vm_type::eos_vm)
             runtime_interface = std::make_unique<webassembly::eos_vm_runtime::eos_vm_runtime<eosio::vm::interpreter>>();
@@ -79,7 +90,7 @@ namespace eosio { namespace chain {
             EOS_THROW(wasm_exception, "${r} wasm runtime not supported on this platform and/or configuration", ("r", vm));
 
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
-         if(eosvmoc_tierup) {
+         if(eosvmoc_tierup != wasm_interface::vm_oc_enable::oc_none) {
             EOS_ASSERT(vm != wasm_interface::vm_type::eos_vm_oc, wasm_exception, "You can't use EOS VM OC as the base runtime when tier up is activated");
             eosvmoc.emplace(data_dir, eosvmoc_config, d);
          }
@@ -94,22 +105,9 @@ namespace eosio { namespace chain {
                });
       }
 
-      std::vector<uint8_t> parse_initial_memory(const Module& module) {
-         std::vector<uint8_t> mem_image;
-
-         for(const DataSegment& data_segment : module.dataSegments) {
-            EOS_ASSERT(data_segment.baseOffset.type == InitializerExpression::Type::i32_const, wasm_exception, "");
-            EOS_ASSERT(module.memories.defs.size(), wasm_exception, "");
-            const U32 base_offset = data_segment.baseOffset.i32;
-            const Uptr memory_size = (module.memories.defs[0].type.size.min << IR::numBytesPerPageLog2);
-            if(base_offset >= memory_size || base_offset + data_segment.data.size() > memory_size)
-               FC_THROW_EXCEPTION(wasm_execution_error, "WASM data segment outside of valid memory range");
-            if(base_offset + data_segment.data.size() > mem_image.size())
-               mem_image.resize(base_offset + data_segment.data.size(), 0x00);
-            memcpy(mem_image.data() + base_offset, data_segment.data.data(), data_segment.data.size());
-         }
-
-         return mem_image;
+      bool is_code_cached(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) const {
+         wasm_cache_index::iterator it = wasm_instantiation_cache.find( boost::make_tuple(code_hash, vm_type, vm_version) );
+         return it != wasm_instantiation_cache.end();
       }
 
       void code_block_num_last_used(const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, const uint32_t& block_num) {
@@ -142,7 +140,6 @@ namespace eosio { namespace chain {
 
             it = wasm_instantiation_cache.emplace( wasm_interface_impl::wasm_cache_entry{
                                                       .code_hash = code_hash,
-                                                      .first_block_num_used = codeobject->first_block_used,
                                                       .last_block_num_used = UINT32_MAX,
                                                       .module = nullptr,
                                                       .vm_type = vm_type,
@@ -158,24 +155,8 @@ namespace eosio { namespace chain {
                trx_context.resume_billing_timer();
             });
             trx_context.pause_billing_timer();
-            IR::Module module;
-            std::vector<U8> bytes = {
-                (const U8*)codeobject->code.data(),
-                (const U8*)codeobject->code.data() + codeobject->code.size()};
-            try {
-               Serialization::MemoryInputStream stream((const U8*)bytes.data(),
-                                                       bytes.size());
-               WASM::scoped_skip_checks no_check;
-               WASM::serialize(stream, module);
-               module.userSections.clear();
-            } catch (const Serialization::FatalSerializationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            } catch (const IR::ValidationException& e) {
-               EOS_ASSERT(false, wasm_serialization_error, e.message.c_str());
-            }
-
             wasm_instantiation_cache.modify(it, [&](auto& c) {
-               c.module = runtime_interface->instantiate_module((const char*)bytes.data(), bytes.size(), parse_initial_memory(module), code_hash, vm_type, vm_version);
+               c.module = runtime_interface->instantiate_module(codeobject->code.data(), codeobject->code.size(), code_hash, vm_type, vm_version);
             });
          }
          return it->module;
@@ -194,7 +175,6 @@ namespace eosio { namespace chain {
                   member<wasm_cache_entry, uint8_t,     &wasm_cache_entry::vm_version>
                >
             >,
-            ordered_non_unique<tag<by_first_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::first_block_num_used>>,
             ordered_non_unique<tag<by_last_block_num>, member<wasm_cache_entry, uint32_t, &wasm_cache_entry::last_block_num_used>>
          >
       > wasm_cache_index;
@@ -202,6 +182,7 @@ namespace eosio { namespace chain {
 
       const chainbase::database& db;
       const wasm_interface::vm_type wasm_runtime_time;
+      const wasm_interface::vm_oc_enable eosvmoc_tierup;
 
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       std::optional<eosvmoc_tier> eosvmoc;

libraries/chain/include/eosio/chain/wast_to_wasm.hpp

@@ -1,6 +1,7 @@
 #pragma once
 #include <vector>
 #include <string>
+#include <stdint.h>
 
 namespace eosio { namespace chain {
 

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp

@@ -19,7 +19,6 @@
 namespace eosio { namespace chain { namespace webassembly { namespace eosvmoc {
 
 using namespace IR;
-using namespace Runtime;
 using namespace fc;
 
 using namespace eosio::chain::eosvmoc;
@@ -28,17 +27,21 @@ class eosvmoc_instantiated_module;
 
 class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface {
    public:
-      eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
+      eosvmoc_runtime(const std::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~eosvmoc_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
+      void init_thread_local_data() override;
 
       friend eosvmoc_instantiated_module;
       eosvmoc::code_cache_sync cc;
       eosvmoc::executor exec;
       eosvmoc::memory mem;
+
+      // Defined in eos-vm-oc.cpp. Used for non-main thread in multi-threaded execution
+      thread_local static std::unique_ptr<eosvmoc::executor> exec_thread_local;
+      thread_local static eosvmoc::memory mem_thread_local;
 };
 
 /**
@@ -349,7 +352,13 @@ auto fn(A... a) {
                       : "cc");
       }
       using native_args = vm::flatten_parameters_t<AUTO_PARAM_WORKAROUND(F)>;
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-value"
+      // If a is unpopulated, this reports "statement has no effect [-Werror=unused-value]"
       eosio::vm::native_value stack[] = { a... };
+#pragma GCC diagnostic pop
+
       constexpr int cb_ctx_ptr_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(ctx);
       apply_context* ctx;
       asm("mov %%gs:%c[applyContextOffset], %[cPtr]\n"
@@ -385,7 +394,7 @@ void register_eosvm_oc(Name n) {
    if(n == BOOST_HANA_STRING("env.eosio_exit")) return;
    constexpr auto fn = create_function<F, Preconditions, injected>();
    constexpr auto index = find_intrinsic_index(n.c_str());
-   intrinsic the_intrinsic(
+   [[maybe_unused]] intrinsic the_intrinsic(
       n.c_str(),
       wasm_function_type_provider<std::remove_pointer_t<decltype(fn)>>::type(),
       reinterpret_cast<void*>(fn),

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -30,21 +30,27 @@ using namespace boost::multi_index;
 using namespace boost::asio;
 
 namespace bip = boost::interprocess;
-namespace bfs = boost::filesystem;
 
 using allocator_t = bip::rbtree_best_fit<bip::null_mutex_family, bip::offset_ptr<void>, alignof(std::max_align_t)>;
 
 struct config;
 
+
 class code_cache_base {
    public:
-      code_cache_base(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
+      code_cache_base(const std::filesystem::path& data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~code_cache_base();
 
       const int& fd() const { return _cache_fd; }
 
       void free_code(const digest_type& code_id, const uint8_t& vm_version);
 
+      // get_descriptor_for_code failure reasons
+      enum class get_cd_failure {
+         temporary, // oc compile not done yet, users like read-only trxs can retry
+         permanent  // oc will not start, users should not retry
+      };
+
    protected:
       struct by_hash;
 
@@ -64,16 +70,27 @@ class code_cache_base {
 
       const chainbase::database& _db;
 
-      bfs::path _cache_file_path;
-      int _cache_fd;
+      std::filesystem::path _cache_file_path;
+      int                   _cache_fd;
 
       io_context _ctx;
       local::datagram_protocol::socket _compile_monitor_write_socket{_ctx};
       local::datagram_protocol::socket _compile_monitor_read_socket{_ctx};
 
-      //these are really only useful to the async code cache, but keep them here so
-      //free_code can be shared
-      std::unordered_set<code_tuple> _queued_compiles;
+      //these are really only useful to the async code cache, but keep them here so free_code can be shared
+      using queued_compilies_t = boost::multi_index_container<
+         code_tuple,
+         indexed_by<
+            sequenced<>,
+            hashed_unique<tag<by_hash>,
+               composite_key< code_tuple,
+                  member<code_tuple, digest_type, &code_tuple::code_id>,
+                  member<code_tuple, uint8_t,     &code_tuple::vm_version>
+               >
+            >
+         >
+      >;
+      queued_compilies_t _queued_compiles;
       std::unordered_map<code_tuple, bool> _outstanding_compiles_and_poison;
 
       size_t _free_bytes_eviction_threshold;
@@ -88,13 +105,13 @@ class code_cache_base {
 
 class code_cache_async : public code_cache_base {
    public:
-      code_cache_async(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
+      code_cache_async(const std::filesystem::path& data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db);
       ~code_cache_async();
 
       //If code is in cache: returns pointer & bumps to front of MRU list
       //If code is not in cache, and not blacklisted, and not currently compiling: return nullptr and kick off compile
       //otherwise: return nullptr
-      const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code(bool high_priority, const digest_type& code_id, const uint8_t& vm_version, bool is_write_window, get_cd_failure& failure);
 
    private:
       std::thread _monitor_reply_thread;
@@ -111,7 +128,7 @@ class code_cache_sync : public code_cache_base {
       ~code_cache_sync();
 
       //Can still fail and return nullptr if, for example, there is an expected instantiation failure
-      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version);
+      const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version, bool is_write_window);
 };
 
-}}}
+}}}

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/config.hpp

@@ -5,7 +5,6 @@
 #include <vector>
 #include <string>
 
-#include <boost/filesystem/path.hpp>
 #include <fc/reflect/reflect.hpp>
 
 namespace eosio { namespace chain { namespace eosvmoc {

libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp

@@ -43,15 +43,11 @@ template<typename Backend>
 class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
 
-      void immediately_exit_currently_running_module() override;
-
    private:
       // todo: managing this will get more complicated with sync calls;
-      //       immediately_exit_currently_running_module() should probably
-      //       move from wasm_runtime_interface to wasm_instantiated_module_interface.
       eos_vm_backend_t<Backend>* _bkend = nullptr;  // non owning pointer to allow for immediate exit
 
    template<typename Impl>
@@ -61,10 +57,8 @@ class eos_vm_runtime : public eosio::chain::wasm_runtime_interface {
 class eos_vm_profile_runtime : public eosio::chain::wasm_runtime_interface {
    public:
       eos_vm_profile_runtime();
-      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t>,
+      std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                              const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override;
-
-      void immediately_exit_currently_running_module() override;
 };
 
 }}}}// eosio::chain::webassembly::eos_vm_runtime

libraries/chain/include/eosio/chain/webassembly/runtime_interface.hpp

@@ -20,13 +20,13 @@ class wasm_instantiated_module_interface {
 
 class wasm_runtime_interface {
    public:
-      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size, std::vector<uint8_t> initial_memory,
+      virtual std::unique_ptr<wasm_instantiated_module_interface> instantiate_module(const char* code_bytes, size_t code_size,
                                                                                      const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) = 0;
 
-      //immediately exit the currently running wasm_instantiated_module_interface. Yep, this assumes only one can possibly run at a time.
-      virtual void immediately_exit_currently_running_module() = 0;
-
       virtual ~wasm_runtime_interface();
+
+      // eosvmoc_runtime needs this
+      virtual void init_thread_local_data() {};
 };
 
 }}

libraries/chain/platform_timer_asio_fallback.cpp

@@ -30,7 +30,7 @@ platform_timer::platform_timer() {
       std::promise<void> p;
       auto f = p.get_future();
       checktime_thread = std::thread([&p]() {
-         fc::set_os_thread_name("checktime");
+         fc::set_thread_name("checktime");
          checktime_ios = std::make_unique<boost::asio::io_service>();
          boost::asio::io_service::work work(*checktime_ios);
          p.set_value();

libraries/chain/platform_timer_kqueue.cpp

@@ -11,9 +11,18 @@
 
 #include <sys/event.h>
 
+// this makes kqueue calls compatible with FreeBSD and this timer can be safely used
+#if defined(__FreeBSD__)
+#define EV_SET64(kev, ident, filter, flags, fflags, data, udata, ext0, ext1) EV_SET(kev, ident, filter, flags, fflags, data, reinterpret_cast<void*>(udata))
+#define kevent64(kq, changelist, nchanges, eventlist, nevents, flags, timeout) kevent(kq, changelist, nchanges, eventlist, nevents, timeout)
+#define kevent64_s kevent
+#define KEVENT_FLAG_IMMEDIATE 0
+#define NOTE_CRITICAL 0
+#endif
+
 namespace eosio { namespace chain {
 
-//a kqueue & thread is shared for all platform_timer_macos instances
+// a kqueue & thread is shared for all platform_timer_macos instances
 static std::mutex timer_ref_mutex;
 static unsigned next_timerid;
 static unsigned refcount;
@@ -42,7 +51,7 @@ platform_timer::platform_timer() {
       FC_ASSERT(kevent64(kqueue_fd, &quit_event, 1, NULL, 0, KEVENT_FLAG_IMMEDIATE, NULL) == 0, "failed to create quit event");
 
       kevent_thread = std::thread([]() {
-         fc::set_os_thread_name("checktime");
+         fc::set_thread_name("checktime");
          while(true) {
             struct kevent64_s anEvent;
             int c = kevent64(kqueue_fd, NULL, 0, &anEvent, 1, 0, NULL);

libraries/chain/protocol_feature_manager.cpp

@@ -4,6 +4,7 @@
 #include <eosio/chain/deep_mind.hpp>
 
 #include <fc/scoped_exit.hpp>
+#include <fc/io/json.hpp>
 
 #include <algorithm>
 #include <boost/assign/list_of.hpp>
@@ -568,7 +569,7 @@ Enables new `get_block_num` intrinsic which returns the current block number.
 
    protocol_feature_manager::protocol_feature_manager(
       protocol_feature_set&& pfs,
-      std::function<deep_mind_handler*()> get_deep_mind_logger
+      std::function<deep_mind_handler*(bool is_trx_transient)> get_deep_mind_logger
    ):_protocol_feature_set( std::move(pfs) ), _get_deep_mind_logger(get_deep_mind_logger)
    {
       _builtin_protocol_features.resize( _protocol_feature_set._recognized_builtin_protocol_features.size() );
@@ -749,7 +750,8 @@ Enables new `get_block_num` intrinsic which returns the current block number.
                   ("digest", feature_digest)
       );
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      // activate_feature is called by init. no transaction specific logging is possible
+      if (auto dm_logger = _get_deep_mind_logger(false)) {
          dm_logger->on_activate_feature(*itr);
       }
 
@@ -778,4 +780,202 @@ Enables new `get_block_num` intrinsic which returns the current block number.
       }
    }
 
+   std::optional<builtin_protocol_feature> read_builtin_protocol_feature( const std::filesystem::path& p  ) {
+      try {
+         return fc::json::from_file<builtin_protocol_feature>( p );
+      } catch( const fc::exception& e ) {
+         wlog( "problem encountered while reading '${path}':\n${details}",
+               ("path", p)("details",e.to_detail_string()) );
+      } catch( ... ) {
+         dlog( "unknown problem encountered while reading '${path}'",
+               ("path", p) );
+      }
+      return {};
+   }
+
+   protocol_feature_set initialize_protocol_features( const std::filesystem::path& p, bool populate_missing_builtins ) {
+      using std::filesystem::directory_iterator;
+
+      protocol_feature_set pfs;
+
+      bool directory_exists = true;
+
+      if( std::filesystem::exists( p ) ) {
+         EOS_ASSERT( std::filesystem::is_directory( p ), plugin_exception,
+                     "Path to protocol-features is not a directory: ${path}",
+                     ("path", p)
+         );
+      } else {
+         if( populate_missing_builtins )
+            std::filesystem::create_directories( p );
+         else
+            directory_exists = false;
+      }
+
+      auto log_recognized_protocol_feature = []( const builtin_protocol_feature& f, const digest_type& feature_digest ) {
+         if( f.subjective_restrictions.enabled ) {
+            if( f.subjective_restrictions.preactivation_required ) {
+               if( f.subjective_restrictions.earliest_allowed_activation_time == time_point{} ) {
+                  ilog( "Support for builtin protocol feature '${codename}' (with digest of '${digest}') is enabled with preactivation required",
+                        ("codename", builtin_protocol_feature_codename(f.get_codename()))
+                              ("digest", feature_digest)
+                  );
+               } else {
+                  ilog( "Support for builtin protocol feature '${codename}' (with digest of '${digest}') is enabled with preactivation required and with an earliest allowed activation time of ${earliest_time}",
+                        ("codename", builtin_protocol_feature_codename(f.get_codename()))
+                              ("digest", feature_digest)
+                              ("earliest_time", f.subjective_restrictions.earliest_allowed_activation_time)
+                  );
+               }
+            } else {
+               if( f.subjective_restrictions.earliest_allowed_activation_time == time_point{} ) {
+                  ilog( "Support for builtin protocol feature '${codename}' (with digest of '${digest}') is enabled without activation restrictions",
+                        ("codename", builtin_protocol_feature_codename(f.get_codename()))
+                              ("digest", feature_digest)
+                  );
+               } else {
+                  ilog( "Support for builtin protocol feature '${codename}' (with digest of '${digest}') is enabled without preactivation required but with an earliest allowed activation time of ${earliest_time}",
+                        ("codename", builtin_protocol_feature_codename(f.get_codename()))
+                              ("digest", feature_digest)
+                              ("earliest_time", f.subjective_restrictions.earliest_allowed_activation_time)
+                  );
+               }
+            }
+         } else {
+            ilog( "Recognized builtin protocol feature '${codename}' (with digest of '${digest}') but support for it is not enabled",
+                  ("codename", builtin_protocol_feature_codename(f.get_codename()))
+                        ("digest", feature_digest)
+            );
+         }
+      };
+
+      map<builtin_protocol_feature_t, std::filesystem::path>  found_builtin_protocol_features;
+      map<digest_type, std::pair<builtin_protocol_feature, bool> > builtin_protocol_features_to_add;
+      // The bool in the pair is set to true if the builtin protocol feature has already been visited to add
+      map< builtin_protocol_feature_t, std::optional<digest_type> > visited_builtins;
+
+      // Read all builtin protocol features
+      if( directory_exists ) {
+         for( directory_iterator enditr, itr{p}; itr != enditr; ++itr ) {
+            auto file_path = itr->path();
+            if( !std::filesystem::is_regular_file( file_path ) || file_path.extension().generic_string().compare( ".json" ) != 0 )
+               continue;
+
+            auto f = read_builtin_protocol_feature( file_path );
+
+            if( !f ) continue;
+
+            auto res = found_builtin_protocol_features.emplace( f->get_codename(), file_path );
+
+            EOS_ASSERT( res.second, plugin_exception,
+                        "Builtin protocol feature '${codename}' was already included from a previous_file",
+                        ("codename", builtin_protocol_feature_codename(f->get_codename()))
+                              ("current_file", file_path)
+                              ("previous_file", res.first->second)
+            );
+
+            const auto feature_digest = f->digest();
+
+            builtin_protocol_features_to_add.emplace( std::piecewise_construct,
+                                                      std::forward_as_tuple( feature_digest ),
+                                                      std::forward_as_tuple( *f, false ) );
+         }
+      }
+
+      // Add builtin protocol features to the protocol feature manager in the right order (to satisfy dependencies)
+      using itr_type = map<digest_type, std::pair<builtin_protocol_feature, bool>>::iterator;
+      std::function<void(const itr_type&)> add_protocol_feature =
+            [&pfs, &builtin_protocol_features_to_add, &visited_builtins, &log_recognized_protocol_feature, &add_protocol_feature]( const itr_type& itr ) -> void {
+               if( itr->second.second ) {
+                  return;
+               } else {
+                  itr->second.second = true;
+                  visited_builtins.emplace( itr->second.first.get_codename(), itr->first );
+               }
+
+               for( const auto& d : itr->second.first.dependencies ) {
+                  auto itr2 = builtin_protocol_features_to_add.find( d );
+                  if( itr2 != builtin_protocol_features_to_add.end() ) {
+                     add_protocol_feature( itr2 );
+                  }
+               }
+
+               pfs.add_feature( itr->second.first );
+
+               log_recognized_protocol_feature( itr->second.first, itr->first );
+            };
+
+      for( auto itr = builtin_protocol_features_to_add.begin(); itr != builtin_protocol_features_to_add.end(); ++itr ) {
+         add_protocol_feature( itr );
+      }
+
+      auto output_protocol_feature = [&p]( const builtin_protocol_feature& f, const digest_type& feature_digest ) {
+         string filename( "BUILTIN-" );
+         filename += builtin_protocol_feature_codename( f.get_codename() );
+         filename += ".json";
+
+         auto file_path = p / filename;
+
+         EOS_ASSERT( !std::filesystem::exists( file_path ), plugin_exception,
+                     "Could not save builtin protocol feature with codename '${codename}' because a file at the following path already exists: ${path}",
+                     ("codename", builtin_protocol_feature_codename( f.get_codename() ))
+                           ("path", file_path)
+         );
+
+         if( fc::json::save_to_file( f, file_path ) ) {
+            ilog( "Saved default specification for builtin protocol feature '${codename}' (with digest of '${digest}') to: ${path}",
+                  ("codename", builtin_protocol_feature_codename(f.get_codename()))
+                        ("digest", feature_digest)
+                        ("path", file_path)
+            );
+         } else {
+            elog( "Error occurred while writing default specification for builtin protocol feature '${codename}' (with digest of '${digest}') to: ${path}",
+                  ("codename", builtin_protocol_feature_codename(f.get_codename()))
+                        ("digest", feature_digest)
+                        ("path", file_path)
+            );
+         }
+      };
+
+      std::function<digest_type(builtin_protocol_feature_t)> add_missing_builtins =
+            [&pfs, &visited_builtins, &output_protocol_feature, &log_recognized_protocol_feature, &add_missing_builtins, populate_missing_builtins]
+                  ( builtin_protocol_feature_t codename ) -> digest_type {
+               auto res = visited_builtins.emplace( codename, std::optional<digest_type>() );
+               if( !res.second ) {
+                  EOS_ASSERT( res.first->second, protocol_feature_exception,
+                              "invariant failure: cycle found in builtin protocol feature dependencies"
+                  );
+                  return *res.first->second;
+               }
+
+               auto f = protocol_feature_set::make_default_builtin_protocol_feature( codename,
+                                                                                     [&add_missing_builtins]( builtin_protocol_feature_t d ) {
+                                                                                        return add_missing_builtins( d );
+                                                                                     } );
+
+               if( !populate_missing_builtins )
+                  f.subjective_restrictions.enabled = false;
+
+               const auto& pf = pfs.add_feature( f );
+               res.first->second = pf.feature_digest;
+
+               log_recognized_protocol_feature( f, pf.feature_digest );
+
+               if( populate_missing_builtins )
+                  output_protocol_feature( f, pf.feature_digest );
+
+               return pf.feature_digest;
+            };
+
+      for( const auto& p : builtin_protocol_feature_codenames ) {
+         auto itr = found_builtin_protocol_features.find( p.first );
+         if( itr != found_builtin_protocol_features.end() ) continue;
+
+         add_missing_builtins( p.first );
+      }
+
+      return pfs;
+   }
+
+
 } }  // eosio::chain

libraries/chain/resource_limits.cpp

@@ -65,7 +65,8 @@ void resource_limits_manager::initialize_database() {
       state.virtual_net_limit = config.net_limit_parameters.max;
    });
 
-   if (auto dm_logger = _get_deep_mind_logger()) {
+   // At startup, no transaction specific logging is possible
+   if (auto dm_logger = _get_deep_mind_logger(false)) {
       dm_logger->on_init_resource_limits(config, state);
    }
 }
@@ -93,7 +94,7 @@ void resource_limits_manager::read_from_snapshot( const snapshot_reader_ptr& sna
    });
 }
 
-void resource_limits_manager::initialize_account(const account_name& account) {
+void resource_limits_manager::initialize_account(const account_name& account, bool is_trx_transient) {
    const auto& limits = _db.create<resource_limits_object>([&]( resource_limits_object& bl ) {
       bl.owner = account;
    });
@@ -101,7 +102,7 @@ void resource_limits_manager::initialize_account(const account_name& account) {
    const auto& usage = _db.create<resource_usage_object>([&]( resource_usage_object& bu ) {
       bu.owner = account;
    });
-   if (auto dm_logger = _get_deep_mind_logger()) {
+   if (auto dm_logger = _get_deep_mind_logger(is_trx_transient)) {
       dm_logger->on_newaccount_resource_limits(limits, usage);
    }
 }
@@ -116,7 +117,9 @@ void resource_limits_manager::set_block_parameters(const elastic_limit_parameter
       c.cpu_limit_parameters = cpu_limit_parameters;
       c.net_limit_parameters = net_limit_parameters;
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      // set_block_parameters is called by controller::finalize_block,
+      // where transaction specific logging is not possible
+      if (auto dm_logger = _get_deep_mind_logger(false)) {
          dm_logger->on_update_resource_limits_config(c);
       }
    });
@@ -133,7 +136,7 @@ void resource_limits_manager::update_account_usage(const flat_set<account_name>&
    }
 }
 
-void resource_limits_manager::add_transaction_usage(const flat_set<account_name>& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t time_slot ) {
+void resource_limits_manager::add_transaction_usage(const flat_set<account_name>& accounts, uint64_t cpu_usage, uint64_t net_usage, uint32_t time_slot, bool is_trx_transient ) {
    const auto& state = _db.get<resource_limits_state_object>();
    const auto& config = _db.get<resource_limits_config_object>();
 
@@ -149,7 +152,7 @@ void resource_limits_manager::add_transaction_usage(const flat_set<account_name>
           bu.net_usage.add( net_usage, time_slot, config.account_net_usage_average_window );
           bu.cpu_usage.add( cpu_usage, time_slot, config.account_cpu_usage_average_window );
 
-         if (auto dm_logger = _get_deep_mind_logger()) {
+         if (auto dm_logger = _get_deep_mind_logger(is_trx_transient)) {
             dm_logger->on_update_account_usage(bu);
          }
       });
@@ -166,8 +169,9 @@ void resource_limits_manager::add_transaction_usage(const flat_set<account_name>
 
          EOS_ASSERT( cpu_used_in_window <= max_user_use_in_window,
                      tx_cpu_usage_exceeded,
-                     "authorizing account '${n}' has insufficient cpu resources for this transaction",
-                     ("n", name(a))
+                     "authorizing account '${n}' has insufficient objective cpu resources for this transaction,"
+                     " used in window ${cpu_used_in_window}us, allowed in window ${max_user_use_in_window}us",
+                     ("n", a)
                      ("cpu_used_in_window",cpu_used_in_window)
                      ("max_user_use_in_window",max_user_use_in_window) );
       }
@@ -185,8 +189,9 @@ void resource_limits_manager::add_transaction_usage(const flat_set<account_name>
 
          EOS_ASSERT( net_used_in_window <= max_user_use_in_window,
                      tx_net_usage_exceeded,
-                     "authorizing account '${n}' has insufficient net resources for this transaction",
-                     ("n", name(a))
+                     "authorizing account '${n}' has insufficient net resources for this transaction,"
+                     " used in window ${net_used_in_window}, allowed in window ${max_user_use_in_window}",
+                     ("n", a)
                      ("net_used_in_window",net_used_in_window)
                      ("max_user_use_in_window",max_user_use_in_window) );
 
@@ -203,7 +208,7 @@ void resource_limits_manager::add_transaction_usage(const flat_set<account_name>
    EOS_ASSERT( state.pending_net_usage <= config.net_limit_parameters.max, block_resource_exhausted, "Block has insufficient net resources" );
 }
 
-void resource_limits_manager::add_pending_ram_usage( const account_name account, int64_t ram_delta ) {
+void resource_limits_manager::add_pending_ram_usage( const account_name account, int64_t ram_delta, bool is_trx_transient ) {
    if (ram_delta == 0) {
       return;
    }
@@ -218,7 +223,7 @@ void resource_limits_manager::add_pending_ram_usage( const account_name account,
    _db.modify( usage, [&]( auto& u ) {
       u.ram_usage += ram_delta;
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      if (auto dm_logger = _get_deep_mind_logger(is_trx_transient)) {
          dm_logger->on_ram_event(account, u.ram_usage, ram_delta);
       }
    });
@@ -241,7 +246,7 @@ int64_t resource_limits_manager::get_account_ram_usage( const account_name& name
 }
 
 
-bool resource_limits_manager::set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight) {
+bool resource_limits_manager::set_account_limits( const account_name& account, int64_t ram_bytes, int64_t net_weight, int64_t cpu_weight, bool is_trx_transient) {
    //const auto& usage = _db.get<resource_usage_object,by_owner>( account );
    /*
     * Since we need to delay these until the next resource limiting boundary, these are created in a "pending"
@@ -287,7 +292,7 @@ bool resource_limits_manager::set_account_limits( const account_name& account, i
       pending_limits.net_weight = net_weight;
       pending_limits.cpu_weight = cpu_weight;
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      if (auto dm_logger = _get_deep_mind_logger(is_trx_transient)) {
          dm_logger->on_set_account_limits(pending_limits);
       }
    });
@@ -354,7 +359,9 @@ void resource_limits_manager::process_account_limit_updates() {
          multi_index.remove(*itr);
       }
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      // process_account_limit_updates is called by controller::finalize_block,
+      // where transaction specific logging is not possible
+      if (auto dm_logger = _get_deep_mind_logger(false)) {
          dm_logger->on_update_resource_limits_state(state);
       }
    });
@@ -374,7 +381,9 @@ void resource_limits_manager::process_block_usage(uint32_t block_num) {
       state.update_virtual_net_limit(config);
       state.pending_net_usage = 0;
 
-      if (auto dm_logger = _get_deep_mind_logger()) {
+      // process_block_usage is called by controller::finalize_block,
+      // where transaction specific logging is not possible
+      if (auto dm_logger = _get_deep_mind_logger(false)) {
          dm_logger->on_update_resource_limits_state(state);
       }
    });

libraries/chain/snapshot.cpp

@@ -355,7 +355,7 @@ struct istream_json_snapshot_reader_impl {
 
 istream_json_snapshot_reader::~istream_json_snapshot_reader() = default;
 
-istream_json_snapshot_reader::istream_json_snapshot_reader(const fc::path& p)
+istream_json_snapshot_reader::istream_json_snapshot_reader(const std::filesystem::path& p)
    : impl{new istream_json_snapshot_reader_impl{0, 0, {}, {}}}
 {
    FILE* fp = fopen(p.string().c_str(), "rb");

libraries/chain/snapshot_scheduler.cpp

@@ -0,0 +1,231 @@
+#include <eosio/chain/controller.hpp>
+#include <eosio/chain/exceptions.hpp>
+#include <eosio/chain/pending_snapshot.hpp>
+#include <eosio/chain/snapshot_scheduler.hpp>
+#include <fc/scoped_exit.hpp>
+
+namespace eosio::chain {
+
+// snapshot_scheduler_listener
+void snapshot_scheduler::on_start_block(uint32_t height, chain::controller& chain) {
+   bool snapshot_executed = false;
+
+   auto execute_snapshot_with_log = [this, height, &snapshot_executed, &chain](const auto& req) {
+      // one snapshot per height
+      if(!snapshot_executed) {
+         dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}",
+              ("start_block_num", req.start_block_num)("end_block_num", req.end_block_num)("block_spacing", req.block_spacing)("height", height));
+
+         execute_snapshot(req.snapshot_request_id, chain);
+         snapshot_executed = true;
+      }
+   };
+
+   std::vector<uint32_t> unschedule_snapshot_request_ids;
+   for(const auto& req: _snapshot_requests.get<0>()) {
+      // -1 since its called from start block
+      bool recurring_snapshot  = req.block_spacing && (height >= req.start_block_num + 1) && (!((height - req.start_block_num - 1) % req.block_spacing));
+      bool onetime_snapshot    = (!req.block_spacing) && (height == req.start_block_num + 1);
+      
+      bool marked_for_deletion = ((!req.block_spacing) && (height >= req.start_block_num + 1)) || // if one time snapshot executed or scheduled for the past, it should be gone
+                                 (height > 0 && ((height-1) >= req.end_block_num));               // any snapshot can expire by end block num (end_block_num can be max value)
+
+      if(recurring_snapshot || onetime_snapshot) {
+         execute_snapshot_with_log(req);
+      }
+
+      // cleanup - remove expired (or invalid) request
+      if(marked_for_deletion) {
+         unschedule_snapshot_request_ids.push_back(req.snapshot_request_id);
+      }
+   }
+
+   for(const auto& i: unschedule_snapshot_request_ids) {
+      unschedule_snapshot(i);
+   }
+}
+
+void snapshot_scheduler::on_irreversible_block(const signed_block_ptr& lib, const chain::controller& chain) {
+   auto& snapshots_by_height = _pending_snapshot_index.get<by_height>();
+   uint32_t lib_height = lib->block_num();
+
+   while(!snapshots_by_height.empty() && snapshots_by_height.begin()->get_height() <= lib_height) {
+      const auto& pending = snapshots_by_height.begin();
+      auto next = pending->next;
+
+      try {
+         next(pending->finalize(chain));
+      }
+      CATCH_AND_CALL(next);
+
+      snapshots_by_height.erase(snapshots_by_height.begin());
+   }
+}
+
+snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::schedule_snapshot(const snapshot_request_information& sri) {
+   auto& snapshot_by_value = _snapshot_requests.get<by_snapshot_value>();
+   auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
+   EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request");
+   EOS_ASSERT(sri.start_block_num <= sri.end_block_num, chain::invalid_snapshot_request, "End block number should be greater or equal to start block number");
+   EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, chain::invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
+
+   _snapshot_requests.emplace(snapshot_schedule_information{{_snapshot_id++}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}, {}});
+   x_serialize();
+
+   // returning snapshot_schedule_result
+   return snapshot_schedule_result{{_snapshot_id - 1}, {sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description}};
+}
+
+snapshot_scheduler::snapshot_schedule_result snapshot_scheduler::unschedule_snapshot(uint32_t sri) {
+   auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+   auto existing = snapshot_by_id.find(sri);
+   EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found");
+
+   snapshot_schedule_result result{{existing->snapshot_request_id}, {existing->block_spacing, existing->start_block_num, existing->end_block_num, existing->snapshot_description}};
+   _snapshot_requests.erase(existing);
+   x_serialize();
+
+   // returning snapshot_schedule_result
+   return result;
+}
+
+snapshot_scheduler::get_snapshot_requests_result snapshot_scheduler::get_snapshot_requests() {
+   get_snapshot_requests_result result;
+   auto& asvector = _snapshot_requests.get<as_vector>();
+   result.snapshot_requests.reserve(asvector.size());
+   result.snapshot_requests.insert(result.snapshot_requests.begin(), asvector.begin(), asvector.end());
+   return result;
+}
+
+void snapshot_scheduler::set_db_path(fs::path db_path) {
+   _snapshot_db.set_path(std::move(db_path));
+   // init from db
+   if(std::filesystem::exists(_snapshot_db.get_json_path())) {
+      std::vector<snapshot_schedule_information> sr;
+      _snapshot_db >> sr;
+      // if db read succeeded, clear/load
+      _snapshot_requests.get<by_snapshot_id>().clear();
+      _snapshot_requests.insert(sr.begin(), sr.end());
+   }
+}
+
+void snapshot_scheduler::set_snapshots_path(fs::path sn_path) {
+   _snapshots_dir = std::move(sn_path);
+}
+
+void snapshot_scheduler::add_pending_snapshot_info(const snapshot_information& si) {
+   auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+   auto snapshot_req = snapshot_by_id.find(_inflight_sid);
+   if(snapshot_req != snapshot_by_id.end()) {
+      _snapshot_requests.modify(snapshot_req, [&si](auto& p) {
+         p.pending_snapshots.emplace_back(si);
+      });
+   }
+}
+
+void snapshot_scheduler::execute_snapshot(uint32_t srid, chain::controller& chain) {
+   _inflight_sid = srid;
+   auto next = [srid, this](const chain::next_function_variant<snapshot_information>& result) {
+      if(std::holds_alternative<fc::exception_ptr>(result)) {
+         try {
+            std::get<fc::exception_ptr>(result)->dynamic_rethrow_exception();
+         } catch(const fc::exception& e) {
+            EOS_THROW(snapshot_execution_exception,
+                     "Snapshot creation error: ${details}",
+                     ("details", e.to_detail_string()));
+         } catch(const std::exception& e) {
+            EOS_THROW(snapshot_execution_exception,
+                     "Snapshot creation error: ${details}",
+                     ("details", e.what()));
+         }
+      } else {
+         // success, snapshot finalized
+         auto snapshot_info = std::get<snapshot_information>(result);
+         auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+         auto snapshot_req = snapshot_by_id.find(srid);
+
+         if(snapshot_req != snapshot_by_id.end()) {
+            _snapshot_requests.modify(snapshot_req, [&](auto& p) {
+               auto& pending = p.pending_snapshots;
+               auto it = std::remove_if(pending.begin(), pending.end(), [&snapshot_info](const snapshot_information& s) { return s.head_block_num <= snapshot_info.head_block_num; });
+               pending.erase(it, pending.end());
+            });
+         }
+      }
+   };
+   create_snapshot(next, chain, {});
+}
+
+void snapshot_scheduler::create_snapshot(next_function<snapshot_information> next, chain::controller& chain, std::function<void(void)> predicate) {
+   auto head_id = chain.head_block_id();
+   const auto head_block_num = chain.head_block_num();
+   const auto head_block_time = chain.head_block_time();
+   const auto& snapshot_path = pending_snapshot<snapshot_information>::get_final_path(head_id, _snapshots_dir);
+   const auto& temp_path = pending_snapshot<snapshot_information>::get_temp_path(head_id, _snapshots_dir);
+
+   // maintain legacy exception if the snapshot exists
+   if(fs::is_regular_file(snapshot_path)) {
+      auto ex = snapshot_exists_exception(FC_LOG_MESSAGE(error, "snapshot named ${name} already exists", ("name", _snapshots_dir)));
+      next(ex.dynamic_copy_exception());
+      return;
+   }
+
+   auto write_snapshot = [&](const fs::path& p) -> void {
+      if(predicate) predicate();
+      fs::create_directory(p.parent_path());
+      auto snap_out = std::ofstream(p.generic_string(), (std::ios::out | std::ios::binary));
+      auto writer = std::make_shared<ostream_snapshot_writer>(snap_out);
+      chain.write_snapshot(writer);
+      writer->finalize();
+      snap_out.flush();
+      snap_out.close();
+   };
+
+   // If in irreversible mode, create snapshot and return path to snapshot immediately.
+   if(chain.get_read_mode() == db_read_mode::IRREVERSIBLE) {
+      try {
+         write_snapshot(temp_path);
+         std::error_code ec;
+         fs::rename(temp_path, snapshot_path, ec);
+         EOS_ASSERT(!ec, snapshot_finalization_exception,
+                    "Unable to finalize valid snapshot of block number ${bn}: [code: ${ec}] ${message}",
+                    ("bn", head_block_num)("ec", ec.value())("message", ec.message()));
+
+         next(snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, snapshot_path.generic_string()});
+      }
+      CATCH_AND_CALL(next);
+      return;
+   }
+
+   // Otherwise, the result will be returned when the snapshot becomes irreversible.
+
+   // determine if this snapshot is already in-flight
+   auto& pending_by_id = _pending_snapshot_index.get<by_id>();
+   auto existing = pending_by_id.find(head_id);
+   if(existing != pending_by_id.end()) {
+      // if a snapshot at this block is already pending, attach this requests handler to it
+      pending_by_id.modify(existing, [&next](auto& entry) {
+         entry.next = [prev = entry.next, next](const next_function_variant<snapshot_information>& res) {
+            prev(res);
+            next(res);
+         };
+      });
+   } else {
+      const auto& pending_path = pending_snapshot<snapshot_information>::get_pending_path(head_id, _snapshots_dir);
+
+      try {
+         write_snapshot(temp_path);// create a new pending snapshot
+
+         std::error_code ec;
+         fs::rename(temp_path, pending_path, ec);
+         EOS_ASSERT(!ec, snapshot_finalization_exception,
+                    "Unable to promote temp snapshot to pending for block number ${bn}: [code: ${ec}] ${message}",
+                    ("bn", head_block_num)("ec", ec.value())("message", ec.message()));
+         _pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string());
+         add_pending_snapshot_info(snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, pending_path.generic_string()});
+      }
+      CATCH_AND_CALL(next);
+   }
+}
+
+}// namespace eosio::chain

libraries/chain/symbol.cpp

@@ -0,0 +1,21 @@
+#include <eosio/chain/symbol.hpp>
+#include <boost/algorithm/string.hpp>
+
+namespace eosio::chain {
+   
+   symbol symbol::from_string(const string& from)
+{
+   try {
+      string s = boost::algorithm::trim_copy(from);
+      EOS_ASSERT(!s.empty(), symbol_type_exception, "creating symbol from empty string");
+      auto comma_pos = s.find(',');
+      EOS_ASSERT(comma_pos != string::npos, symbol_type_exception, "missing comma in symbol");
+      auto prec_part = s.substr(0, comma_pos);
+      uint8_t p = fc::to_int64(prec_part);
+      string name_part = s.substr(comma_pos + 1);
+      EOS_ASSERT( p <= max_precision, symbol_type_exception, "precision ${p} should be <= 18", ("p", p));
+      return symbol(string_to_symbol(p, name_part.c_str()));
+   } FC_CAPTURE_LOG_AND_RETHROW((from));
+}
+   
+} // namespace eosio::chain

libraries/chain/thread_utils.cpp

@@ -1,36 +0,0 @@
-#include <eosio/chain/thread_utils.hpp>
-#include <fc/log/logger_config.hpp>
-
-namespace eosio { namespace chain {
-
-
-//
-// named_thread_pool
-//
-named_thread_pool::named_thread_pool( std::string name_prefix, size_t num_threads )
-: _thread_pool( num_threads )
-, _ioc( num_threads )
-{
-   _ioc_work.emplace( boost::asio::make_work_guard( _ioc ) );
-   for( size_t i = 0; i < num_threads; ++i ) {
-      boost::asio::post( _thread_pool, [&ioc = _ioc, name_prefix, i]() {
-         std::string tn = name_prefix + "-" + std::to_string( i );
-         fc::set_os_thread_name( tn );
-         ioc.run();
-      } );
-   }
-}
-
-named_thread_pool::~named_thread_pool() {
-   stop();
-}
-
-void named_thread_pool::stop() {
-   _ioc_work.reset();
-   _ioc.stop();
-   _thread_pool.join();
-   _thread_pool.stop();
-}
-
-
-} } // eosio::chain

libraries/chain/transaction.cpp

@@ -62,16 +62,19 @@ fc::microseconds transaction::get_signature_keys( const vector<signature_type>&
 { try {
    auto start = fc::time_point::now();
    recovered_pub_keys.clear();
-   const digest_type digest = sig_digest(chain_id, cfd);
 
-   for(const signature_type& sig : signatures) {
-      auto now = fc::time_point::now();
-      EOS_ASSERT( now < deadline, tx_cpu_usage_exceeded, "transaction signature verification executed for too long ${time}us",
-                  ("time", now - start)("now", now)("deadline", deadline)("start", start) );
-      auto[ itr, successful_insertion ] = recovered_pub_keys.emplace( sig, digest );
-      EOS_ASSERT( allow_duplicate_keys || successful_insertion, tx_duplicate_sig,
-                  "transaction includes more than one signature signed using the same key associated with public key: ${key}",
-                  ("key", *itr ) );
+   if ( !signatures.empty() ) {
+      const digest_type digest = sig_digest(chain_id, cfd);
+
+      for(const signature_type& sig : signatures) {
+         auto now = fc::time_point::now();
+         EOS_ASSERT( now < deadline, tx_cpu_usage_exceeded, "transaction signature verification executed for too long ${time}us",
+                     ("time", now - start)("now", now)("deadline", deadline)("start", start) );
+         auto[ itr, successful_insertion ] = recovered_pub_keys.emplace( sig, digest );
+         EOS_ASSERT( allow_duplicate_keys || successful_insertion, tx_duplicate_sig,
+                     "transaction includes more than one signature signed using the same key associated with public key: ${key}",
+                     ("key", *itr ) );
+      }
    }
 
    return fc::time_point::now() - start;