fix

Large tx-sig fix

.gitignore

@@ -11,33 +11,8 @@ pids
 *.seed
 *.pid.lock
 
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
 # Dependency directories
 node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
 
 # Optional npm cache directory
 .npm
@@ -51,18 +26,11 @@ typings/
 # Output of 'npm pack'
 *.tgz
 
-# Yarn Integrity file
-.yarn-integrity
-
 # dotenv environment variables file
 .env
 
-# next.js build output
-.next
-
 .idea
 *.pem
-ecosystem.config.js
 package-lock.json
 connections.json
 connections.json2
@@ -100,15 +68,32 @@ helpers/**/*.js.map
 interfaces/**/*.js
 interfaces/**/*.js.map
 
-modules/**/*.js
-modules/**/*.js.map
+#modules/**/*.js
+#modules/**/*.js.map
 
 addons/**/*.js
 addons/**/*.js.map
 
+scripts/hpm.js
+scripts/hpm.js.map
+
+plugins/.state.json
 plugins/repos
+plugins/*.js
+.gitmodules
 
 docker/redis/data
 docker/elasticsearch/data
-docker/eosio/data
 docker/rabbitmq/data
+docker/eosio/data
+
+configuration_backups
+
+scripts/hyp-config.js
+scripts/hyp-config.js.map
+
+scripts/hyp-repair.js
+scripts/hyp-repair.js.map
+
+scripts/repair-cli/*.js
+scripts/repair-cli/*.js.map

DEPRECATED_install.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash -
-sudo apt-get install -y nodejs
-sudo npm install pm2 -g
-npm install

DEPRECATED_install_env.sh

@@ -1,303 +0,0 @@
-#!/usr/bin/env bash
-set -eo pipefail
-
-#terminal colors
-COLOR_NC=$(tput sgr0)
-export COLOR_NC
-COLOR_RED=$(tput setaf 1)
-export COLOR_RED
-COLOR_GREEN=$(tput setaf 2)
-export COLOR_GREEN
-COLOR_YELLOW=$(tput setaf 3)
-export COLOR_YELLOW
-COLOR_BLUE=$(tput setaf 4)
-export COLOR_BLUE
-
-#variables
-INITIAL_PARAMS="$*"
-GLOBAL="y"
-RABBIT_USER="hyperion"
-RABBIT_PASSWORD="123456"
-NODE=false
-ELASTIC=false
-REDIS=false
-RABBIT=false
-RAM=0
-
-help_function() {
-    echo -e "\nAutomated shell script that installs all dependencies and then configure Hyperion."
-    echo -e "\nUsage: ./install.env <option>\n"
-    echo -e "Options:"
-    echo -e "  --help ==> Show this menu"
-    echo -e "  --version ==> Show Hyperion current version\n"
-    exit 0
-}
-
-version (){
-  echo -e "\nv3.0.0\n"
-  exit 0
-
-}
-
-arg_checker() {
-    if [ "${INITIAL_PARAMS}" == "--help" ]; then
-        help_function
-
-    elif [ "${INITIAL_PARAMS}" == "--version" ]; then
-        version
-    fi
-}
-
-check_ram() {
-  RAM=$(free --giga | awk '/Mem/ {print $2}')
-}
-
-# first check if is installed, after check version
-check_dependencies() {
-  echo -e "\n\n${COLOR_BLUE}Checking dependencies...${COLOR_NC}\n\n"
-  CMD=$(if (dpkg --compare-versions $(dpkg -s nodejs 2>/dev/null | awk '/Version/ {print $2}') ge "5" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    NODE=true
-    echo -e "\n\n${COLOR_BLUE}Nodejs compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}Nodejs installed version is < 13. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n\n${COLOR_BLUE}Nodejs not installed ${COLOR_NC}"
-  fi
-  CMD=$(if (dpkg --compare-versions $(dpkg -s elasticsearch 2>/dev/null | awk '/Version/ {print $2}') ge "7.6" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    ELASTIC=true
-    echo -e "\n${COLOR_BLUE}Elasticsearch compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}Elasticsearch installed version is < 7.6. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n${COLOR_BLUE}Elasticsearch not installed ${COLOR_NC}"
-  fi
-  CMD=$(if (dpkg --compare-versions $(dpkg -s redis-server 2>/dev/null | awk '/Version/ {print $2}') ge "5" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    REDIS=true
-    echo -e "\n${COLOR_BLUE}Redis compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}Redis installed version is < 5. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n${COLOR_BLUE}Redis not installed ${COLOR_NC}"
-  fi
-  CMD=$(if (dpkg --compare-versions $(dpkg -s rabbitmq-server 2>/dev/null | awk '/Version/ {print $2}') ge "3.8" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    RABBIT=true
-    echo -e "\n${COLOR_BLUE}RabbitMQ compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}RabbitMQ installed version is < 3.8. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n${COLOR_BLUE}RabbitMQ not installed ${COLOR_NC}"
-  fi
-  CMD=$(if (dpkg --compare-versions $(dpkg -s kibana 2>/dev/null | awk '/Version/ {print $2}') ge "7.6" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    RABBIT=true
-    echo -e "\n${COLOR_BLUE}Kibana compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}Kibana installed version is < 7.6. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n${COLOR_BLUE}Kibana not installed ${COLOR_NC}\n\n"
-  fi
-}
-
-# Make a directory for global installations
-configure_npm() {
-  mkdir ~/.npm-global
-  npm config set prefix "$HOME/.npm-global"
-  # export path for the current session
-  export PATH=$HOME/.npm-global/bin:$PATH
-  # make PATH persistent
-  echo "export PATH=~/.npm-global/bin:$PATH" >>"$HOME/.profile"
-  # shellcheck source=/dev/null
-  source ~/.profile
-}
-
-install_keys_sources() {
-  echo -e "\n\n${COLOR_BLUE}Configuring keys and sources...${COLOR_NC}\n\n"
-
-  PPA="https://artifacts.elastic.co/packages/7.x/apt stable main"
-  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
-    wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
-    echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
-  fi
-
-  PPA="http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang"
-  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
-    curl -fsSL https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc | sudo apt-key add -
-    echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" | sudo tee /etc/apt/sources.list.d/bintray.rabbitmq.list
-    curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | sudo bash
-  fi
-
-  PPA="https://deb.nodesource.com/node_13.x bionic main"
-  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
-    curl -sL "https://deb.nodesource.com/setup_13.x" | sudo -E bash -
-  fi
-
-  sudo apt update -y
-}
-
-install_dep() {
- echo -e "\n\n${COLOR_BLUE}Installing dependencies...${COLOR_NC}\n\n"
- # sudo apt install -y build-essential curl
- sudo apt install -y curl
-}
-
-install_node() {
-  echo -e "\n\n${COLOR_BLUE}Installing nodejs...${COLOR_NC}\n\n"
-  sudo apt install -y nodejs
-  read -p "Do you want to create a directory for npm global installations [Y/n] : " GLOBAL
-  GLOBAL=${GLOBAL:-y}
-  if [ "$GLOBAL" = "y" ]; then
-    configure_npm
-  fi
-}
-
-install_build_hyperion() {
-  echo -e "\n\n${COLOR_BLUE}Installing packages and building hyperion...${COLOR_NC}\n\n"
-  npm install
-}
-
-install_pm2() {
-  echo -e "\n\n${COLOR_BLUE}Installing pm2...${COLOR_NC}\n\n"
-  if [ "$GLOBAL" = "y" ]; then
-    npm install pm2@latest -g
-  else
-    sudo npm install pm2@latest -g
-  fi
-}
-
-install_redis() {
-  echo -e "\n\n${COLOR_BLUE}Installing redis...${COLOR_NC}\n\n"
-  sudo apt install -y redis-server
-  sudo sed -ie 's/supervised no/supervised systemd/' /etc/redis/redis.conf
-  sudo systemctl restart redis.service
-}
-
-install_erlang() {
-  echo -e "\n\n${COLOR_BLUE}Installing erlang...${COLOR_NC}\n\n"
-  sudo apt -y install erlang
-}
-
-#ask user for rabbit credentials
-rabbit_credentials() {
-  read -p "Enter rabbitmq user [hyperion]: " RABBIT_USER
-  RABBIT_USER=${RABBIT_USER:-hyperion}
-
-  read -p "Enter rabbitmq password [123456]: " RABBIT_PASSWORD
-  RABBIT_PASSWORD=${RABBIT_PASSWORD:-123456}
-
-}
-
-install_rabittmq() {
-  echo -e "\n\n${COLOR_BLUE}Installing rabbit-mq...${COLOR_NC}\n\n"
-  sudo apt install -y rabbitmq-server
-  #enable web gui
-  sudo rabbitmq-plugins enable rabbitmq_management
-  sudo rabbitmqctl add_vhost /hyperion
-  sudo rabbitmqctl add_user ${RABBIT_USER} ${RABBIT_PASSWORD}
-  sudo rabbitmqctl set_user_tags ${RABBIT_USER} administrator
-  sudo rabbitmqctl set_permissions -p /hyperion ${RABBIT_USER} ".*" ".*" ".*"
-}
-
-install_elastic() {
-  echo -e "\n\n${COLOR_BLUE}Installing elastic...${COLOR_NC}\n\n"
-  sudo apt install -y elasticsearch
-
-  echo -e "\n\n${COLOR_BLUE}Configuring elastic...${COLOR_NC}\n\n"
-
-  # edit configs
-  sudo sed -ie 's/#cluster.name: my-application/cluster.name: myCluster/; s/#bootstrap.memory_lock: true/bootstrap.memory_lock: true/' /etc/elasticsearch/elasticsearch.yml
-  # set jvm options based on system RAM
-  check_ram
-  if [ "$RAM" -lt 32 ]; then
-    (( RAM=RAM/2 ))
-    sudo sed -ie 's/-Xms1g/-Xms'"$RAM"'g/; s/-Xmx1g/-Xmx'"$RAM"'g/' /etc/elasticsearch/jvm.options
-  else
-    sudo sed -ie 's/-Xms1g/-Xms16g/; s/-Xmx1g/-Xmx16g/' /etc/elasticsearch/jvm.options
-  fi
-
-  sudo bash -c 'echo "xpack.security.enabled: true" >> /etc/elasticsearch/elasticsearch.yml'
-
-  sudo mkdir -p /etc/systemd/system/elasticsearch.service.d/
-  echo -e "[Service]\nLimitMEMLOCK=infinity" | sudo tee /etc/systemd/system/elasticsearch.service.d/override.conf
-  sudo systemctl daemon-reload
-  sudo service elasticsearch start
-  sudo systemctl enable elasticsearch
-
-  echo -e "\n\n${COLOR_BLUE}Generating Elasticsearch cluster passwords...${COLOR_NC}\n\n"
-
-  echo "y" | sudo /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto >elastic_pass.txt
-
-}
-
-install_kibana() {
-  echo -e "\n\n${COLOR_BLUE}Installing and configuring Kibana...${COLOR_NC}\n\n"
-  sudo apt install -y kibana
-
-  KIBANA_PASSWORD=$(awk <elastic_pass.txt '/PASSWORD kibana =/ {print $4}')
-
-  sudo sed -ie 's/#elasticsearch.password: "pass"/elasticsearch.password: '"$KIBANA_PASSWORD"'/; s/#elasticsearch.username: "kibana"/elasticsearch.username: "kibana"/' /etc/kibana/kibana.yml
-
-  sudo systemctl enable kibana
-  sudo systemctl start kibana
-}
-
-#******************
-# End of functions
-#******************
-arg_checker
-
-echo -e "\n\n${COLOR_BLUE}*** STARTING HYPERION HISTORY API CONFIGURATION ***${COLOR_NC}\n\n"
-
-check_dependencies
-
-if [ "$RABBIT" = false ]; then
-  rabbit_credentials
-fi
-
-install_dep
-
-install_keys_sources
-if [ "$NODE" = false ]; then
-  install_node
-fi
-
-install_pm2
-
-install_build_hyperion
-
-if [ "$RABBIT" = false ]; then
-  install_rabittmq
-fi
-
-if [ "$REDIS" = false ]; then
-  install_redis
-fi
-
-if [ "$ELASTIC" = false ]; then
-  install_elastic
-  install_kibana
-fi
-
-printf "
- _   ___   ______  _____ ____  ___ ___  _   _
-| | | \ \ / /  _ \| ____|  _ \|_ _/ _ \| \ | |
-| |_| |\ V /| |_) |  _| | |_) || | | | |  \| |
-|  _  | | | |  __/| |___|  _ < | | |_| | |\  |
-|_| |_| |_| |_|   |_____|_| \_\___\___/|_| \_|
-Made with ♥ by EOS Rio
-"
-echo -e "\n\n${COLOR_GREEN}Hyperion History API successfully installed!!${COLOR_NC}"
-echo -e "\n${COLOR_YELLOW}Please, check your installation: ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- Elastic: http://localhost:9200 ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- Kibana: http://localhost:5601 ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- RabbitMQ: http://localhost:15672 ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- Refer to the elastic_pas.txt file for the elasticsearch passwords. ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- Make pm2 auto-boot at server restart: \$ pm2 startup ${COLOR_NC}\n"

README.md

@@ -1,54 +1,99 @@
-# Hyperion History API
+<!--suppress HtmlUnknownTarget, HtmlDeprecatedAttribute -->
+<br></br>
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://eosrio.io/hyperion-white.png">
+    <img alt="Hyperion Logo"
+         src="https://eosrio.io/hyperion.png">
+  </picture>
+</p>
 
-<img height="64" src="https://eosrio.io/hyperion.png">
-<br/>
-Scalable Full History API Solution for EOSIO based blockchains
+<h4 align="center">
+    Scalable Full History API Solution for 
+    <a href="https://antelope.io">
+        Antelope
+    </a>
+    (former EOSIO) based blockchains
+</h4>
 
-Made with ♥ by [EOS Rio](https://eosrio.io/)
+<br>
 
-Official documentation: https://hyperion.docs.eosrio.io/
+Made with ♥ by [Rio Blocks / EOS Rio](https://rioblocks.io/?lang=en)
+
+### [Official documentation](https://hyperion.docs.eosrio.io)
+
+### How to use
+
+ - [For Providers](https://hyperion.docs.eosrio.io/manual_installation/)
+
+ - [For Developers](https://hyperion.docs.eosrio.io/howtouse/)
+
+### Official plugins:
+
+- [Hyperion Lightweight Explorer](https://github.com/eosrio/hyperion-explorer-plugin)
 
 ### 1. Overview
 
-Hyperion is a full history solution for indexing, storing and retrieving EOSIO blockchain`s historical data. EOSIO protocol is highly scalable reaching up to tens of thousands of transactions per second demanding high performance indexing and optimized storage and querying solutions. Hyperion is developed to tackle those challenges providing open source software to be operated by block producers, infrastructure providers and dApp developers.
-
-Focused on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers, Hyperion implements an improved data structure
-actions are stored in a flattened format
-a parent field is added to the inline actions to point to the parent global sequence
-if the inline action data is identical to the parent it is considered a notification and thus removed from the database
-no blocks or transaction data is stored, all information can be reconstructed from actions
+Hyperion is a full history solution for indexing, storing and retrieving Antelope blockchain's historical data.
+Antelope protocol is highly scalable reaching up to tens of thousands of transactions per second demanding high
+performance indexing and optimized storage and querying solutions. Hyperion is developed to tackle those challenges
+providing open source software to be operated by block producers, infrastructure providers and dApp developers.
 
+Focused on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers,
+Hyperion implements an improved data structure. Actions are stored in a flattened format, transaction ids are added to 
+all inline actions, allowing to group by transaction without storing a full transaction index. Besides that if the inline 
+action data is identical to the parent, it is considered a notification and thus removed from the database.
+No full block or transaction data is stored, all information can be reconstructed from actions and deltas, only a block 
+header index is stored.
 
 ### 2. Architecture
-The following components are required in order to have a fully functional Hyperion API deployment, 
-for small use cases its fine to run all components on a single machine. But for larger chains and 
-production environments we recommend setting them up into different servers under a high-speed local network.
 
-#### 2.1 - Elasticsearch Cluster
+The following components are required in order to have a fully functional Hyperion API deployment.
+* For small use cases, it is absolutely fine to run all components on a single machine.
+* For larger chains and production environments, we recommend setting them up into different servers under a high-speed local network.
+
+#### 2.1 Elasticsearch Cluster
+
 The ES cluster is responsible for storing all indexed data.
-Direct access to the Hyperion API and Indexer must be provided. We recommend nodes in the 
-cluster to have at least 32GB of RAM and 8 cpu cores. SSD/NVME drives are recommended for 
-maximum indexing throughput. For production environments a multi-node cluster is highly recommended.
+Direct access to the Hyperion API and Indexer must be provided. We recommend nodes in the
+cluster to have at least 32 GB of RAM and 8 cpu cores. SSD/NVME drives are recommended for
+maximum indexing throughput, although HDDs can be used for cold storage nodes.
+For production environments, a multi-node cluster is highly recommended.
+
+#### 2.2 Hyperion Indexer
 
-#### 2.2 - Hyperion Indexer
 The Indexer is a Node.js based app that process data from the state history plugin and allows it to be indexed.
-The PM2 process manager is used to launch and operate the indexer. The configuration flexibility is very extensive, 
-so system recommendations will depend on the use case and data load. It will require access to at least one ES node, 
+The [PM2 process manager](https://pm2.keymetrics.io) is used to launch and operate the indexer. The configuration
+flexibility is very extensive,
+so system recommendations will depend on the use case and data load. It will require access to at least one ES node,
 RabbitMQ and the state history node.
 
-#### 2.3 - Hyperion API
+#### 2.3 Hyperion API
+
 Parallelizable API server that provides the V2 and V1 (legacy history plugin) endpoints.
-It is launched by PM2 and can also operate in cluster mode. It requires direct access to 
+It is launched by PM2 and can also operate in cluster mode. It requires direct access to
 at least one ES node for the queries and all other services for full healthcheck
 
-#### 2.4 - RabbitMQ
-Use as messaging queue and data transport between the indexer stages
+#### 2.4 RabbitMQ
 
-#### 2.5 - EOSIO State History
-Nodeos plugin used to collect action traces and state deltas. Provides data via websocket to the indexer
+Used as messaging queue and data transport between the indexer stages and for real-time data streaming
 
-### 3. How to use
+#### 2.5 Redis
 
-#### 3.1 [For Providers](https://eosrio.github.io/hyperion-docs/quickstart/)
+Used for transient data storage across processes and for the preemptive transaction caching used on
+the `v2/history/get_transaction` and `v2/history/check_transaction` endpoints
 
-#### 3.2 [For Developers](https://eosrio.github.io/hyperion-docs/howtouse/)
+#### 2.6 Leap State History
+
+[Leap / Nodeos](https://github.com/AntelopeIO/leap/tree/main/plugins/state_history_plugin) plugin used
+to collect action traces and state deltas. Provides data via websocket to the indexer
+
+#### 2.7 Hyperion Stream Client (optional)
+
+Web and Node.js client for real-time streaming on enabled hyperion
+providers. [Documentation](https://hyperion.docs.eosrio.io/dev/stream_client/)
+
+#### 2.8 Hyperion Plugins (optional)
+
+Hyperion includes a flexible plugin architecture to allow further customization.
+Plugins are managed by the `hpm` (hyperion plugin manager) command line tool.

api/config/open_api.ts

@@ -4,6 +4,7 @@ export function generateOpenApiConfig(config: HyperionConfig) {
     const packageData = require('../../package');
     const health_link = `https://${config.api.server_name}/v2/health`;
     const explorer_link = `https://${config.api.server_name}/v2/explore`;
+
     let description = `
 <img height="64" src="https://eosrio.io/hyperion.png">
 ### Scalable Full History API Solution for EOSIO based blockchains
@@ -13,9 +14,15 @@ export function generateOpenApiConfig(config: HyperionConfig) {
 #### Provided by [${config.api.provider_name}](${config.api.provider_url})
 #### Health API: <a target="_blank" href="${health_link}">${health_link}</a>
 `;
-    if(config.api.enable_explorer) {
-        description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
+
+    if (config.plugins) {
+        if (config.plugins.explorer) {
+            if (config.plugins.explorer.enabled) {
+                description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
+            }
+        }
     }
+
     return {
         routePrefix: '/v2/docs',
         exposeRoute: true,

api/helpers/cacheManager.ts

@@ -0,0 +1,86 @@
+import {HyperionConfig} from "../../interfaces/hyperionConfig";
+import {createHash} from "crypto";
+import {FastifyRequest} from "fastify";
+
+export interface CacheConfig {
+    ttl: number;
+}
+
+export interface CachedEntry {
+    data: string;
+    exp: number;
+}
+
+export class CacheManager {
+
+    v1CacheConfigs: Map<string, CacheConfig> = new Map<string, CacheConfig>();
+    v1Caches: Map<string, Map<string, CachedEntry>> = new Map<string, Map<string, CachedEntry>>();
+
+    constructor(conf: HyperionConfig) {
+        if (conf.api.v1_chain_cache) {
+            conf.api.v1_chain_cache.forEach(value => {
+                this.v1CacheConfigs.set(value.path, {
+                    ttl: value.ttl
+                });
+                this.v1Caches.set(value.path, new Map<string, CachedEntry>());
+            });
+        }
+        setInterval(() => {
+            try {
+                // remove expired entries
+                // let removeCount = 0;
+                const now = Date.now();
+                this.v1Caches.forEach((pathCacheMap: Map<string, CachedEntry>, pathKey: string) => {
+                    pathCacheMap.forEach((cache: CachedEntry, entryKey: string, map: Map<string, CachedEntry>) => {
+                        if (cache.exp < now) {
+                            map.delete(entryKey);
+                            // removeCount++;
+                        }
+                    });
+                });
+                // if (removeCount > 0) {
+                //     console.log(`${removeCount} expired cache entries removed`);
+                // }
+            } catch (e) {
+                console.log(e);
+            }
+        }, 5000);
+    }
+
+    hashPayload(input: string): string {
+        return createHash('sha256').update(input).digest().toString('hex');
+    }
+
+    setCachedData(hash: string, path: string, payload: any): void {
+        if (this.v1CacheConfigs.has(path) && this.v1Caches.has(path)) {
+            this.v1Caches.get(path).set(hash, {
+                data: payload,
+                exp: this.v1CacheConfigs.get(path).ttl + Date.now()
+            });
+        }
+    }
+
+    getCachedData(request: FastifyRequest): [string | null, string, string] {
+        const urlParts = request.url.split("?");
+        const pathComponents = urlParts[0].split('/');
+        const path = pathComponents.at(-1);
+        let payload = '';
+        if (request.method === 'POST') {
+            payload = JSON.stringify(request.body);
+        } else if (request.method === 'GET') {
+            payload = request.url;
+        }
+        const hashedString = this.hashPayload(payload);
+        if (this.v1Caches.has(path)) {
+            const entry = this.v1Caches.get(path).get(hashedString);
+            if (entry && entry.exp > Date.now()) {
+                return [entry.data, hashedString, path];
+            } else {
+                return [null, hashedString, path];
+            }
+        } else {
+            return [null, hashedString, path];
+        }
+    }
+
+}

api/plugins.ts

@@ -2,22 +2,32 @@ import * as Fastify from "fastify";
 import {IncomingMessage, Server, ServerResponse} from "http";
 
 // fastify plugins
-import * as fastify_elasticsearch from 'fastify-elasticsearch';
-import * as fastify_oas from 'fastify-oas';
-import * as fastify_cors from 'fastify-cors';
-import * as fastify_formbody from 'fastify-formbody';
-import * as fastify_redis from 'fastify-redis';
-import * as fastify_rate_limit from 'fastify-rate-limit';
+import * as fastifyElasticsearch from 'fastify-elasticsearch';
+import fastifySwagger from '@fastify/swagger';
+import fastifyCors from '@fastify/cors';
+import formBodyPlugin from '@fastify/formbody';
+import fastifyRedis from '@fastify/redis';
+import fastifyRateLimit from '@fastify/rate-limit';
 
 // custom plugins
 import fastify_eosjs from "./plugins/fastify-eosjs";
 
 export function registerPlugins(server: Fastify.FastifyInstance<Server, IncomingMessage, ServerResponse>, params: any) {
-    server.register(fastify_elasticsearch, params.fastify_elasticsearch);
-    server.register(fastify_oas, params.fastify_oas);
-    server.register(fastify_cors);
-    server.register(fastify_formbody);
-    server.register(fastify_redis, params.fastify_redis);
+    server.register(fastifyElasticsearch, params.fastify_elasticsearch);
+
+    if (params.fastify_swagger) {
+        server.register(fastifySwagger, params.fastify_swagger);
+    }
+
+    server.register(fastifyCors);
+
+    server.register(formBodyPlugin);
+
+    server.register(fastifyRedis, params.fastify_redis);
+
+    if (params.fastify_rate_limit) {
+        server.register(fastifyRateLimit, params.fastify_rate_limit);
+    }
+
     server.register(fastify_eosjs, params.fastify_eosjs);
-    server.register(fastify_rate_limit, params.fastify_rate_limit);
 }

api/plugins/eosjs.js

@@ -1,21 +0,0 @@
-const fp = require('fastify-plugin');
-const {Api} = require("eosjs");
-const {ConnectionManager} = require('../../connections/manager');
-const manager = new ConnectionManager();
-
-module.exports = fp(async (fastify, options, next) => {
-    const rpc = manager.nodeosJsonRPC;
-    const chain_data = await rpc.get_info();
-    const api = new Api({
-        rpc,
-        signatureProvider: null,
-        chainId: chain_data.chain_id,
-        textDecoder: new TextDecoder(),
-        textEncoder: new TextEncoder(),
-    });
-    fastify.decorate('eosjs', {api, rpc});
-    next();
-}, {
-    fastify: '>=2.0.0',
-    name: 'fastify-eosjs'
-});

api/plugins/fastify-eosjs.ts

@@ -1,19 +1,17 @@
-import * as fp from 'fastify-plugin';
-import {FastifyInstance} from "fastify";
+import {FastifyInstance, FastifyPluginOptions} from "fastify";
 import {Api} from "eosjs/dist";
+import fp from "fastify-plugin";
 
-export default fp(async (fastify: FastifyInstance, options, next) => {
+export default fp(async (fastify: FastifyInstance, options: FastifyPluginOptions): Promise<void> => {
     const rpc = fastify.manager.nodeosJsonRPC;
-    const chain_data = await rpc.get_info();
     const api = new Api({
         rpc,
         signatureProvider: null,
-        chainId: chain_data.chain_id,
+        chainId: options.chain_id,
         textDecoder: new TextDecoder(),
         textEncoder: new TextEncoder(),
     });
     fastify.decorate('eosjs', {api, rpc});
-    next();
 }, {
     fastify: '>=2.0.0',
     name: 'fastify-eosjs'

api/plugins/websocket.js

@@ -1,30 +0,0 @@
-const fp = require('fastify-plugin');
-const WebSocket = require('ws');
-
-module.exports = fp((fastify, opts, next) => {
-
-    const wss = new WebSocket.Server({
-        server: fastify.server
-    });
-
-    const ws = new WebSocket('ws://127.0.0.1:7001', [], {
-        perMessageDeflate: false
-    });
-
-    ws.on('open', function open() {
-        console.log('router connected');
-    });
-
-    fastify.decorate('wss', wss);
-    fastify.decorate('ws', ws);
-
-    fastify.addHook('onClose', (fastify, done) => {
-        fastify.ws.close(done);
-        fastify.wss.close(done);
-    });
-
-    next();
-}, {
-    fastify: '2.x',
-    name: 'fastify-websocket'
-});

api/routes.ts

@@ -1,27 +1,28 @@
-import * as fastify_static from "fastify-static";
 import {join} from "path";
-import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
-import {createReadStream, existsSync, readFileSync, unlinkSync} from "fs";
-import * as AutoLoad from "fastify-autoload";
+import {FastifyError, FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {createReadStream} from "fs";
 import {addSharedSchemas, handleChainApiRedirect} from "./helpers/functions";
+import autoLoad from '@fastify/autoload';
 import got from "got";
 
 function addRedirect(server: FastifyInstance, url: string, redirectTo: string) {
     server.route({
         url,
         method: 'GET',
-        schema: {hide: true},
-        handler: async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        schema: {
+            hide: true
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
             reply.redirect(redirectTo);
         }
     });
 }
 
 function addRoute(server: FastifyInstance, handlersPath: string, prefix: string) {
-    server.register(AutoLoad, {
+    server.register(autoLoad, {
         dir: join(__dirname, 'routes', handlersPath),
         ignorePattern: /.*(handler|schema).js/,
+        dirNameRoutePrefix: false,
         options: {prefix}
     });
 }
@@ -36,11 +37,11 @@ export function registerRoutes(server: FastifyInstance) {
         '/v2/history',
         '/v2/state',
         '/v1/chain/*',
-        '/v1/chain',
+        '/v1/chain'
     ];
     server.addHook('onRoute', opts => {
         if (!ignoreList.includes(opts.url)) {
-            if (opts.url.startsWith('/v') && !opts.url.startsWith('/v2/explore') && !opts.url.startsWith('/v2/docs')) {
+            if (opts.url.startsWith('/v')) {
                 routeSet.add(opts.url);
             }
         }
@@ -60,6 +61,8 @@ export function registerRoutes(server: FastifyInstance) {
 
     // chain api redirects
     addRoute(server, 'v1-chain', '/v1/chain');
+
+    // other v1 requests
     server.route({
         url: '/v1/chain/*',
         method: ["GET", "POST"],
@@ -67,7 +70,7 @@ export function registerRoutes(server: FastifyInstance) {
             summary: "Wildcard chain api handler",
             tags: ["chain"]
         },
-        handler: async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
             await handleChainApiRedirect(request, reply, server);
         }
     });
@@ -80,7 +83,7 @@ export function registerRoutes(server: FastifyInstance) {
             summary: "Get list of supported APIs",
             tags: ["node"]
         },
-        handler: async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
             const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
             if (data.apis && data.apis.length > 0) {
                 const apiSet = new Set(server.routeSet);
@@ -92,88 +95,39 @@ export function registerRoutes(server: FastifyInstance) {
         }
     });
 
-    server.addHook('onError', (request, reply, error, done) => {
-        console.log(`[${request.req.headers['x-real-ip']}] ${request.req.method} ${request.req.url} failed with error: ${error.message}`);
+    // global onRequest hook to collect hourly statistics for each path on redis
+    server.addHook('onResponse', (request: FastifyRequest, reply: FastifyReply, done) => {
+        const path = request.url.split('?')[0].replace(/\/$/, '');
+        const now = new Date();
+        const ts = now.getTime();
+        // normalized time in hours
+        const factor = 1000 * 60 * 60;
+        const ts_hour_unix = Math.floor(ts / factor) * factor;
+        const key = `stats:${server.manager.chain}:H:${ts_hour_unix}`;
+        // console.log(reply.statusCode, key, new Date(ts_hour_unix));
+        server.redis.multi()
+            .incr(`stats:${server.manager.chain}:total:${reply.statusCode}:${path}`)
+            .zincrby(key, 1, `[${reply.statusCode}]${path}`)
+            .expire(key, 60 * 60 * 24 * 7)
+            .exec()
+            .catch(console.log);
         done();
     });
 
-    // server.addHook('onResponse', (request, reply, done) => {
-    //     if (reply.res.statusCode !== 200) {
-    //         console.log(`${request.req.url} - code: ${reply.res.statusCode}`);
-    //     }
-    //     done();
-    // });
-
-    // Serve integrated explorer
-    if (server.manager.config.api.enable_explorer) {
-
-        server.register(require('fastify-compress'), {global: false});
-
-        try {
-            const _data = readFileSync(join(__dirname, '..', 'hyperion-explorer', 'src', 'manifest.webmanifest'));
-            const tempPath = join(__dirname, '..', 'hyperion-explorer', 'dist', 'manifest.webmanifest');
-            if (existsSync(tempPath)) {
-                unlinkSync(tempPath);
-            }
-            const baseManifest = JSON.parse(_data.toString());
-            baseManifest.name = `Hyperion Explorer - ${server.manager.config.api.chain_name}`;
-            baseManifest.short_name = baseManifest.name;
-            server.get('/v2/explore/manifest.webmanifest', (request, reply) => {
-                reply.send(baseManifest);
-            });
-        } catch (e) {
-            console.log(e);
-        }
-
-        server.register(fastify_static, {
-            root: join(__dirname, '..', 'hyperion-explorer', 'dist'),
-            redirect: true,
-            wildcard: false,
-            prefix: '/v2/explore',
-            setHeaders: (res: ServerResponse, path) => {
-                if (path.endsWith('/ngsw-worker.js')) {
-                    res.setHeader('Service-Worker-Allowed', '/');
-                }
-            }
-        });
-
-        server.get(
-            '/v2/explore/**/*',
-            {
-                schema: {
-                    tags: ['internal']
-                }
-            },
-            (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-                reply.sendFile('index.html', join(__dirname, '..', 'hyperion-explorer', 'dist'));
-            }
-        );
-
-        server.get(
-            '/v2/explorer_metadata',
-            {
-                schema: {
-                    tags: ['internal']
-                }
-            },
-            (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-                reply.send({
-                    logo: server.manager.config.api.chain_logo_url,
-                    provider: server.manager.config.api.provider_name,
-                    provider_url: server.manager.config.api.provider_url,
-                    chain_name: server.manager.config.api.chain_name,
-                    chain_id: server.manager.conn.chains[server.manager.chain].chain_id,
-                    custom_core_token: server.manager.config.api.custom_core_token
-                });
-            });
-    }
+    server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
+        console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
+        done();
+    });
 
     if (server.manager.config.features.streaming) {
         // steam client lib
-        server.get('/stream-client.js', {schema: {tags: ['internal']}}, (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-            const stream = createReadStream('./client_bundle.js');
-            reply.type('application/javascript').send(stream);
-        });
+        server.get(
+            '/stream-client.js',
+            {schema: {tags: ['internal']}},
+            (request: FastifyRequest, reply: FastifyReply) => {
+                const stream = createReadStream('./hyperion-stream-client.js');
+                reply.type('application/javascript').send(stream);
+            });
     }
 
     // Redirect routes to documentation

api/routes/history/getTransactedAccounts.js

@@ -1,200 +0,0 @@
-const {getCacheByHash} = require("../../helpers/functions");
-
-const {getTransactedAccountsSchema} = require("../../schemas");
-
-async function getTransactedAccounts(fastify, request) {
-    const t0 = Date.now();
-    const {redis, elastic} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, JSON.stringify(request.query));
-    if (cachedResponse) {
-        return cachedResponse;
-    }
-    const {account, min, max, direction, limit, contract, symbol} = request.query;
-    let _limit = 100;
-    if (limit) {
-        _limit = parseInt(limit, 10);
-        if (_limit > 500) {
-            _limit = 500;
-        }
-    }
-    let _min;
-    if (min) {
-        _min = parseInt(min, 10);
-        if (_min < 0) {
-            _min = 0;
-        }
-    }
-    let _max;
-    if (max) {
-        _max = parseInt(max, 10);
-        if (_max < 0) {
-            _max = null;
-        }
-    }
-
-    const response = {
-        query_time: null,
-        account: account
-    };
-
-    const must_array = [
-        {term: {"notified": account}},
-        {term: {"act.name": "transfer"}}
-    ];
-
-    if (min || max) {
-        const _range = {
-            "@transfer.amount": {}
-        };
-        if (min) {
-            _range["@transfer.amount"]["gte"] = _min;
-        }
-        if (max) {
-            _range["@transfer.amount"]["lt"] = _max;
-        }
-        must_array.push({range: _range});
-    }
-
-    if (contract) {
-        must_array.push({term: {"act.account": contract}});
-        response['contract'] = contract;
-    }
-
-    if (symbol) {
-        must_array.push({term: {"@transfer.symbol": symbol}});
-        response['symbol'] = symbol;
-    }
-
-    const sum_agg = {
-        "total_amount": {
-            sum: {
-                field: "@transfer.amount"
-            }
-        }
-    };
-
-    let filter_array = [];
-    if (request.query['after'] || request.query['before']) {
-        let _lte = "now";
-        let _gte = 0;
-        if (request.query['before']) {
-            _lte = request.query['before'];
-        }
-        if (request.query['after']) {
-            _gte = request.query['after'];
-        }
-        filter_array.push({
-            range: {
-                "@timestamp": {
-                    "gte": _gte,
-                    "lte": _lte
-                }
-            }
-        });
-    } else {
-        filter_array.push({
-            range: {
-                "block_num": {
-                    "gte": 0
-                }
-            }
-        });
-    }
-
-    if (direction === 'out' || direction === 'both') {
-        const outResults = await elastic.search({
-            index: process.env.CHAIN + '-action-*',
-            body: {
-                size: 0,
-                aggs: {
-                    "receivers": {
-                        terms: {
-                            field: "@transfer.to",
-                            size: _limit,
-                            order: {
-                                "total_amount": "desc"
-                            }
-                        },
-                        aggs: sum_agg
-                    }
-                },
-                query: {
-                    bool: {
-                        must: must_array,
-                        filter: filter_array,
-                        must_not: [
-                            {term: {"@transfer.to": account}}
-                        ]
-                    }
-                }
-            }
-        });
-        response['total_out'] = 0;
-        response['outputs'] = outResults['body']['aggregations']['receivers']['buckets'].map((bucket) => {
-            const _sum = parseFloat(bucket.total_amount.value.toFixed(4));
-            response['total_out'] += _sum;
-            return {
-                account: bucket.key,
-                sum: _sum,
-                transfers: bucket['doc_count'],
-                average: parseFloat((bucket.total_amount.value / bucket['doc_count']).toFixed(4))
-            };
-        });
-        response['total_out'] = parseFloat(response['total_out'].toFixed(4));
-    }
-
-    if (direction === 'in' || direction === 'both') {
-        const inResults = await elastic.search({
-            index: process.env.CHAIN + '-action-*',
-            body: {
-                size: 0,
-                aggs: {
-                    "senders": {
-                        terms: {
-                            field: "@transfer.from",
-                            size: _limit,
-                            order: {
-                                "total_amount": "desc"
-                            }
-                        },
-                        aggs: sum_agg
-                    }
-                },
-                query: {
-                    bool: {
-                        must: must_array,
-                        filter: filter_array,
-                        must_not: [
-                            {term: {"@transfer.from": account}}
-                        ]
-                    }
-                }
-            }
-        });
-        response['total_in'] = 0;
-        response['inputs'] = inResults['body']['aggregations']['senders']['buckets'].map((bucket) => {
-            const _sum = parseFloat(bucket.total_amount.value.toFixed(4));
-            response['total_in'] += _sum;
-            return {
-                account: bucket.key,
-                sum: _sum,
-                transfers: bucket['doc_count'],
-                average: parseFloat((bucket.total_amount.value / bucket['doc_count']).toFixed(4))
-            };
-        });
-        response['total_in'] = parseFloat(response['total_in'].toFixed(4));
-    }
-
-    response['query_time'] = Date.now() - t0;
-    redis.set(hash, JSON.stringify(response), 'EX', 600);
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get('/get_transacted_accounts', {
-        schema: getTransactedAccountsSchema.GET
-    }, async (request, reply) => {
-        reply.send(await getTransactedAccounts(fastify, request));
-    });
-    next()
-};

api/routes/history/getTransfers.js

@@ -1,98 +0,0 @@
-const {getTransfersSchema} = require("../../schemas");
-const _ = require('lodash');
-const {getCacheByHash} = require("../../helpers/functions");
-const route = '/get_transfers';
-
-const maxActions = 1000;
-
-function processActions(results) {
-    const action_traces = results['body']['hits']['hits'];
-    const actions = [];
-    let sum = 0;
-    for (const aTrace of action_traces) {
-        const action = aTrace['_source'];
-        const name = action.act.name;
-        if (action['@' + name]) {
-            if (action['@transfer']) {
-                sum += parseFloat(action['@transfer']['amount']);
-            }
-            action['act']['data'] = _.merge(action['@' + name], action['act']['data']);
-            delete action['@' + name];
-        }
-        actions.push(action);
-    }
-    return [sum, actions];
-}
-
-async function getTransfers(fastify, request) {
-    const {redis, elastic} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, route + JSON.stringify(request.query));
-    if (cachedResponse) {
-        return cachedResponse;
-    }
-    const must_array = [];
-    must_array.push({"term": {"act.name": {"value": "transfer"}}});
-    if (request.query['from']) {
-        must_array.push({"term": {"@transfer.from": {"value": request.query['from'].toLowerCase()}}});
-    }
-    if (request.query['to']) {
-        must_array.push({"term": {"@transfer.to": {"value": request.query['to'].toLowerCase()}}});
-    }
-    if (request.query['symbol']) {
-        must_array.push({"term": {"@transfer.symbol": {"value": request.query['symbol'].toUpperCase()}}});
-    }
-    if (request.query['contract']) {
-        must_array.push({"term": {"act.account": {"value": request.query['contract'].toLowerCase()}}});
-    }
-    if (request.query['after'] || request.query['before']) {
-        let _lte = "now";
-        let _gte = 0;
-        if (request.query['before']) {
-            _lte = request.query['before'];
-        }
-        if (request.query['after']) {
-            _gte = request.query['after'];
-        }
-        must_array.push({
-            range: {
-                "@timestamp": {
-                    "gte": _gte,
-                    "lte": _lte
-                }
-            }
-        });
-    }
-    let limit, skip;
-    limit = parseInt(request.query.limit, 10);
-    if (limit < 1) {
-        return 'invalid limit parameter';
-    }
-    skip = parseInt(request.query.skip, 10);
-    if (skip < 0) {
-        return 'invalid skip parameter';
-    }
-    const body = {"query": {"bool": {"must": must_array}}};
-    const results = await elastic.search({
-        "index": process.env.CHAIN + '-action-*',
-        "from": skip || 0,
-        "size": (limit > maxActions ? maxActions : limit) || 10,
-        "body": body
-    });
-    const [sum, _actions] = processActions(results);
-    const response = {
-        "action_count": results['body']['hits']['total']['value'],
-        "total_amount": sum,
-        "actions": _actions
-    };
-    redis.set(hash, JSON.stringify(response), 'EX', 30);
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get(route, {
-        schema: getTransfersSchema.GET
-    }, async (request, reply) => {
-        reply.send(await getTransfers(fastify, request));
-    });
-    next()
-};

api/routes/v1-chain/abi_bin_to_json/index.ts

@@ -7,13 +7,9 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Returns an object containing rows from the specified table.',
         {
-            "code": 'AccountName#',
-            "action": 'AccountName#',
-            "binargs": {
-                "type": "string",
-                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
-                "title": "Hex"
-            }
+            "code": {$ref: 'AccountName#'},
+            "action": {$ref: 'AccountName#'},
+            "binargs": {"type": "string"}
         },
         ["code", "action", "binargs"]
     );

api/routes/v1-chain/abi_json_to_bin/index.ts

@@ -7,13 +7,14 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Convert JSON object to binary',
         {
-            "binargs": {
-                "type": "string",
-                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
-                "title": "Hex"
+            "code": {$ref: 'AccountName#'},
+            "action": {$ref: 'AccountName#'},
+            "args": {
+                type: 'object',
+                additionalProperties: true
             }
         },
-        ["binargs"]
+        ["code", "action", "args"]
     );
     next();
 }

api/routes/v1-chain/get_abi/index.ts

@@ -6,7 +6,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         fastify,
         getRouteName(__filename),
         'Retrieves the ABI for a contract based on its account name',
-        {"account_name": 'AccountName#'},
+        {"account_name": {$ref: 'AccountName#'}},
         ["account_name"]
     );
     next();

api/routes/v1-chain/get_account/index.ts

@@ -6,7 +6,9 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         fastify,
         getRouteName(__filename),
         'Returns an object containing various details about a specific account on the blockchain.',
-        {"account_name": 'AccountName#'},
+        {
+            "account_name": {$ref: 'AccountName#'}
+        },
         ["account_name"]
     );
     next();

api/routes/v1-chain/get_code/index.ts

@@ -7,7 +7,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves contract code',
         {
-            "account_name": 'AccountName#',
+            "account_name": {$ref: 'AccountName#'},
             "code_as_wasm": {
                 "type": "integer",
                 "default": 1,

api/routes/v1-chain/get_currency_balance/index.ts

@@ -7,9 +7,9 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves the current balance',
         {
-            "code": 'AccountName#',
-            "account": 'AccountName#',
-            "symbol": 'Symbol#',
+            "code": {$ref: 'AccountName#'},
+            "account": {$ref: 'AccountName#'},
+            "symbol": {$ref: 'Symbol#'}
         },
         ["code", "account", "symbol"]
     );

api/routes/v1-chain/get_currency_stats/index.ts

@@ -7,12 +7,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves currency stats',
         {
-            "code": {
-                description: 'contract name',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
+            "code": {$ref: 'AccountName#'},
             "symbol": {
                 description: 'token symbol',
                 type: 'string',

api/routes/v1-chain/get_raw_abi/index.ts

@@ -7,7 +7,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves raw ABI for a contract based on account name',
         {
-            "account_name": 'AccountName#'
+            "account_name": {$ref: 'AccountName#'}
         },
         ["account_name"]
     );

api/routes/v1-chain/get_raw_code_and_abi/index.ts

@@ -7,7 +7,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves raw code and ABI for a contract based on account name',
         {
-            "account_name": 'AccountName#'
+            "account_name": {$ref: 'AccountName#'}
         },
         ["account_name"]
     );

api/routes/v1-chain/get_table_rows/index.ts

@@ -36,7 +36,8 @@ export default function (fastify: FastifyInstance, opts: any, next) {
             "lower_bound": {
                 "type": "string"
             }
-        }
+        },
+        ["code", "table", "scope"]
     );
     next();
 }

api/routes/v1-chain/push_transaction/index.ts

@@ -17,7 +17,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     signatures: {
                         "type": "array",
                         "description": "array of signatures required to authorize transaction",
-                        "items": 'Signature#'
+                        "items": {$ref: 'Signature#'}
                     },
                     compression: {
                         "type": "boolean",

api/routes/v1-chain/push_transactions/index.ts

@@ -2,45 +2,54 @@ import {FastifyInstance} from "fastify";
 import {addApiRoute, chainApiHandler, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    addApiRoute(
-        fastify,
-        'POST',
-        getRouteName(__filename),
-        chainApiHandler,
-        {
-            description: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
-            summary: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
-            tags: ['chain'],
-            body: {
-                type: ['array', 'object', 'string'],
-                items: {
-                    type: ['object', 'string'],
-                    additionalProperties: false,
-                    minProperties: 8,
-                    required: [
-                        "expiration",
-                        "ref_block_num",
-                        "ref_block_prefix",
-                        "max_net_usage_words",
-                        "max_cpu_usage_ms",
-                        "delay_sec",
-                        "context_free_actions",
-                        "actions"
-                    ],
-                    properties: {
-                        "expiration": 'Expiration#',
-                        "ref_block_num": {"type": "integer"},
-                        "ref_block_prefix": {"type": "integer"},
-                        "max_net_usage_words": 'WholeNumber#',
-                        "max_cpu_usage_ms": 'WholeNumber#',
-                        "delay_sec": {"type": "integer"},
-                        "context_free_actions": {"type": "array", "items": 'ActionItems#'},
-                        "actions": {"type": "array", "items": 'ActionItems#'},
-                        "transaction_extensions": {"type": "array", "items": 'BlockExtensions#'}
-                    },
-                }
-            }
-        }
-    );
-    next();
+	addApiRoute(
+		fastify,
+		'POST',
+		getRouteName(__filename),
+		chainApiHandler,
+		{
+			description: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+			summary: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+			tags: ['chain'],
+			body: {
+				type: ['array', 'object', 'string'],
+				items: {
+					type: ['object', 'string'],
+					additionalProperties: false,
+					minProperties: 8,
+					required: [
+						"expiration",
+						"ref_block_num",
+						"ref_block_prefix",
+						"max_net_usage_words",
+						"max_cpu_usage_ms",
+						"delay_sec",
+						"context_free_actions",
+						"actions"
+					],
+					properties: {
+						"expiration": {$ref: 'Expiration#'},
+						"ref_block_num": {"type": "integer"},
+						"ref_block_prefix": {"type": "integer"},
+						"max_net_usage_words": {$ref: 'WholeNumber#'},
+						"max_cpu_usage_ms": {$ref: 'WholeNumber#'},
+						"delay_sec": {"type": "integer"},
+						"context_free_actions": {
+							"type": "array",
+							"items": {$ref: 'ActionItems#'}
+						},
+						"actions": {
+							"type": "array",
+							"items": {$ref: 'ActionItems#'}
+						},
+						"transaction_extensions": {
+							"type": "array",
+							"items": {$ref: 'BlockExtensions#'}
+						}
+					},
+				}
+			}
+		}
+	);
+	next();
 }

api/routes/v1-chain/send_transaction/index.ts

@@ -17,7 +17,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     signatures: {
                         "type": "array",
                         "description": "array of signatures required to authorize transaction",
-                        "items": 'Signature#'
+                        "items": {$ref: 'Signature#'}
                     },
                     compression: {
                         "type": "boolean",

api/routes/v1-history/get_actions/get_actions.ts

@@ -1,9 +1,150 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
-import * as flatstr from 'flatstr';
+import {Serialize} from "eosjs";
+import {hLog} from "../../../../helpers/common_functions";
+import * as AbiEOS from "@eosrio/node-abieos";
+import {ApiResponse} from "@elastic/elasticsearch";
+import {TextDecoder, TextEncoder} from "util";
+import {JsonRpc} from "eosjs/dist";
+import {terms} from "../../v2-history/get_actions/definitions";
+
+const abi_remapping = {
+    "_Bool": "bool"
+};
+
+const txEnc = new TextEncoder();
+const txDec = new TextDecoder();
+
+async function fetchAbiHexAtBlockElastic(client, chain, contract_name, last_block, get_json) {
+    try {
+        const _includes = ["actions", "tables", "block"];
+        if (get_json) {
+            _includes.push("abi");
+        } else {
+            _includes.push("abi_hex");
+        }
+        const query = {
+            bool: {
+                must: [
+                    {term: {account: contract_name}},
+                    {range: {block: {lte: last_block}}}
+                ]
+            }
+        };
+        const queryResult: ApiResponse = await client.search({
+            index: `${chain}-abi-*`,
+            body: {
+                size: 1, query,
+                sort: [{block: {order: "desc"}}],
+                _source: {includes: _includes}
+            }
+        });
+        const results = queryResult.body.hits.hits;
+        if (results.length > 0) {
+            const nextRefResponse: ApiResponse = await client.search({
+                index: `${chain}-abi-*`,
+                body: {
+                    size: 1,
+                    query: {
+                        bool: {
+                            must: [
+                                {term: {account: contract_name}},
+                                {range: {block: {gte: last_block}}}
+                            ]
+                        }
+                    },
+                    sort: [{block: {order: "asc"}}],
+                    _source: {includes: ["block"]}
+                }
+            });
+            const nextRef = nextRefResponse.body.hits.hits;
+            if (nextRef.length > 0) {
+                return {
+                    valid_until: nextRef[0]._source.block,
+                    ...results[0]._source
+                };
+            }
+            return results[0]._source;
+        } else {
+            return null;
+        }
+    } catch (e) {
+        hLog(e);
+        return null;
+    }
+}
+
+async function getAbiFromHeadBlock(rpc: JsonRpc, code) {
+    let _abi;
+    try {
+        _abi = (await rpc.get_abi(code)).abi;
+    } catch (e) {
+        hLog(e);
+    }
+    return {abi: _abi, valid_until: null, valid_from: null};
+}
+
+async function getContractAtBlock(esClient, rpc, chain, accountName: string, block_num: number, check_action?: string) {
+    let savedAbi, abi;
+    savedAbi = await fetchAbiHexAtBlockElastic(esClient, chain, accountName, block_num, true);
+    if (savedAbi === null || (savedAbi.actions && !savedAbi.actions.includes(check_action))) {
+        savedAbi = await getAbiFromHeadBlock(rpc, accountName);
+        if (!savedAbi) return [null, null];
+        abi = savedAbi.abi;
+    } else {
+        try {
+            abi = JSON.parse(savedAbi.abi);
+        } catch (e) {
+            hLog(e);
+            return [null, null];
+        }
+    }
+    if (!abi) return [null, null];
+    const initialTypes = Serialize.createInitialTypes();
+    let types;
+    try {
+        types = Serialize.getTypesFromAbi(initialTypes, abi);
+    } catch (e) {
+        let remapped = false;
+        for (const struct of abi.structs) {
+            for (const field of struct.fields) {
+                if (abi_remapping[field.type]) {
+                    field.type = abi_remapping[field.type];
+                    remapped = true;
+                }
+            }
+        }
+        if (remapped) {
+            try {
+                types = Serialize.getTypesFromAbi(initialTypes, abi);
+            } catch (e) {
+                hLog('failed after remapping abi');
+                hLog(accountName, block_num, check_action);
+                hLog(e);
+            }
+        } else {
+            hLog(accountName, block_num);
+            hLog(e);
+        }
+    }
+    const actions = new Map();
+    for (const {name, type} of abi.actions) {
+        actions.set(name, Serialize.getType(types, type));
+    }
+    const result = {types, actions, tables: abi.tables};
+    if (check_action) {
+        if (actions.has(check_action)) {
+            try {
+                AbiEOS['load_abi'](accountName, JSON.stringify(abi));
+            } catch (e) {
+                hLog(e);
+            }
+        }
+    }
+    return [result, abi];
+}
+
 
-const terms = ["notified", "act.authorization.actor"];
 const extendedActions = new Set(["transfer", "newaccount", "updateauth"]);
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
@@ -12,7 +153,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
         request.body = JSON.parse(request.body);
     }
 
-    const reqBody = request.body;
+    const reqBody = request.body as any;
     const should_array = [];
     for (const entry of terms) {
         const tObj = {term: {}};
@@ -41,8 +182,8 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
     }
     pos = parseInt(reqBody.pos || 0, 10);
     offset = parseInt(reqBody.offset || 20, 10);
-    let from, size;
-    from = size = 0;
+    let from = 0;
+    let size = 0;
     if (pos === -1) {
         if (offset < 0) {
             from = 0;
@@ -131,6 +272,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
         }
     };
+    // console.log(JSON.stringify(esOpts, null, 2));
     const pResults = await Promise.all([fastify.eosjs.rpc.get_info(), fastify.elastic['search'](esOpts)]);
     const results = pResults[1];
     const response = {
@@ -155,11 +297,13 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
             actions = actions.slice(index);
         }
-        actions.forEach((action, index) => {
+
+        for (let i = 0; i < actions.length; i++) {
+            let action = actions[i];
             action = action._source;
             let act: any = {
                 "global_action_seq": action.global_sequence,
-                "account_action_seq": index,
+                "account_action_seq": i,
                 "block_num": action.block_num,
                 "block_time": action['@timestamp'],
                 "action_trace": {
@@ -175,7 +319,31 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             };
             mergeActionMeta(action);
             act.action_trace.act = action.act;
-            act.action_trace.act.hex_data = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+
+            if (reqBody.hex_data) {
+                try {
+                    const [contract, _] = await getContractAtBlock(
+                        fastify.elastic,
+                        fastify.eosjs.rpc,
+                        fastify.manager.chain,
+                        action.act.account,
+                        action.block_num
+                    );
+                    action.act.hex_data = Serialize.serializeActionData(
+                        contract,
+                        action.act.account,
+                        action.act.name,
+                        action.act.data,
+                        txEnc,
+                        txDec
+                    );
+                } catch (e: any) {
+                    console.log(e);
+                }
+            }
+
+            // TODO: Optionally re-encode using the original ABI, will increase query time
+            // act.action_trace.act.hex_data = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
             if (action.act.account_ram_deltas) {
                 act.action_trace.account_ram_deltas = action.account_ram_deltas
             }
@@ -185,13 +353,13 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             act.action_trace.receiver = receipt.receiver;
             act.account_action_seq = receipt['recv_sequence'];
             response.actions.push(act);
-        });
+        }
     }
     return response;
 }
 
 export function getActionsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getActions, fastify, request, route));
     }
 }

api/routes/v1-history/get_controlled_accounts/get_controlled_accounts.ts

@@ -1,72 +1,72 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {ApiResponse} from "@elastic/elasticsearch";
 
 async function getControlledAccounts(fastify: FastifyInstance, request: FastifyRequest) {
-    if (typeof request.body === 'string') {
-        request.body = JSON.parse(request.body)
-    }
-    let controlling_account = request.body.controlling_account;
-    const results: ApiResponse = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        size: 100,
-        body: {
-            query: {
-                bool: {
-                    should: [
-                        {
-                            term: {"@updateauth.auth.accounts.permission.actor": controlling_account}},
-                        {
-                            bool: {
-                                must: [
-                                    {term: {"act.account": "eosio"}},
-                                    {term: {"act.name": "newaccount"}},
-                                    {term: {"act.authorization.actor": controlling_account}}
-                                ]
-                            }
-                        }
-                    ],
-                    minimum_should_match: 1
-                }
-            },
-            sort: [
-                {"global_sequence": {"order": "desc"}}
-            ]
-        }
-    });
+	if (typeof request.body === 'string') {
+		request.body = JSON.parse(request.body)
+	}
+	let controlling_account = request.body["controlling_account"];
+	const results: ApiResponse = await fastify.elastic.search({
+		index: fastify.manager.chain + '-action-*',
+		size: 100,
+		body: {
+			query: {
+				bool: {
+					should: [
+						{
+							term: {"@updateauth.auth.accounts.permission.actor": controlling_account}
+						},
+						{
+							bool: {
+								must: [
+									{term: {"act.account": "eosio"}},
+									{term: {"act.name": "newaccount"}},
+									{term: {"act.authorization.actor": controlling_account}}
+								]
+							}
+						}
+					],
+					minimum_should_match: 1
+				}
+			},
+			sort: [
+				{"global_sequence": {"order": "desc"}}
+			]
+		}
+	});
 
-    const response = {
-        controlled_accounts: []
-    };
+	const response = {
+		controlled_accounts: []
+	};
 
-    const hits = results.body.hits.hits;
+	const hits = results.body.hits.hits;
 
-    if (hits.length > 0) {
-        response.controlled_accounts = hits.map((v) => {
-            if (v._source.act.name === 'newaccount') {
-                if (v._source['@newaccount'].newact) {
-                    return v._source['@newaccount'].newact;
-                } else if (v._source.act.data.newact) {
-                    return v._source.act.data.newact;
-                } else {
-                    return null;
-                }
-            } else if (v._source.act.name === 'updateauth') {
-                return v._source.act.data.account;
-            } else {
-                return null;
-            }
-        });
-    }
-    if (response.controlled_accounts.length > 0) {
-        response.controlled_accounts = [...(new Set(response.controlled_accounts))];
-    }
-    return response;
+	if (hits.length > 0) {
+		response.controlled_accounts = hits.map((v) => {
+			if (v._source.act.name === 'newaccount') {
+				if (v._source['@newaccount'].newact) {
+					return v._source['@newaccount'].newact;
+				} else if (v._source.act.data.newact) {
+					return v._source.act.data.newact;
+				} else {
+					return null;
+				}
+			} else if (v._source.act.name === 'updateauth') {
+				return v._source.act.data.account;
+			} else {
+				return null;
+			}
+		});
+	}
+	if (response.controlled_accounts.length > 0) {
+		response.controlled_accounts = [...(new Set(response.controlled_accounts))];
+	}
+	return response;
 }
 
 export function getControlledAccountsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getControlledAccounts, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getControlledAccounts, fastify, request, route));
+	}
 }

api/routes/v1-history/get_transaction/get_transaction.ts

@@ -1,195 +1,190 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
-import {createHash} from "crypto";
-import * as flatstr from 'flatstr';
+import {GetInfoResult} from "eosjs/dist/eosjs-rpc-interfaces";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
-	if (typeof request.body === 'string') {
-		request.body = JSON.parse(request.body)
-	}
-	const pResults = await Promise.all([fastify.eosjs.rpc.get_info(), fastify.elastic['search']({
-		"index": fastify.manager.chain + '-action-*',
-		"body": {
-			"query": {
-				"bool": {
-					must: [
-						{term: {"trx_id": request.body.id.toLowerCase()}}
-					]
-				}
-			},
-			"sort": {
-				"global_sequence": "asc"
-			}
-		}
-	})]);
-	const results = pResults[1];
-	const response: any = {
-		"id": request.body.id,
-		"trx": {
-			"receipt": {
-				"status": "executed",
-				"cpu_usage_us": 0,
-				"net_usage_words": 0,
-				"trx": [1, {
-					"signatures": "",
-					"compression": "none",
-					"packed_context_free_data": "",
-					"packed_trx": ""
-				}]
-			},
-			"trx": {}
-		},
-		"block_num": 0,
-		"block_time": "",
-		"last_irreversible_block": pResults[0].last_irreversible_block_num,
-		"traces": []
-	};
 
-	const hits = results['body']['hits']['hits'];
+    if (request.method === 'POST' && !request.body) {
+        throw new Error("missing POST body");
+    }
 
-	if (hits.length > 0) {
-		const actions = hits;
-		response.trx.trx = {
-			"expiration": "",
-			"ref_block_num": 0,
-			"ref_block_prefix": 0,
-			"max_net_usage_words": 0,
-			"max_cpu_usage_ms": 0,
-			"delay_sec": 0,
-			"context_free_actions": [],
-			"actions": [],
-			"transaction_extensions": [],
-			"signatures": [],
-			"context_free_data": []
-		};
-		let traces = {};
-		let seqNum = 0;
-		for (let action of actions) {
-			action = action._source;
-			mergeActionMeta(action);
-			action.act['hex_data'] = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
-			if (action.parent === 0) {
-				response.trx.trx.actions.push(action.act);
-			}
-			response.block_num = action.block_num;
-			response.block_time = action['@timestamp'];
-			seqNum += 1;
-			let trace = {
-				receipt: {
-					receiver: action.act.account,
-					global_sequence: String(action.global_sequence),
-					auth_sequence: [
-						action.act.authorization[0].actor,
-						seqNum
-					],
-					act_digest: '',
-					recv_sequence: seqNum,
-					code_sequence: 1,
-					abi_sequence: 1
-				},
-				act: action.act,
-				account_ram_deltas: action.account_ram_deltas || [],
-				context_free: false,
-				block_num: action.block_num,
-				block_time: action['@timestamp'],
-				console: "",
-				elapsed: 0,
-				except: null,
-				inline_traces: [],
-				producer_block_id: "",
-				trx_id: request.body.id,
-				notified: action.notified
-			};
-			let hash = createHash('sha256');
-			hash.update(JSON.stringify(action.act));
-			trace.receipt.act_digest = hash.digest('hex');
-			traces[action.global_sequence] = trace;
-		}
+    if (typeof request.body === 'string') {
+        request.body = JSON.parse(request.body)
+    }
+    const body: any = request.body;
+    const redis = fastify.redis;
+    const trxId = body.id.toLowerCase();
 
-		actions.forEach(action => {
-			action = action._source;
-			for (let i = 0; i < traces[action.global_sequence].notified.length; i++) {
-				if (traces[action.global_sequence].notified[i] === action.act.account) {
-					traces[action.global_sequence].notified.splice(i, 1);
-					break;
-				}
-			}
-			if (action.parent !== 0 && action.parent) {
-				if (traces[action.parent]) {
-					for (let i = 0; i < traces[action.parent].notified.length; i++) {
-						if (traces[action.parent].notified[i] === action.act.account) {
-							traces[action.parent].notified.splice(i, 1);
-							break;
-						}
-					}
-					traces[action.parent].inline_traces.push(traces[action.global_sequence]);
-				}
-			}
-		});
+    const conf = fastify.manager.config;
+    const cachedData = await redis.hgetall('trx_' + trxId);
 
-		actions.forEach(action => {
-			action = action._source;
-			response.traces.push(traces[action.global_sequence]);
-			if (traces[action.global_sequence] && traces[action.global_sequence].notified) {
-				traces[action.global_sequence].notified.forEach((note, index) => {
-					seqNum += 1;
-					let trace = {
-						receipt: {
-							receiver: note,
-							global_sequence: String(action.global_sequence + index + 1),
-							auth_sequence: [
-								action.act.authorization[0].actor,
-								seqNum
-							],
-							act_digest: traces[action.global_sequence].receipt.act_digest,
-							recv_sequence: seqNum,
-							code_sequence: 1,
-							abi_sequence: 1
-						},
-						account_ram_deltas: action.account_ram_deltas || [],
-						act: action.act,
-						block_num: action.block_num,
-						block_time: action['@timestamp'],
-						console: "",
-						context_free: false,
-						elapsed: 0,
-						except: null,
-						inline_traces: [],
-						producer_block_id: "",
-						trx_id: request.body.id,
-					};
-					traces[action.global_sequence].inline_traces.unshift(trace);
-					response.traces.push(trace);
-				});
-				delete traces[action.global_sequence].notified;
-			}
-		});
-	} else {
-		const errmsg = "Transaction " + request.body.id.toLowerCase() + " not found in history and no block hint was given";
-		return {
-			code: 500,
-			message: "Internal Service Error",
-			error: {
-				code: 3040011,
-				name: "tx_not_found",
-				what: "The transaction can not be found",
-				details: [
-					{
-						message: errmsg,
-						file: "",
-						line_number: 1,
-						method: "get_transaction"
-					}
-				]
-			}
-		}
-	}
-	return response;
+    const response: any = {
+        "id": body.id,
+        "trx": {
+            "receipt": {
+                "status": "executed",
+                "cpu_usage_us": 0,
+                "net_usage_words": 0,
+                "trx": [1, {
+                    "signatures": "",
+                    "compression": "none",
+                    "packed_context_free_data": "",
+                    "packed_trx": ""
+                }]
+            },
+            "trx": {}
+        },
+        "block_num": 0,
+        "block_time": "",
+        "last_irreversible_block": undefined,
+        "traces": []
+    };
+
+    let hits;
+
+    // build get_info request with caching
+    const $getInfo = new Promise<GetInfoResult>(resolve => {
+        const key = `${fastify.manager.chain}_get_info`;
+        fastify.redis.get(key).then(value => {
+            if (value) {
+                resolve(JSON.parse(value));
+            } else {
+                fastify.eosjs.rpc.get_info().then(value1 => {
+                    fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
+                    resolve(value1);
+                }).catch((reason) => {
+                    console.log(reason);
+                    response.error = 'failed to get last_irreversible_block_num'
+                    resolve(null);
+                });
+            }
+        });
+    });
+
+    // reconstruct hits from cached data
+    if (cachedData && Object.keys(cachedData).length > 0) {
+        const gsArr = [];
+        for (let cachedDataKey in cachedData) {
+            gsArr.push(cachedData[cachedDataKey]);
+        }
+        gsArr.sort((a, b) => {
+            return a.global_sequence - b.global_sequence;
+        });
+        hits = gsArr.map(value => {
+            return {
+                _source: JSON.parse(value)
+            };
+        });
+        const promiseResults = await Promise.all([
+            redis.ttl('trx_' + trxId),
+            $getInfo
+        ]);
+        response.cache_expires_in = promiseResults[0];
+        response.last_irreversible_block = promiseResults[1].last_irreversible_block_num;
+    }
+
+    // search on ES if cache is not present
+    if (!hits) {
+        const _size = conf.api.limits.get_trx_actions || 100;
+        const blockHint = parseInt(body.block_num_hint, 10);
+        let indexPattern = '';
+        if (blockHint) {
+            const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
+            indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
+        } else {
+            indexPattern = fastify.manager.chain + '-action-*';
+        }
+        let pResults;
+        try {
+
+            // build search request
+            const $search = fastify.elastic.search({
+                index: indexPattern,
+                size: _size,
+                body: {
+                    query: {bool: {must: [{term: {trx_id: trxId}}]}},
+                    sort: {global_sequence: "asc"}
+                }
+            });
+
+            // execute in parallel
+            pResults = await Promise.all([$getInfo, $search]);
+        } catch (e) {
+            console.log(e.message);
+            if (e.meta.statusCode === 404) {
+                return response;
+            }
+        }
+        hits = pResults[1]['body']['hits']['hits'];
+        response.last_irreversible_block = pResults[0].last_irreversible_block_num;
+    }
+
+
+    if (hits.length > 0) {
+        const actions = hits;
+        response.trx.trx = {
+            "expiration": "",
+            "ref_block_num": 0,
+            "ref_block_prefix": 0,
+            "max_net_usage_words": 0,
+            "max_cpu_usage_ms": 0,
+            "delay_sec": 0,
+            "context_free_actions": [],
+            "actions": [],
+            "transaction_extensions": [],
+            "signatures": [],
+            "context_free_data": []
+        };
+        for (let action of actions) {
+            action = action._source;
+            mergeActionMeta(action);
+            response.block_num = action.block_num;
+            response.block_time = action['@timestamp'];
+            for (const receipt of action.receipts) {
+                if (action.act_digest) {
+                    receipt.act_digest = action.act_digest.toLowerCase();
+                }
+                response.traces.push({
+                    receipt: receipt,
+                    act: action.act,
+                    account_ram_deltas: action.account_ram_deltas || [],
+                    context_free: false,
+                    block_num: action.block_num,
+                    block_time: action['@timestamp'],
+                    console: "",
+                    elapsed: 0,
+                    except: null,
+                    inline_traces: [],
+                    producer_block_id: "",
+                    trx_id: body.id
+                });
+            }
+        }
+    } else {
+        const errmsg = "Transaction " + body.id.toLowerCase() + " not found in history and no block hint was given";
+        return {
+            code: 500,
+            message: "Internal Service Error",
+            error: {
+                code: 3040011,
+                name: "tx_not_found",
+                what: "The transaction can not be found",
+                details: [
+                    {
+                        message: errmsg,
+                        file: "",
+                        line_number: 1,
+                        method: "get_transaction"
+                    }
+                ]
+            }
+        }
+    }
+    return response;
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-		reply.send(await timedQuery(getTransaction, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getTransaction, fastify, request, route));
+    }
 }

api/routes/v1-history/get_transaction/index.ts

@@ -9,7 +9,10 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         tags: ['history'],
         body: {
             type: ['object', 'string'],
-            properties: {id: {description: 'transaction id', type: 'string'}},
+            properties: {
+                id: {description: 'transaction id', type: 'string'},
+                block_num_hint: {description: 'block number hint', type: 'integer'},
+            },
             required: ["id"]
         }
     });

api/routes/v1-trace/get_block/get_block.ts

@@ -1,5 +1,4 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import * as _ from "lodash";
 
@@ -19,7 +18,7 @@ async function getBlockTrace(fastify: FastifyInstance, request: FastifyRequest)
         request.body = JSON.parse(request.body);
     }
 
-    const reqBody = request.body;
+    const reqBody: any = request.body;
     const targetBlock = parseInt(reqBody.block_num);
     let searchBody;
 
@@ -121,7 +120,7 @@ async function getBlockTrace(fastify: FastifyInstance, request: FastifyRequest)
 }
 
 export function getBlockTraceHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getBlockTrace, fastify, request, route));
     }
 }

api/routes/v2-history/check_transaction/check_transaction.ts

@@ -0,0 +1,33 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {timedQuery} from "../../../helpers/functions";
+
+async function checkTransaction(fastify: FastifyInstance, request: FastifyRequest) {
+	const query: any = request.query;
+	const txId = query.id.toLowerCase();
+	const data = await fastify.redis.get(txId) as any;
+	if (data) {
+		const jsonObj = JSON.parse(data);
+		const response = {
+			id: txId,
+			status: jsonObj.status,
+			block_num: jsonObj.b,
+			root_action: jsonObj.a,
+			signatures: [],
+		};
+		if (jsonObj.s && jsonObj.s.length > 0) {
+			response.signatures = jsonObj.s;
+		}
+		return response;
+	} else {
+		return {
+			id: txId,
+			status: 'unknown'
+		};
+	}
+}
+
+export function checkTransactionHandler(fastify: FastifyInstance, route: string) {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(checkTransaction, fastify, request, route));
+	}
+}

api/routes/v2-history/check_transaction/index.ts

@@ -0,0 +1,55 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {checkTransactionHandler} from "./check_transaction";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+	const schema = {
+		description: 'check if a transaction was included in a block',
+		summary: 'check if a transaction was included in a block',
+		tags: ['history'],
+		querystring: {
+			type: 'object',
+			properties: {
+				"id": {
+					description: 'transaction id',
+					type: 'string',
+					minLength: 64,
+					maxLength: 64
+				}
+			},
+			required: ["id"]
+		},
+		response: extendResponseSchema({
+			"id": {type: "string"},
+			"status": {type: "string"},
+			"block_num": {type: "number"},
+			"root_action": {
+				type: "object",
+				properties: {
+					"account": {type: "string"},
+					"name": {type: "string"},
+					"authorization": {
+						type: "array",
+						items: {
+							type: 'object',
+							properties: {
+								"actor": {type: "string"},
+								"permission": {type: "string"}
+							}
+						}
+					},
+					"data": {type: "string"}
+				}
+			},
+			"signatures": {type: "array", items: {type: 'string'}}
+		})
+	};
+	addApiRoute(
+		fastify,
+		'GET',
+		getRouteName(__filename),
+		checkTransactionHandler,
+		schema
+	);
+	next();
+}

api/routes/v2-history/export_actions/export_actions.ts

@@ -0,0 +1,12 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {timedQuery} from "../../../helpers/functions";
+
+async function exportActions(fastify: FastifyInstance, request: FastifyRequest) {
+
+}
+
+export function exportActionsHandler(fastify: FastifyInstance, route: string) {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(exportActions, fastify, request, route));
+	}
+}

api/routes/v2-history/export_actions/index.ts

@@ -0,0 +1,19 @@
+import {FastifyInstance, FastifySchema} from "fastify";
+import {addApiRoute, getRouteName} from "../../../helpers/functions";
+import {getCreatorHandler} from "../get_creator/get_creator";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+	const schema: FastifySchema = {
+		description: 'request large action data export',
+		summary: 'request large action data export',
+		tags: ['history']
+	};
+	addApiRoute(
+		fastify,
+		'GET',
+		getRouteName(__filename),
+		getCreatorHandler,
+		schema
+	);
+	next();
+}

api/routes/v2-history/get_abi_snapshot/get_abi_snapshot.ts

@@ -1,5 +1,4 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 
 async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest) {
@@ -8,9 +7,11 @@ async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest)
         block_num: null
     };
 
-    const code = request.query.contract;
-    const block = request.query.block;
-    const should_fetch = request.query.fetch;
+    const query: any = request.query;
+
+    const code = query.contract;
+    const block = query.block;
+    const should_fetch = query.fetch;
 
     const mustArray = [];
 
@@ -21,7 +22,7 @@ async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest)
     }
 
     const results = await fastify.elastic.search({
-        index: fastify.manager.chain + '-abi',
+        index: fastify.manager.chain + '-abi-*',
         size: 1,
         body: {
             query: {bool: {must: mustArray}},
@@ -44,7 +45,7 @@ async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest)
 }
 
 export function getAbiSnapshotHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getAbiSnapshot, fastify, request, route));
     }
 }

api/routes/v2-history/get_actions/definitions.ts

@@ -1,5 +1,6 @@
 export const terms = [
     "notified",
+    "receipts.receiver",
     "act.authorization.actor"
 ];
 
@@ -11,12 +12,19 @@ export const extendedActions = new Set([
     "buyrambytes",
     "delegatebw",
     "undelegatebw",
-    "voteproducer"
+    "voteproducer",
+    'statement',
+    'decision',
+    'batch',
+    'act',
+    'exec',
+    'votefor',
+    'voteagainst'
 ]);
 
 export const primaryTerms = [
-    "notified",
     "block_num",
+    "block_id",
     "global_sequence",
     "producer",
     "@timestamp",
@@ -24,5 +32,10 @@ export const primaryTerms = [
     "action_ordinal",
     "cpu_usage_us",
     "net_usage_words",
-    "trx_id"
+    "trx_id",
+    "receipts.receiver",
+    "receipts.global_sequence",
+    "receipts.recv_sequence",
+    "receipts.auth_sequence.account",
+    "receipts.auth_sequence.sequence"
 ];

api/routes/v2-history/get_actions/functions.ts

@@ -1,4 +1,4 @@
-import {extendedActions, primaryTerms, terms} from "./definitions";
+import {primaryTerms, terms} from "./definitions";
 
 export function addSortedBy(query, queryBody, sort_direction) {
     if (query['sortedBy']) {
@@ -70,68 +70,119 @@ function addRangeQuery(queryStruct, prop, pkey, query) {
 
 export function applyTimeFilter(query, queryStruct) {
     if (query['after'] || query['before']) {
-        let _lte = "now";
-        let _gte = "0";
-        if (query['before']) {
-            _lte = query['before'];
-            if (!_lte.endsWith("Z")) {
-                _lte += "Z";
-            }
+
+        if (query['after']?.includes(' ')) {
+            query['after'] = query['after'].replace(' ', 'T');
         }
-        if (query['after']) {
-            _gte = query['after'];
-            if (!_gte.endsWith("Z")) {
-                _gte += "Z";
-            }
+
+        if (query['before']?.includes(' ')) {
+            query['before'] = query['before'].replace(' ', 'T');
         }
-        if (!queryStruct.bool['filter']) {
-            queryStruct.bool['filter'] = [];
-        }
-        queryStruct.bool['filter'].push({
-            range: {
-                "@timestamp": {
-                    "gte": _gte,
-                    "lte": _lte
+
+        if (query['after']?.includes('T') || query['before']?.includes('T')) {
+            let _lte = "now";
+            let _gte = "0";
+            if (query['before']) {
+                try {
+                    _lte = new Date(query['before']).toISOString();
+                } catch (e) {
+                    throw new Error(e.message + ' [before]');
                 }
             }
-        });
+            if (query['after']) {
+                try {
+                    _gte = new Date(query['after']).toISOString();
+                } catch (e) {
+                    throw new Error(e.message + ' [after]');
+                }
+            }
+            if (!queryStruct.bool['filter']) {
+                queryStruct.bool['filter'] = [];
+            }
+            queryStruct.bool['filter'].push({
+                range: {
+                    "@timestamp": {
+                        "gte": _gte,
+                        "lte": _lte
+                    }
+                }
+            });
+        } else {
+            // search by block number
+            const rangeObj = {
+                range: {
+                    block_num: {}
+                }
+            };
+            if (parseInt(query['after']) > 0) {
+                rangeObj.range.block_num['gte'] = query['after'];
+            }
+            if (parseInt(query['before']) > 0) {
+                rangeObj.range.block_num['lte'] = query['before'];
+            }
+            if (Object.keys(rangeObj.range.block_num).length > 0) {
+                if (!queryStruct.bool['filter']) {
+                    queryStruct.bool['filter'] = [];
+                }
+                queryStruct.bool['filter'].push(rangeObj);
+            }
+        }
     }
 }
 
-export function applyGenericFilters(query, queryStruct) {
+export function applyGenericFilters(query, queryStruct, allowedExtraParams: Set<string>) {
     for (const prop in query) {
         if (Object.prototype.hasOwnProperty.call(query, prop)) {
             const pair = prop.split(".");
             if (pair.length > 1 || primaryTerms.includes(pair[0])) {
                 let pkey;
-                if (pair.length > 1) {
-                    pkey = extendedActions.has(pair[0]) ? "@" + prop : prop;
+                if (pair.length > 1 && allowedExtraParams) {
+                    pkey = allowedExtraParams.has(pair[0]) ? "@" + prop : prop;
                 } else {
                     pkey = prop;
                 }
                 if (query[prop].indexOf("-") !== -1) {
                     addRangeQuery(queryStruct, prop, pkey, query);
                 } else {
-                    const _termQuery = {};
+                    const _qObj = {};
                     const parts = query[prop].split(",");
                     if (parts.length > 1) {
                         processMultiVars(queryStruct, parts, prop);
                     } else if (parts.length === 1) {
-                        const andParts = parts[0].split(" ");
-                        if (andParts.length > 1) {
-                            andParts.forEach(value => {
-                                const _q = {};
-                                console.log(value);
-                                _q[pkey] = value;
-                                queryStruct.bool.must.push({term: _q});
+
+                        // @transfer.memo special case
+                        if (pkey === '@transfer.memo') {
+                            _qObj[pkey] = {
+                                query: parts[0]
+                            };
+
+                            if (query.match_fuzziness) {
+                                _qObj[pkey].fuzziness = query.match_fuzziness;
+                            }
+
+                            if (query.match_operator) {
+                                _qObj[pkey].operator = query.match_operator;
+                            }
+
+                            queryStruct.bool.must.push({
+                                match: _qObj
                             });
                         } else {
-                            if (parts[0].startsWith("!")) {
-                                _termQuery[pkey] = parts[0].replace("!", "");
-                                queryStruct.bool.must_not.push({term: _termQuery});
+                            const andParts = parts[0].split(" ");
+                            if (andParts.length > 1) {
+                                andParts.forEach(value => {
+                                    const _q = {};
+                                    _q[pkey] = value;
+                                    queryStruct.bool.must.push({term: _q});
+                                });
                             } else {
-                                _termQuery[pkey] = parts[0];
-                                queryStruct.bool.must.push({term: _termQuery});
+                                if (parts[0].startsWith("!")) {
+                                    _qObj[pkey] = parts[0].replace("!", "");
+                                    queryStruct.bool.must_not.push({term: _qObj});
+                                } else {
+                                    _qObj[pkey] = parts[0];
+                                    queryStruct.bool.must.push({term: _qObj});
+                                }
                             }
                         }
                     }

api/routes/v2-history/get_actions/get_actions.ts

@@ -1,5 +1,4 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {getTrackTotalHits, mergeActionMeta, timedQuery} from "../../../helpers/functions";
 import {
     addSortedBy,
@@ -12,7 +11,7 @@ import {
 } from "./functions";
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
-    const query = request.query;
+    const query: any = request.query;
     const maxActions = fastify.manager.config.api.limits.get_actions;
     const queryStruct = {
         "bool": {
@@ -23,11 +22,17 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
     };
 
     const {skip, limit} = getSkipLimit(query, maxActions);
+
     const sort_direction = getSortDir(query);
+
     applyAccountFilters(query, queryStruct);
-    applyGenericFilters(query, queryStruct);
+
+    applyGenericFilters(query, queryStruct, fastify.allowedActionQueryParamSet);
+
     applyTimeFilter(query, queryStruct);
+
     applyCodeActionFilters(query, queryStruct);
+
     // allow precise counting of total hits
     const trackTotalHits = getTrackTotalHits(query);
 
@@ -47,12 +52,15 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
         indexPattern = fastify.manager.chain + '-action';
     }
 
-    const esResults = await fastify.elastic.search({
+    const esOpts = {
         "index": indexPattern,
         "from": skip || 0,
         "size": (limit > maxActions ? maxActions : limit) || 10,
         "body": query_body
-    });
+    };
+
+    // console.log(JSON.stringify(esOpts, null, 2));
+    const esResults = await fastify.elastic.search(esOpts);
 
     const results = esResults['body']['hits'];
     const response: any = {
@@ -77,8 +85,18 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
 
     if (results['hits'].length > 0) {
         const actions = results['hits'];
-        for (let action of actions) {
-            action = action._source;
+        for (let action of actions.map(a => a._source)) {
+
+            try {
+                if (action.act.data) {
+                    if (action.act.data.account && action.act.data.name && action.act.data.authorization) {
+                        action.act.data = action.act.data.data;
+                    }
+                }
+            } catch (e: any) {
+                console.log(e);
+            }
+
             mergeActionMeta(action);
 
             if (query.noBinary === true) {
@@ -92,13 +110,14 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
 
             if (query.simple) {
+                let notified = new Set(action.receipts.map(r => r.receiver));
                 response.simple_actions.push({
                     block: action['block_num'],
                     irreversible: response.lib !== 0 ? action['block_num'] < response.lib : undefined,
                     timestamp: action['@timestamp'],
                     transaction_id: action['trx_id'],
                     actors: action['act']['authorization'].map(a => `${a.actor}@${a.permission}`).join(","),
-                    notified: action['notified'].join(','),
+                    notified: [...notified].join(','),
                     contract: action['act']['account'],
                     action: action['act']['name'],
                     data: action['act']['data']
@@ -113,7 +132,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
 }
 
 export function getActionsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getActions, fastify, request, route));
     }
 }

api/routes/v2-history/get_actions/index.ts

@@ -1,4 +1,4 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {getActionsHandler} from "./get_actions";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
@@ -6,17 +6,46 @@ export const getActionResponseSchema = {
     "@timestamp": {type: "string"},
     "timestamp": {type: "string"},
     "block_num": {type: "number"},
+    "block_id": {type: "string"},
     "trx_id": {type: "string"},
     "act": {
         type: 'object',
         properties: {
             "account": {type: "string"},
-            "name": {type: "string"}
+            "name": {type: "string"},
+            "authorization": {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        "actor": {type: "string"},
+                        "permission": {type: "string"},
+                    }
+                }
+            }
         },
         additionalProperties: true
     },
-    "notified": {
-        type: "array", items: {type: "string"}
+    "receipts": {
+        type: "array",
+        items: {
+            type: "object",
+            properties: {
+                "receiver": {type: "string"},
+                "global_sequence": {type: "number"},
+                "recv_sequence": {type: "number"},
+                "auth_sequence": {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            "account": {type: "string"},
+                            "sequence": {type: "number"},
+                        }
+                    }
+                }
+            }
+        }
     },
     "cpu_usage_us": {type: "number"},
     "net_usage_words": {type: "number"},
@@ -32,15 +61,15 @@ export const getActionResponseSchema = {
         }
     },
     "global_sequence": {type: "number"},
-    "receiver": {type: 'string'},
     "producer": {type: "string"},
     "parent": {type: "number"},
     "action_ordinal": {type: 'number'},
-    "creator_action_ordinal": {type: 'number'}
+    "creator_action_ordinal": {type: 'number'},
+    "signatures": {type: "array", items: {type: 'string'}}
 };
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
+    const schema: FastifySchema = {
         description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
             ' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
         summary: 'get root actions',

api/routes/v2-history/get_created_accounts/get_created_accounts.ts

@@ -1,58 +1,58 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {getSkipLimit} from "../get_actions/functions";
 
 async function getCreatedAccounts(fastify: FastifyInstance, request: FastifyRequest) {
 
 
-    const {skip, limit} = getSkipLimit(request.query);
-    const maxActions = fastify.manager.config.api.limits.get_created_accounts;
-    const results = await fastify.elastic.search({
-        "index": fastify.manager.chain + '-action-*',
-        "from": skip || 0,
-        "size": (limit > maxActions ? maxActions : limit) || 100,
-        "body": {
-            "query": {
-                "bool": {
-                    must: [
-                        {term: {"act.authorization.actor": request.query.account.toLowerCase()}},
-                        {term: {"act.name": "newaccount"}},
-                        {term: {"act.account": "eosio"}}
-                    ]
-                }
-            },
-            sort: {
-                "global_sequence": "desc"
-            }
-        }
-    });
+	const query: any = request.query;
+	const {skip, limit} = getSkipLimit(query);
+	const maxActions = fastify.manager.config.api.limits.get_created_accounts;
+	const results = await fastify.elastic.search({
+		"index": fastify.manager.chain + '-action-*',
+		"from": skip || 0,
+		"size": (limit > maxActions ? maxActions : limit) || 100,
+		"body": {
+			"query": {
+				"bool": {
+					must: [
+						{term: {"act.authorization.actor": query.account.toLowerCase()}},
+						{term: {"act.name": "newaccount"}},
+						{term: {"act.account": "eosio"}}
+					]
+				}
+			},
+			sort: {
+				"global_sequence": "desc"
+			}
+		}
+	});
 
-    const response = {accounts: []};
+	const response = {accounts: []};
 
-    if (results['body']['hits']['hits'].length > 0) {
-        const actions = results['body']['hits']['hits'];
-        for (let action of actions) {
-            action = action._source;
-            const _tmp = {};
-            if (action['act']['data']['newact']) {
-                _tmp['name'] = action['act']['data']['newact'];
-            } else if (action['@newaccount']['newact']) {
-                _tmp['name'] = action['@newaccount']['newact'];
-            } else {
-                console.log(action);
-            }
-            _tmp['trx_id'] = action['trx_id'];
-            _tmp['timestamp'] = action['@timestamp'];
-            response.accounts.push(_tmp);
-        }
-    }
+	if (results['body']['hits']['hits'].length > 0) {
+		const actions = results['body']['hits']['hits'];
+		for (let action of actions) {
+			action = action._source;
+			const _tmp = {};
+			if (action['act']['data']['newact']) {
+				_tmp['name'] = action['act']['data']['newact'];
+			} else if (action['@newaccount']['newact']) {
+				_tmp['name'] = action['@newaccount']['newact'];
+			} else {
+				console.log(action);
+			}
+			_tmp['trx_id'] = action['trx_id'];
+			_tmp['timestamp'] = action['@timestamp'];
+			response.accounts.push(_tmp);
+		}
+	}
 
-    return response;
+	return response;
 }
 
 export function getCreatedAccountsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getCreatedAccounts, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getCreatedAccounts, fastify, request, route));
+	}
 }

api/routes/v2-history/get_created_accounts/index.ts

@@ -1,44 +1,44 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {getCreatedAccountsHandler} from "./get_created_accounts";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
-        description: 'get all accounts created by one creator',
-        summary: 'get created accounts',
-        tags: ['accounts'],
-        querystring: extendQueryStringSchema({
-            "account": {
-                description: 'creator account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            }
-        }, ["account"]),
-        response: extendResponseSchema({
-            "query_time": {
-                type: "number"
-            },
-            "total": {
-                type: "object",
-                properties: {
-                    "value": {type: "number"},
-                    "relation": {type: "string"}
-                }
-            },
-            "accounts": {
-                type: "array",
-                items: {
-                    type: 'object',
-                    properties: {
-                        'name': {type: 'string'},
-                        'timestamp': {type: 'string'},
-                        'trx_id': {type: 'string'}
-                    }
-                }
-            }
-        })
-    };
-    addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatedAccountsHandler, schema);
-    next();
+	const schema: FastifySchema = {
+		description: 'get all accounts created by one creator',
+		summary: 'get created accounts',
+		tags: ['accounts'],
+		querystring: extendQueryStringSchema({
+			"account": {
+				description: 'creator account',
+				type: 'string',
+				minLength: 1,
+				maxLength: 12
+			}
+		}, ["account"]),
+		response: extendResponseSchema({
+			"query_time": {
+				type: "number"
+			},
+			"total": {
+				type: "object",
+				properties: {
+					"value": {type: "number"},
+					"relation": {type: "string"}
+				}
+			},
+			"accounts": {
+				type: "array",
+				items: {
+					type: 'object',
+					properties: {
+						'name': {type: 'string'},
+						'timestamp': {type: 'string'},
+						'trx_id': {type: 'string'}
+					}
+				}
+			}
+		})
+	};
+	addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatedAccountsHandler, schema);
+	next();
 }

api/routes/v2-history/get_creator/get_creator.ts

@@ -1,99 +1,106 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 
 async function getCreator(fastify: FastifyInstance, request: FastifyRequest) {
 
-	const response = {
-		account: request.query.account,
-		creator: '',
-		timestamp: '',
-		block_num: 0,
-		trx_id: '',
-	};
+    const query: any = request.query;
 
-	if (request.query.account === fastify.manager.config.settings.eosio_alias) {
-		const genesisBlock = await fastify.eosjs.rpc.get_block(1);
-		if (genesisBlock) {
-			response.creator = '__self__';
-			response.timestamp = genesisBlock.timestamp;
-			response.block_num = 1;
-			response.trx_id = "";
-			return response;
-		} else {
-			throw new Error("genesis block not found");
-		}
-	}
+    const response = {
+        account: query.account,
+        creator: '',
+        timestamp: '',
+        block_num: 0,
+        trx_id: '',
+    };
 
-	const results = await fastify.elastic.search({
-		"index": fastify.manager.chain + '-action-*',
-		"body": {
-			size: 1,
-			query: {
-				bool: {
-					must: [{term: {"@newaccount.newact": request.query.account}}]
-				}
-			}
-		}
-	});
+    if (query.account === fastify.manager.config.settings.eosio_alias) {
+        try {
+            const genesisBlock = await fastify.eosjs.rpc.get_block(1);
+            if (genesisBlock) {
+                response.timestamp = genesisBlock.timestamp;
+            }
+        } catch (e) {
+            console.log(e.message);
+        }
+        response.creator = '__self__';
+        response.block_num = 1;
+        response.trx_id = "";
+        return response;
+    }
 
-	if (results['body']['hits']['hits'].length === 1) {
-		const result = results['body']['hits']['hits'][0]._source;
-		response.trx_id = result.trx_id;
-		response.block_num = result.block_num;
-		response.creator = result.act.data.creator;
-		response.timestamp = result['@timestamp'];
-		return response;
-	} else {
-		let accountInfo;
-		try {
-			accountInfo = await fastify.eosjs.rpc.get_account(request.query.account);
-		} catch (e) {
-			throw new Error("account not found");
-		}
-		if (accountInfo) {
-			try {
-				response.timestamp = accountInfo.created;
-				const blockHeader = await fastify.elastic.search({
-					"index": fastify.manager.chain + '-block-*',
-					"body": {
-						size: 1,
-						query: {
-							bool: {
-								must: [{term: {"@timestamp": response.timestamp}}]
-							}
-						}
-					}
-				});
-				const blockId = blockHeader['body']['hits']['hits'][0]._source.block_id;
-				const blockData = await fastify.eosjs.rpc.get_block(blockId);
-				response.block_num = blockData.block_num;
-				for (const transaction of blockData["transactions"]) {
-					if (typeof transaction.trx !== 'string') {
-						const actions = transaction.trx.transaction.actions;
-						for (const act of actions) {
-							if (act.name === 'newaccount') {
-								if (act.data.name === request.query.account) {
-									response.creator = act.data.creator;
-									response.trx_id = transaction.id;
-									return response;
-								}
-							}
-						}
-					}
-				}
-				return response;
-			} catch (e) {
-				throw new Error("account creation not found");
-			}
-		} else {
-			throw new Error("account creation not found");
-		}
-	}
+    const results = await fastify.elastic.search({
+        "index": fastify.manager.chain + '-action-*',
+        "body": {
+            size: 1,
+            query: {
+                bool: {
+                    must: [{term: {"@newaccount.newact": query.account}}]
+                }
+            }
+        }
+    });
+
+    if (results['body']['hits']['hits'].length === 1) {
+        const result = results['body']['hits']['hits'][0]._source;
+        response.trx_id = result.trx_id;
+        response.block_num = result.block_num;
+        response.creator = result.act.data.creator;
+        response.timestamp = result['@timestamp'];
+        return response;
+    } else {
+        let accountInfo;
+        try {
+            accountInfo = await fastify.eosjs.rpc.get_account(query.account);
+        } catch (e) {
+            throw new Error("account not found");
+        }
+        if (accountInfo) {
+            try {
+                response.timestamp = accountInfo.created;
+                const blockHeader = await fastify.elastic.search({
+                    "index": fastify.manager.chain + '-block-*',
+                    "body": {
+                        size: 1,
+                        query: {
+                            bool: {
+                                must: [{term: {"@timestamp": response.timestamp}}]
+                            }
+                        }
+                    }
+                });
+                const hits = blockHeader['body']['hits']['hits'];
+                if (hits.length > 0 && hits[0]._source) {
+                    const blockId = blockHeader['body']['hits']['hits'][0]._source.block_id;
+                    const blockData = await fastify.eosjs.rpc.get_block(blockId);
+                    response.block_num = blockData.block_num;
+                    for (const transaction of blockData["transactions"]) {
+                        if (typeof transaction.trx !== 'string') {
+                            const actions = transaction.trx.transaction.actions;
+                            for (const act of actions) {
+                                if (act.name === 'newaccount') {
+                                    if (act.data.name === query.account) {
+                                        response.creator = act.data.creator;
+                                        response.trx_id = transaction.id;
+                                        return response;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                return response;
+            } catch (e) {
+                console.log(e.message);
+                throw new Error("account creation not found");
+            }
+        } else {
+            throw new Error("account not found");
+        }
+    }
 }
 
 export function getCreatorHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-		reply.send(await timedQuery(getCreator, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getCreator, fastify, request, route));
+    }
 }

api/routes/v2-history/get_creator/index.ts

@@ -1,45 +1,45 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {getCreatorHandler} from "./get_creator";
 import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
-        description: 'get account creator',
-        summary: 'get account creator',
-        tags: ['accounts'],
-        querystring: {
-            type: 'object',
-            properties: {
-                "account": {
-                    description: 'created account',
-                    type: 'string',
-                    minLength: 1,
-                    maxLength: 12
-                }
-            },
-            required: ["account"]
-        },
-        response: extendResponseSchema({
-            "account": {
-                type: "string"
-            },
-            "creator": {
-                type: "string"
-            },
-            "timestamp": {
-                type: "string"
-            },
-            "block_num": {
-                type: "integer"
-            },
-            "trx_id": {
-                type: "string"
-            },
-            "indirect_creator": {
-                type: "string"
-            }
-        })
-    };
-    addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatorHandler, schema);
-    next();
+	const schema: FastifySchema = {
+		description: 'get account creator',
+		summary: 'get account creator',
+		tags: ['accounts'],
+		querystring: {
+			type: 'object',
+			properties: {
+				"account": {
+					description: 'created account',
+					type: 'string',
+					minLength: 1,
+					maxLength: 12
+				}
+			},
+			required: ["account"]
+		},
+		response: extendResponseSchema({
+			"account": {
+				type: "string"
+			},
+			"creator": {
+				type: "string"
+			},
+			"timestamp": {
+				type: "string"
+			},
+			"block_num": {
+				type: "integer"
+			},
+			"trx_id": {
+				type: "string"
+			},
+			"indirect_creator": {
+				type: "string"
+			}
+		})
+	};
+	addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatorHandler, schema);
+	next();
 }

api/routes/v2-history/get_deltas/get_deltas.ts

@@ -1,91 +1,99 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {mergeDeltaMeta, timedQuery} from "../../../helpers/functions";
 import {applyTimeFilter} from "../get_actions/functions";
 
 async function getDeltas(fastify: FastifyInstance, request: FastifyRequest) {
-	let skip, limit;
-	let sort_direction = 'desc';
-	const mustArray = [];
-	for (const param in request.query) {
-		if (Object.prototype.hasOwnProperty.call(request.query, param)) {
-			const value = request.query[param];
-			switch (param) {
-				case 'limit': {
-					limit = parseInt(value, 10);
-					if (limit < 1) {
-						return 'invalid limit parameter';
-					}
-					break;
-				}
-				case 'skip': {
-					skip = parseInt(value, 10);
-					if (skip < 0) {
-						return 'invalid skip parameter';
-					}
-					break;
-				}
-				case 'sort': {
-					if (value === 'asc' || value === '1') {
-						sort_direction = 'asc';
-					} else if (value === 'desc' || value === '-1') {
-						sort_direction = 'desc'
-					} else {
-						return 'invalid sort direction';
-					}
-					break;
-				}
-				case 'before': {
-					break;
-				}
-				case 'after': {
-					break;
-				}
-				default: {
-					const values = request.query[param].split(",");
-					if (values.length > 1) {
-						const terms = {};
-						terms[param] = values;
-						mustArray.push({terms: terms});
-					} else {
-						const term = {};
-						term[param] = values[0];
-						mustArray.push({term: term});
-					}
-					break;
-				}
-			}
-		}
-	}
+    let skip, limit;
+    let sort_direction = 'desc';
+    const mustArray = [];
+    const query: any = request.query;
+    for (const param in query) {
+        if (Object.prototype.hasOwnProperty.call(query, param)) {
+            const value = query[param];
+            switch (param) {
+                case 'limit': {
+                    limit = parseInt(value, 10);
+                    if (limit < 1) {
+                        return 'invalid limit parameter';
+                    }
+                    break;
+                }
+                case 'skip': {
+                    skip = parseInt(value, 10);
+                    if (skip < 0) {
+                        return 'invalid skip parameter';
+                    }
+                    break;
+                }
+                case 'sort': {
+                    if (value === 'asc' || value === '1') {
+                        sort_direction = 'asc';
+                    } else if (value === 'desc' || value === '-1') {
+                        sort_direction = 'desc'
+                    } else {
+                        return 'invalid sort direction';
+                    }
+                    break;
+                }
+                case 'before': {
+                    break;
+                }
+                case 'after': {
+                    break;
+                }
+                default: {
+                    if (typeof value === 'string') {
+                        const values = query[param].split(",");
+                        if (values.length > 1) {
+                            const terms = {};
+                            terms[param] = values;
+                            mustArray.push({terms: terms});
+                        } else {
+                            const term = {};
+                            term[param] = values[0];
+                            mustArray.push({term: term});
+                        }
+                    } else {
+                        if (typeof value === 'number') {
+                            const term = {};
+                            term[param] = value;
+                            mustArray.push({term: term});
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+    }
 
-	const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
-	const queryStruct = {bool: {must: mustArray}};
+    const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
+    const queryStruct = {bool: {must: mustArray}};
 
-	applyTimeFilter(request.query, queryStruct);
+    applyTimeFilter(query, queryStruct);
 
-	const results = await fastify.elastic.search({
-		"index": fastify.manager.chain + '-delta-*',
-		"from": skip || 0,
-		"size": (limit > maxDeltas ? maxDeltas : limit) || 10,
-		"body": {
-			query: queryStruct,
-			sort: {
-				"block_num": sort_direction
-			}
-		}
-	});
-	const deltas = results['body']['hits']['hits'].map((d) => {
-		return mergeDeltaMeta(d._source);
-	});
-	return {
-		query_time: null,
-		total: results['body']['hits']['total'],
-		deltas
-	};
+    const results = await fastify.elastic.search({
+        "index": fastify.manager.chain + '-delta-*',
+        "from": skip || 0,
+        "size": (limit > maxDeltas ? maxDeltas : limit) || 10,
+        "body": {
+            query: queryStruct,
+            sort: {
+                "block_num": sort_direction
+            }
+        }
+    });
+    const deltas = results['body']['hits']['hits'].map((d) => {
+        return mergeDeltaMeta(d._source);
+    });
+    return {
+        query_time: null,
+        total: results['body']['hits']['total'],
+        deltas
+    };
 }
 
 export function getDeltasHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-		reply.send(await timedQuery(getDeltas, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getDeltas, fastify, request, route));
+    }
 }

api/routes/v2-history/get_deltas/index.ts

@@ -1,9 +1,9 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {getDeltasHandler} from "./get_deltas";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
+    const schema: FastifySchema = {
         description: 'get state deltas',
         summary: 'get state deltas',
         tags: ['history'],
@@ -31,7 +31,11 @@ export default function (fastify: FastifyInstance, opts: any, next) {
             "before": {
                 description: 'filter before specified date (ISO8601)',
                 type: 'string'
-            }
+            },
+            "present": {
+                description: 'delta present flag',
+                type: 'number'
+            },
         }),
         response: extendResponseSchema({
             "deltas": {
@@ -40,12 +44,12 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: 'object',
                     properties: {
                         "timestamp": {type: 'string'},
+                        "present": {type: 'number'},
                         "code": {type: 'string'},
                         "scope": {type: 'string'},
                         "table": {type: 'string'},
                         "primary_key": {type: 'string'},
                         "payer": {type: 'string'},
-                        "present": {type: 'boolean'},
                         "block_num": {type: 'number'},
                         "block_id": {type: 'string'},
                         "data": {

api/routes/v2-history/get_schedule/get_schedule.ts

@@ -1,5 +1,4 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {createHash} from "crypto";
 import {base58ToBinary, binaryToBase58} from "eosjs/dist/eosjs-numeric";
@@ -23,49 +22,84 @@ function convertToLegacyKey(block_signing_key: string) {
 }
 
 async function getSchedule(fastify: FastifyInstance, request: FastifyRequest) {
+    const query: any = request.query;
     const response: any = {
         producers: []
     };
-    const searchParams: Search<any> = {
-        track_total_hits: true,
-        index: fastify.manager.chain + "-block-*",
-        size: 1,
-        body: {
-            query: {
-                bool: {
-                    must: []
-                }
-            },
-            sort: {block_num: "desc"}
+    if (query.mode === 'activated') {
+        const searchParams: Search<any> = {
+            track_total_hits: true,
+            index: fastify.manager.chain + "-block-*",
+            size: 1,
+            body: {
+                query: {bool: {must: []}},
+                sort: {block_num: query.sort || "desc"}
+            }
+        };
+        if (query.version) {
+            searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": query.version}}});
+        } else {
+            searchParams.body.query.bool.must.push({"exists": {"field": "new_producers.version"}});
+        }
+        const apiResponse = await fastify.elastic.search(searchParams);
+        const results = apiResponse.body.hits.hits;
+        if (results && results.length > 0) {
+            response.timestamp = results[0]._source["@timestamp"];
+            response.block_num = results[0]._source.block_num;
+            response.version = results[0]._source.new_producers.version;
+            response.producers = results[0]._source.new_producers.producers.map(prod => {
+                return {
+                    ...prod,
+                    legacy_key: convertToLegacyKey(prod.block_signing_key)
+                }
+            });
         }
-    };
-    if (request.query.version) {
-        searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": request.query.version}}});
     } else {
-        searchParams.body.query.bool.must.push({
-            "exists": {
-                "field": "new_producers.version"
+        // default "proposed" mode
+        const searchParams: Search<any> = {
+            track_total_hits: true,
+            index: fastify.manager.chain + "-schedule-*",
+            size: 1,
+            body: {
+                query: {bool: {must: []}},
+                sort: {version: query.sort || "desc"}
             }
-        });
-    }
-    const apiResponse = await fastify.elastic.search(searchParams);
-    const results = apiResponse.body.hits.hits;
-    if (results) {
-        response.timestamp = results[0]._source["@timestamp"];
-        response.block_num = results[0]._source.block_num;
-        response.version = results[0]._source.new_producers.version;
-        response.producers = results[0]._source.new_producers.producers.map(prod => {
-            return {
-                ...prod,
-                legacy_key: convertToLegacyKey(prod.block_signing_key)
-            }
-        });
+        };
+        if (query.version) {
+            searchParams.body.query.bool.must.push({"term": {"version": {"value": query.version}}});
+        } else {
+            searchParams.body.query.bool.must.push({"exists": {"field": "version"}});
+        }
+        if (query.producer) {
+            searchParams.body.query.bool.must.push({"term": {"producers.name": {"value": query.producer}}});
+        }
+        if (query.key) {
+            searchParams.body.query.bool.must.push({"term": {"producers.keys": {"value": query.key}}});
+        }
+        const apiResponse = await fastify.elastic.search(searchParams);
+        const results = apiResponse.body.hits.hits;
+        if (results && results.length > 0) {
+            response.timestamp = results[0]._source["@timestamp"];
+            response.proposal_block_num = results[0]._source.block_num;
+            response.version = results[0]._source.version;
+            response.producers = results[0]._source.producers;
+            // sort producers by name
+            // response.producers.sort((a, b) => {
+            //     if (a.name < b.name) {
+            //         return -1;
+            //     }
+            //     if (a.name > b.name) {
+            //         return 1;
+            //     }
+            //     return 0;
+            // });
+        }
     }
     return response;
 }
 
 export function getScheduleHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getSchedule, fastify, request, route));
     }
 }

api/routes/v2-history/get_schedule/index.ts

@@ -15,7 +15,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: 'string'
                 },
                 "key": {
-                    description: 'search by key',
+                    description: 'search by signing key',
                     type: 'string'
                 },
                 "after": {
@@ -26,6 +26,10 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     description: 'filter before specified date (ISO8601)',
                     type: 'string'
                 },
+                "mode": {
+                    description: 'search mode (activated or proposed)',
+                    type: 'string',
+                },
                 "version": {
                     description: 'schedule version',
                     type: 'integer',
@@ -36,6 +40,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         response: extendResponseSchema({
             "timestamp": {type: "string"},
             "block_num": {type: "number"},
+            "proposed_block_num": {type: "number"},
             "version": {type: "number"},
             "producers": {
                 type: "array",
@@ -43,8 +48,10 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: "object",
                     properties: {
                         "producer_name": {type: "string"},
+                        "name": {type: "string"},
                         "block_signing_key": {type: "string"},
-                        "legacy_key": {type: "string"}
+                        "legacy_key": {type: "string"},
+                        "keys": {type: "array", items: {type: "string"}}
                     }
                 }
             },

api/routes/v2-history/get_table_state/get_table_state.ts

@@ -0,0 +1,88 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {mergeDeltaMeta, timedQuery} from "../../../helpers/functions";
+
+async function getTableState(fastify: FastifyInstance, request: FastifyRequest) {
+	const query: any = request.query;
+	const response = {
+		...query,
+		results: [],
+		next_key: null
+	};
+	const mustArray: any[] = [
+		{"term": {"code": query.code}},
+		{"term": {"table": query.table}}
+	];
+	if (query.block_num) {
+		mustArray.push({"range": {"block_num": {"lte": query.block_num}}});
+	}
+	let after_key = "";
+	if (query.after_key) {
+		after_key = query.after_key;
+	}
+	const results = await fastify.elastic.search({
+		index: fastify.manager.chain + '-delta-*',
+		body: {
+			"query": {
+				"bool": {
+					"must": mustArray
+				}
+			},
+			"size": 0,
+			"sort": [
+				{"block_num": "desc"},
+				{"primary_key": "desc"}
+			],
+			"aggs": {
+				"scope_buckets": {
+					"composite": {
+						"size": 25,
+						"after": {
+							"scope_pk": after_key
+						},
+						"sources": [
+							{
+								"scope_pk": {
+									"terms": {
+										"script": {
+											"source": "return doc['scope'].value + '-' + doc['primary_key'].value",
+											"lang": "painless"
+										}
+									}
+								}
+							}
+						]
+					},
+					"aggs": {
+						"last_doc": {
+							"top_hits": {
+								"size": 1,
+								"sort": [{"block_num": "desc"}],
+								"_source": {
+									"excludes": ["code"]
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	});
+	const scope_buckets = results.body.aggregations.scope_buckets;
+	if (scope_buckets && scope_buckets.buckets.length > 0) {
+		if (scope_buckets.after_key) {
+			response.next_key = scope_buckets.after_key.scope_pk;
+		}
+		response.results = scope_buckets.buckets.map(bucket => {
+			const row = mergeDeltaMeta(bucket.last_doc.hits.hits[0]._source);
+			delete row.table;
+			return row;
+		});
+	}
+	return response;
+}
+
+export function getTableStateHandler(fastify: FastifyInstance, route: string) {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getTableState, fastify, request, route));
+	}
+}

api/routes/v2-history/get_table_state/index.ts

@@ -0,0 +1,69 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getTableStateHandler} from "./get_table_state";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get table state at a specific block height',
+        summary: 'get table state at a specific block height',
+        tags: ['history'],
+        querystring: {
+            type: 'object',
+            required: ["code","table"],
+            properties: {
+                "code": {
+                    description: 'search by contract',
+                    type: 'string'
+                },
+                "table": {
+                    description: 'search by key',
+                    type: 'string'
+                },
+                "block_num": {
+                    description: 'target block',
+                    type: 'integer',
+                    minimum: 1
+                },
+                "after_key": {
+                    description: 'last key for pagination',
+                    type: 'string'
+                }
+            }
+        },
+        response: extendResponseSchema({
+            "code": {type: 'string'},
+            "table": {type: 'string'},
+            "block_num": {type: 'number'},
+            "after_key": {type: 'string'},
+            "next_key": {type: 'string'},
+            "results": {
+                type: "array",
+                items: {
+                    type: 'object',
+                    properties: {
+                        "scope": {type: 'string'},
+                        "primary_key": {type: 'string'},
+                        "payer": {type: 'string'},
+                        "timestamp": {type: 'string'},
+                        "block_num": {type: 'number'},
+                        "block_id": {type: 'string'},
+                        "present": {type: 'number'},
+                        "data": {
+                            type: 'object',
+                            additionalProperties: true
+                        }
+                    },
+                    additionalProperties: true
+                }
+            }
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getTableStateHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-history/get_transaction/get_transaction.ts

@@ -1,28 +1,49 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
+import {GetInfoResult} from "eosjs/dist/eosjs-rpc-interfaces";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
-
 	const redis = fastify.redis;
-	const trxId = request.query.id.toLowerCase();
-	const cachedData = await redis.hgetall(trxId);
-
-	let hits;
-	let hits2;
-
+	const query: any = request.query;
+	const trxId = query.id.toLowerCase();
+	const conf = fastify.manager.config;
+	const cachedData = await redis.hgetall('trx_' + trxId);
 	const response = {
-		"query_time_ms": undefined,
-		"cached": undefined,
-		"cache_expires_in": undefined,
-		"executed": false,
-		"hot_only": false,
-		"trx_id": trxId,
-		"lib": undefined,
-		"actions": [],
-		"generated": undefined
+		query_time_ms: undefined,
+		executed: false,
+		cached: undefined,
+		cache_expires_in: undefined,
+		trx_id: query.id,
+		lib: undefined,
+		cached_lib: false,
+		actions: undefined,
+		generated: undefined,
+		error: undefined
 	};
+	let hits;
 
+	// build get_info request with caching
+	const $getInfo = new Promise<GetInfoResult>(resolve => {
+		const key = `${fastify.manager.chain}_get_info`;
+		fastify.redis.get(key).then(value => {
+			if (value) {
+				response.cached_lib = true;
+				resolve(JSON.parse(value));
+			} else {
+				fastify.eosjs.rpc.get_info().then(value1 => {
+					fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
+					response.cached_lib = false;
+					resolve(value1);
+				}).catch((reason) => {
+					console.log(reason);
+					response.error = 'failed to get last_irreversible_block_num'
+					resolve(null);
+				});
+			}
+		});
+	});
+
+	// reconstruct hits from cached data
 	if (cachedData && Object.keys(cachedData).length > 0) {
 		const gsArr = [];
 		for (let cachedDataKey in cachedData) {
@@ -31,83 +52,77 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
 		gsArr.sort((a, b) => {
 			return a.global_sequence - b.global_sequence;
 		});
-		response.cache_expires_in = await redis.ttl(trxId);
 		hits = gsArr.map(value => {
-			return {_source: JSON.parse(value)}
+			return {
+				_source: JSON.parse(value)
+			};
 		});
+		const promiseResults = await Promise.all([
+			redis.ttl('trx_' + trxId),
+			$getInfo
+		]);
+		response.cache_expires_in = promiseResults[0];
+		response.lib = promiseResults[1].last_irreversible_block_num;
 	}
 
+	// search on ES if cache is not present
 	if (!hits) {
-		const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
-		let indexPattern = fastify.manager.chain + '-action-*';
-		if (request.query.hot_only) {
-			indexPattern = fastify.manager.chain + '-action';
+		const _size = conf.api.limits.get_trx_actions || 100;
+		const blockHint = parseInt(query.block_hint, 10);
+		let indexPattern = '';
+		if (blockHint) {
+			const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
+			indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
+		} else {
+			indexPattern = fastify.manager.chain + '-action-*';
 		}
+		let pResults;
+		try {
 
-		const pResults = await Promise.all([
-			fastify.eosjs.rpc.get_info(),
-			fastify.elastic.search({
+			// build search request
+			const $search = fastify.elastic.search({
 				index: indexPattern,
 				size: _size,
 				body: {
-					query: {bool: {must: [{term: {trx_id: request.query.id.toLowerCase()}}]}},
+					query: {bool: {must: [{term: {trx_id: trxId}}]}},
 					sort: {global_sequence: "asc"}
 				}
-			}),
-			fastify.elastic.search({
-				index: fastify.manager.chain + '-gentrx-*',
-				size: _size,
-				body: {
-					query: {bool: {must: [{term: {trx_id: request.query.id.toLowerCase()}}]}}
-				}
-			})
-		]);
-
-		const results = pResults[1];
-		const genTrxRes = pResults[2];
+			});
 
+			// execute in parallel
+			pResults = await Promise.all([$getInfo, $search]);
+		} catch (e) {
+			console.log(e.message);
+			if (e.meta.statusCode === 404) {
+				response.error = 'no data near block_hint'
+				return response;
+			}
+		}
+		hits = pResults[1]['body']['hits']['hits'];
 		response.lib = pResults[0].last_irreversible_block_num;
-
-		if (request.query.hot_only) {
-			response.hot_only = true;
-		}
-
-		hits = results['body']['hits']['hits'];
-		hits2 = genTrxRes['body']['hits']['hits'];
 	}
 
-	if (hits) {
-		if (hits.length > 0) {
-			let highestBlockNum = 0;
-			for (let action of hits) {
-				if (action._source.block_num > highestBlockNum) {
-					highestBlockNum = action._source.block_num;
-				}
+	if (hits.length > 0) {
+		let highestBlockNum = 0;
+		for (let action of hits) {
+			if (action._source.block_num > highestBlockNum) {
+				highestBlockNum = action._source.block_num;
 			}
-			for (let action of hits) {
-				if (action._source.block_num === highestBlockNum) {
-					action = action._source;
-					mergeActionMeta(action);
-					response.actions.push(action);
-				}
+		}
+		response.actions = [];
+		for (let action of hits) {
+			if (action._source.block_num === highestBlockNum) {
+				mergeActionMeta(action._source);
+				response.actions.push(action._source);
 			}
-			response.executed = true;
 		}
+		response.executed = true;
 	}
-
-	if (hits2 && hits2.length > 0) {
-		if (hits2[0]._source['@timestamp']) {
-			hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
-			delete hits2[0]._source['@timestamp'];
-		}
-		response.generated = hits2[0]._source;
-	}
-
 	return response;
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
 		reply.send(await timedQuery(getTransaction, fastify, request, route));
 	}
 }

api/routes/v2-history/get_transaction/index.ts

@@ -10,9 +10,13 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         querystring: {
             type: 'object',
             properties: {
-                "id": {
+                id: {
                     description: 'transaction id',
                     type: 'string'
+                },
+                block_hint: {
+                    description: 'block hint to speed up tx recovery',
+                    type: 'integer'
                 }
             },
             required: ["id"]

api/routes/v2-history/get_transaction_legacy/get_transaction_legacy.ts

@@ -1,12 +1,12 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
     const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
+    const query: any = request.query;
 
     let indexPattern = fastify.manager.chain + '-action-*';
-    if (request.query.hot_only) {
+    if (query.hot_only) {
         indexPattern = fastify.manager.chain + '-action';
     }
 
@@ -19,7 +19,7 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
                 query: {
                     bool: {
                         must: [
-                            {term: {trx_id: request.query.id.toLowerCase()}}
+                            {term: {trx_id: query.id.toLowerCase()}}
                         ]
                     }
                 },
@@ -35,7 +35,7 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
                 query: {
                     bool: {
                         must: [
-                            {term: {trx_id: request.query.id.toLowerCase()}}
+                            {term: {trx_id: query.id.toLowerCase()}}
                         ]
                     }
                 }
@@ -49,13 +49,13 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
     const response = {
         "executed": false,
         "hot_only": false,
-        "trx_id": request.query.id,
+        "trx_id": query.id,
         "lib": pResults[0].last_irreversible_block_num,
         "actions": [],
         "generated": undefined
     };
 
-    if (request.query.hot_only) {
+    if (query.hot_only) {
         response.hot_only = true;
     }
 
@@ -73,15 +73,15 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
         // 		}
         // 	}
         // }
-        // let useBlocknumber;
+        // let useBlockNumber;
         // if (Object.keys(producers).length > 1) {
-        // 	// multiple producers of the same tx id, forked actions are present, attempt to cleanup
+        // 	// multiple producers of the same tx id, forked actions are present, attempt to clean-up
         // 	let trueProd = '';
         // 	let highestActCount = 0;
         // 	for (const prod in producers) {
         // 		if (producers.hasOwnProperty(prod)) {
         // 			if(producers[prod] === highestActCount) {
-        // 				useBlocknumber = true;
+        // 				useBlockNumber = true;
         // 			} else if (producers[prod] > highestActCount) {
         // 				highestActCount = producers[prod];
         // 				trueProd = prod;
@@ -122,7 +122,7 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getTransaction, fastify, request, route));
     }
 }

api/routes/v2-state/get_account/get_account.ts

@@ -1,10 +1,12 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import got from "got";
 import {timedQuery} from "../../../helpers/functions";
 
 async function getAccount(fastify: FastifyInstance, request: FastifyRequest) {
 
+
+    const query: any = request.query;
+
     const response = {
         account: null,
         actions: null,
@@ -13,7 +15,7 @@ async function getAccount(fastify: FastifyInstance, request: FastifyRequest) {
         links: null
     };
 
-    const account = request.query.account;
+    const account = query.account;
     const reqQueue = [];
 
     try {
@@ -45,7 +47,7 @@ async function getAccount(fastify: FastifyInstance, request: FastifyRequest) {
 }
 
 export function getAccountHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getAccount, fastify, request, route));
     }
 }

api/routes/v2-state/get_key_accounts/get_key_accounts.ts

@@ -1,137 +1,139 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {Numeric} from "eosjs/dist";
 import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 function invalidKey() {
-    const err: any = new Error();
-    err.statusCode = 400;
-    err.message = 'invalid public key';
-    throw err;
+	const err: any = new Error();
+	err.statusCode = 400;
+	err.message = 'invalid public key';
+	throw err;
 }
 
 async function getKeyAccounts(fastify: FastifyInstance, request: FastifyRequest) {
 
-    let publicKey;
-    if (typeof request.body === 'string' && request.req.method === 'POST') {
-        request.body = JSON.parse(request.body);
-    }
+	let publicKey;
+	if (typeof request.body === 'string' && request.method === 'POST') {
+		request.body = JSON.parse(request.body);
+	}
 
-    const {skip, limit} = getSkipLimit(request.query);
-    const maxDocs = fastify.manager.config.api.limits.get_key_accounts ?? 1000;
+	const body: any = request.body;
+	const query: any = request.query;
 
-    const public_Key = request.req.method === 'POST' ? request.body.public_key : request.query.public_key;
+	const {skip, limit} = getSkipLimit(request.query);
+	const maxDocs = fastify.manager.config.api.limits.get_key_accounts ?? 1000;
 
-    if (public_Key.startsWith("PUB_")) {
-        publicKey = public_Key;
-    } else if (public_Key.startsWith("EOS")) {
-        try {
-            publicKey = Numeric.convertLegacyPublicKey(public_Key);
-        } catch (e) {
-            console.log(e.message);
-            invalidKey();
-        }
-    } else {
-        invalidKey();
-    }
+	const public_Key = request.method === 'POST' ? body.public_key : query.public_key;
 
-    const response = {
-        account_names: []
-    } as any;
+	if (public_Key.startsWith("PUB_")) {
+		publicKey = public_Key;
+	} else if (public_Key.startsWith("EOS")) {
+		try {
+			publicKey = Numeric.convertLegacyPublicKey(public_Key);
+		} catch (e) {
+			console.log(e.message);
+			invalidKey();
+		}
+	} else {
+		invalidKey();
+	}
 
-    if (request.req.method === 'GET' && request.query.details) {
-        response.permissions = [];
-    }
+	const response = {
+		account_names: []
+	} as any;
 
-    try {
+	if (request.method === 'GET' && query.details) {
+		response.permissions = [];
+	}
 
-        const permTableResults = await fastify.elastic.search({
-            index: fastify.manager.chain + '-perm-*',
-            size: (limit > maxDocs ? maxDocs : limit) || 100,
-            from: skip || 0,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                term: {
-                                    "auth.keys.key.keyword": publicKey
-                                }
-                            }
-                        ],
-                    }
-                }
-            }
-        });
+	try {
 
-        if (permTableResults.body.hits.hits.length > 0) {
-            for (const perm of permTableResults.body.hits.hits) {
-                response.account_names.push(perm._source.owner);
-                if (request.req.method === 'GET' && request.query.details) {
-                    response.permissions.push(perm._source);
-                }
-            }
-        }
+		const permTableResults = await fastify.elastic.search({
+			index: fastify.manager.chain + '-perm-*',
+			size: (limit > maxDocs ? maxDocs : limit) || 100,
+			from: skip || 0,
+			body: {
+				query: {
+					bool: {
+						must: [
+							{
+								term: {
+									"auth.keys.key.keyword": publicKey
+								}
+							}
+						],
+					}
+				}
+			}
+		});
 
-    } catch (e) {
-        console.log(e);
-    }
+		if (permTableResults.body.hits.hits.length > 0) {
+			for (const perm of permTableResults.body.hits.hits) {
+				response.account_names.push(perm._source.owner);
+				if (request.method === 'GET' && query.details) {
+					response.permissions.push(perm._source);
+				}
+			}
+		}
 
-    if (response.account_names.length > 0) {
-        response.account_names = [...(new Set(response.account_names))];
-        return response;
-    }
+	} catch (e) {
+		console.log(e);
+	}
 
-    // Fallback to action search
-    const _body = {
-        query: {
-            bool: {
-                should: [
-                    {term: {"@updateauth.auth.keys.key.keyword": publicKey}},
-                    {term: {"@newaccount.active.keys.key.keyword": publicKey}},
-                    {term: {"@newaccount.owner.keys.key.keyword": publicKey}}
-                ],
-                minimum_should_match: 1
-            }
-        },
-        sort: [{"global_sequence": {"order": "desc"}}]
-    };
+	if (response.account_names.length > 0) {
+		response.account_names = [...(new Set(response.account_names))];
+		return response;
+	}
 
-    const results = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        size: (limit > maxDocs ? maxDocs : limit) || 100,
-        from: skip || 0,
-        body: _body
-    });
+	// Fallback to action search
+	const _body = {
+		query: {
+			bool: {
+				should: [
+					{term: {"@updateauth.auth.keys.key.keyword": publicKey}},
+					{term: {"@newaccount.active.keys.key.keyword": publicKey}},
+					{term: {"@newaccount.owner.keys.key.keyword": publicKey}}
+				],
+				minimum_should_match: 1
+			}
+		},
+		sort: [{"global_sequence": {"order": "desc"}}]
+	};
 
-    if (results['body']['hits']['hits'].length > 0) {
-        response.account_names = results['body']['hits']['hits'].map((v) => {
-            if (v._source.act.name === 'newaccount') {
-                if (v._source['@newaccount'].newact) {
-                    return v._source['@newaccount'].newact;
-                } else if (v._source.act.data.newact) {
-                    return v._source.act.data.newact;
-                } else {
-                    return null;
-                }
-            } else if (v._source.act.name === 'updateauth') {
-                return v._source.act.data.account;
-            } else {
-                return null;
-            }
-        });
-    }
+	const results = await fastify.elastic.search({
+		index: fastify.manager.chain + '-action-*',
+		size: (limit > maxDocs ? maxDocs : limit) || 100,
+		from: skip || 0,
+		body: _body
+	});
 
-    if (response.account_names.length > 0) {
-        response.account_names = [...(new Set(response.account_names))];
-    }
+	if (results['body']['hits']['hits'].length > 0) {
+		response.account_names = results['body']['hits']['hits'].map((v) => {
+			if (v._source.act.name === 'newaccount') {
+				if (v._source['@newaccount'].newact) {
+					return v._source['@newaccount'].newact;
+				} else if (v._source.act.data.newact) {
+					return v._source.act.data.newact;
+				} else {
+					return null;
+				}
+			} else if (v._source.act.name === 'updateauth') {
+				return v._source.act.data.account;
+			} else {
+				return null;
+			}
+		});
+	}
 
-    return response;
+	if (response.account_names.length > 0) {
+		response.account_names = [...(new Set(response.account_names))];
+	}
+
+	return response;
 }
 
 export function getKeyAccountsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getKeyAccounts, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getKeyAccounts, fastify, request, route));
+	}
 }

api/routes/v2-state/get_key_accounts/index.ts

@@ -38,7 +38,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                                 last_updated: {type: 'string'},
                                 auth: {},
                                 name: {type: 'string'},
-                                present: {type: 'boolean'}
+                                present: {type: 'number'}
                             }
                         }
                     }

api/routes/v2-state/get_links/get_links.ts

@@ -1,71 +1,71 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {getTrackTotalHits, timedQuery} from "../../../helpers/functions";
 import {ApiResponse} from "@elastic/elasticsearch";
 import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 async function getLinks(fastify: FastifyInstance, request: FastifyRequest) {
-    const query = request.query;
-    const {account, code, action, permissions} = query;
-    const {skip, limit} = getSkipLimit(query);
+	const query: any = request.query;
+	const {account, code, action, permissions} = query;
+	const {skip, limit} = getSkipLimit(query);
 
-    const queryStruct = {
-        "bool": {
-            must: []
-        }
-    };
+	const queryStruct = {
+		"bool": {
+			must: [],
+			must_not: []
+		}
+	};
 
-    if (account) {
-        queryStruct.bool.must.push({'term': {'account': account}});
-    }
+	if (account) {
+		queryStruct.bool.must.push({'term': {'account': account}});
+	}
 
-    if (code) {
-        queryStruct.bool.must.push({'term': {'code': code}});
-    }
+	if (code) {
+		queryStruct.bool.must.push({'term': {'code': code}});
+	}
 
-    if (action) {
-        queryStruct.bool.must.push({'term': {'action': action}});
-    }
+	if (action) {
+		queryStruct.bool.must.push({'term': {'action': action}});
+	}
 
-    if (permissions) {
-        queryStruct.bool.must.push({'term': {'permissions': permissions}});
-    }
+	if (permissions) {
+		queryStruct.bool.must.push({'term': {'permissions': permissions}});
+	}
 
-    // only present deltas
-    queryStruct.bool.must.push({'term': {'present': true}});
+	// only present deltas
+	queryStruct.bool.must_not.push({'term': {'present': 0}});
 
-    // Prepare query body
-    const query_body = {
-        track_total_hits: getTrackTotalHits(query),
-        query: queryStruct,
-        sort: {block_num: 'desc'}
-    };
+	// Prepare query body
+	const query_body = {
+		track_total_hits: getTrackTotalHits(query),
+		query: queryStruct,
+		sort: {block_num: 'desc'}
+	};
 
-    const maxLinks = fastify.manager.config.api.limits.get_links;
-    const results: ApiResponse = await fastify.elastic.search({
-        index: fastify.manager.chain + '-link-*',
-        from: skip || 0,
-        size: (limit > maxLinks ? maxLinks : limit) || 50,
-        body: query_body
-    });
-    const hits = results.body.hits.hits;
-    const response: any = {
-        cached: false,
-        total: results.body.hits.total,
-        links: []
-    };
+	const maxLinks = fastify.manager.config.api.limits.get_links;
+	const results: ApiResponse = await fastify.elastic.search({
+		index: fastify.manager.chain + '-link-*',
+		from: skip || 0,
+		size: (limit > maxLinks ? maxLinks : limit) || 50,
+		body: query_body
+	});
+	const hits = results.body.hits.hits;
+	const response: any = {
+		cached: false,
+		total: results.body.hits.total,
+		links: []
+	};
 
-    for (const hit of hits) {
-        const link = hit._source;
-        link.timestamp = link['@timestamp'];
-        delete link['@timestamp'];
-        response.links.push(link);
-    }
-    return response;
+	for (const hit of hits) {
+		const link = hit._source;
+		link.timestamp = link['@timestamp'];
+		delete link['@timestamp'];
+		response.links.push(link);
+	}
+	return response;
 }
 
 export function getLinksHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getLinks, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getLinks, fastify, request, route));
+	}
 }

api/routes/v2-state/get_proposals/get_proposals.ts

@@ -1,16 +1,17 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {getTrackTotalHits, timedQuery} from "../../../helpers/functions";
 
 async function getProposals(fastify: FastifyInstance, request: FastifyRequest) {
 
+    const query: any = request.query;
+
     // Pagination
     let skip, limit;
-    skip = parseInt(request.query.skip, 10);
+    skip = parseInt(query.skip, 10);
     if (skip < 0) {
         return 'invalid skip parameter';
     }
-    limit = parseInt(request.query.limit, 10);
+    limit = parseInt(query.limit, 10);
     if (limit < 1) {
         return 'invalid limit parameter';
     }
@@ -22,8 +23,8 @@ async function getProposals(fastify: FastifyInstance, request: FastifyRequest) {
     };
 
     // Filter by accounts
-    if (request.query.account) {
-        const accounts = request.query.account.split(',');
+    if (query.account) {
+        const accounts = query.account.split(',');
         for(const acc of accounts) {
             queryStruct.bool.must.push({
                 "bool": {
@@ -37,28 +38,28 @@ async function getProposals(fastify: FastifyInstance, request: FastifyRequest) {
     }
 
     // Filter by proposer account
-    if (request.query.proposer) {
-        queryStruct.bool.must.push({"term": {"proposer": request.query.proposer}});
+    if (query.proposer) {
+        queryStruct.bool.must.push({"term": {"proposer": query.proposer}});
     }
 
     // Filter by proposal name
-    if (request.query.proposal) {
-        queryStruct.bool.must.push({"term": {"proposal_name": request.query.proposal}});
+    if (query.proposal) {
+        queryStruct.bool.must.push({"term": {"proposal_name": query.proposal}});
     }
 
     // Filter by execution status
-    if (typeof request.query.executed !== 'undefined') {
-        queryStruct.bool.must.push({"term": {"executed": request.query.executed}});
+    if (typeof query.executed !== 'undefined') {
+        queryStruct.bool.must.push({"term": {"executed": query.executed}});
     }
 
     // Filter by requested actors
-    if (request.query.requested) {
-        queryStruct.bool.must.push({"term": {"requested_approvals.actor": request.query.requested}});
+    if (query.requested) {
+        queryStruct.bool.must.push({"term": {"requested_approvals.actor": query.requested}});
     }
 
     // Filter by provided actors
-    if (request.query.provided) {
-        queryStruct.bool.must.push({"term": {"provided_approvals.actor": request.query.provided}});
+    if (query.provided) {
+        queryStruct.bool.must.push({"term": {"provided_approvals.actor": query.provided}});
     }
 
     // If no filter switch to full match
@@ -97,7 +98,7 @@ async function getProposals(fastify: FastifyInstance, request: FastifyRequest) {
 }
 
 export function getProposalsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getProposals, fastify, request, route));
     }
 }

api/routes/v2-state/get_tokens/get_tokens.ts

@@ -1,4 +1,3 @@
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {getSkipLimit} from "../../v2-history/get_actions/functions";
@@ -6,7 +5,12 @@ import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
 
-    const response = {'account': request.query.account, 'tokens': []};
+    const query: any = request.query;
+
+    const response = {
+        account: query.account,
+        tokens: []
+    };
 
     const {skip, limit} = getSkipLimit(request.query);
     const maxDocs = fastify.manager.config.api.limits.get_tokens ?? 100;
@@ -18,55 +22,69 @@ async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
         "body": {
             query: {
                 bool: {
-                    filter: [{term: {"scope": request.query.account}}]
+                    filter: [{term: {scope: query.account}}]
                 }
             }
         }
     });
 
+    const testSet = new Set();
     for (const hit of stateResult.body.hits.hits) {
         const data = hit._source;
-        if (typeof data.present !== "undefined" && data.present === false) {
+        if (typeof data.present !== "undefined" && data.present === 0) {
             continue;
         }
         let precision;
+        let token_data;
+        let errorMsg;
+
         const key = `${data.code}_${data.symbol}`;
+
+        if (testSet.has(key)) {
+            continue;
+        }
+
+        testSet.add(key);
         if (!fastify.tokenCache) {
             fastify.tokenCache = new Map<string, any>();
         }
         if (fastify.tokenCache.has(key)) {
             precision = fastify.tokenCache.get(key).precision;
         } else {
-            let token_data;
             try {
-                token_data = await fastify.eosjs.rpc.get_currency_balance(data.code, request.query.account, data.symbol);
+                token_data = await fastify.eosjs.rpc.get_currency_balance(data.code, query.account, data.symbol);
                 if (token_data.length > 0) {
                     const [amount, symbol] = token_data[0].split(" ");
                     const amount_arr = amount.split(".");
                     if (amount_arr.length === 2) {
                         precision = amount_arr[1].length;
                         fastify.tokenCache.set(key, {precision});
-                        // console.log('Caching token precision -', key, precision);
                     }
                 }
             } catch (e) {
-                console.log(`get_currency_balance error - contract:${data.code} - account:${request.query.account}`);
+                errorMsg = e.message;
             }
         }
 
-        response.tokens.push({
+        const resp: Record<string, any> = {
             symbol: data.symbol,
             precision: precision,
             amount: parseFloat(data.amount),
-            contract: data.code
-        });
+            contract: data.code,
+        };
+
+        if (errorMsg) {
+            resp.error = errorMsg;
+        }
+
+        response.tokens.push(resp);
     }
 
     return response;
 }
 
 export function getTokensHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getTokens, fastify, request, route));
     }
 }

api/routes/v2-state/get_voters/get_voters.ts

@@ -1,63 +1,63 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 async function getVoters(fastify: FastifyInstance, request: FastifyRequest) {
 
-    const {skip, limit} = getSkipLimit(request.query);
+	const query: any = request.query;
+	const {skip, limit} = getSkipLimit(request.query);
 
-    const response = {
-        voter_count: 0,
-        'voters': []
-    };
-    let queryStruct: any = {
-        "bool": {
-            "must": []
-        }
-    };
+	const response = {
+		voter_count: 0,
+		'voters': []
+	};
+	let queryStruct: any = {
+		"bool": {
+			"must": []
+		}
+	};
 
-    if (request.query.producer) {
-        for (const bp of request.query.producer.split(",")) {
-            queryStruct.bool.must.push({"term": {"producers": bp}});
-        }
-    }
+	if (query.producer) {
+		for (const bp of query.producer.split(",")) {
+			queryStruct.bool.must.push({"term": {"producers": bp}});
+		}
+	}
 
-    if (request.query.proxy === 'true') {
-        queryStruct.bool.must.push({"term": {"is_proxy": true}});
-    }
+	if (query.proxy === 'true') {
+		queryStruct.bool.must.push({"term": {"is_proxy": true}});
+	}
 
-    if (queryStruct.bool.must.length === 0) {
-        queryStruct = {
-            "match_all": {}
-        };
-    }
+	if (queryStruct.bool.must.length === 0) {
+		queryStruct = {
+			"match_all": {}
+		};
+	}
 
-    const maxDocs = fastify.manager.config.api.limits.get_voters ?? 100;
-    const results = await fastify.elastic.search({
-        "index": fastify.manager.chain + '-table-voters-*',
-        "from": skip || 0,
-        "size": (limit > maxDocs ? maxDocs : limit) || 10,
-        "body": {
-            "query": queryStruct,
-            "sort": [{"last_vote_weight": "desc"}]
-        }
-    });
-    const hits = results['body']['hits']['hits'];
-    for (const hit of hits) {
-        const voter = hit._source;
-        response.voters.push({
-            account: voter.voter,
-            weight: voter.last_vote_weight,
-            last_vote: voter.block_num
-        });
-    }
-    response.voter_count = results['body']['hits']['total']['value'];
-    return response;
+	const maxDocs = fastify.manager.config.api.limits.get_voters ?? 100;
+	const results = await fastify.elastic.search({
+		"index": fastify.manager.chain + '-table-voters-*',
+		"from": skip || 0,
+		"size": (limit > maxDocs ? maxDocs : limit) || 10,
+		"body": {
+			"query": queryStruct,
+			"sort": [{"last_vote_weight": "desc"}]
+		}
+	});
+	const hits = results['body']['hits']['hits'];
+	for (const hit of hits) {
+		const voter = hit._source;
+		response.voters.push({
+			account: voter.voter,
+			weight: voter.last_vote_weight,
+			last_vote: voter.block_num
+		});
+	}
+	response.voter_count = results['body']['hits']['total']['value'];
+	return response;
 }
 
 export function getVotersHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getVoters, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getVoters, fastify, request, route));
+	}
 }

api/routes/v2-stats/get_action_usage/get_action_usage.ts

@@ -1,121 +1,120 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 
 async function getLastSeq(fastify: FastifyInstance, date: string) {
-    const req = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        body: {
-            "size": 1,
-            "query": {
-                "bool": {
-                    "must": [
-                        {"range": {"@timestamp": {"lt": date}}}
-                    ]
-                }
-            },
-            "_source": {
-                "includes": ["global_sequence"]
-            },
-            "sort": [{"global_sequence": "desc"}]
-        }
-    });
-    return req.body.hits.hits[0]._source.global_sequence;
+	const req = await fastify.elastic.search({
+		index: fastify.manager.chain + '-action-*',
+		body: {
+			"size": 1,
+			"query": {
+				"bool": {
+					"must": [
+						{"range": {"@timestamp": {"lt": date}}}
+					]
+				}
+			},
+			"_source": {
+				"includes": ["global_sequence"]
+			},
+			"sort": [{"global_sequence": "desc"}]
+		}
+	});
+	return req.body.hits.hits[0]._source.global_sequence;
 }
 
 async function getTxCount(fastify: FastifyInstance, dateFrom: string, dateTo: string) {
-    const req = await fastify.elastic.count({
-        index: fastify.manager.chain + '-action-*',
-        body: {
-            "query": {
-                "bool": {
-                    "must": [
-                        {"term": {"creator_action_ordinal": 0}},
-                        {"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}
-                    ]
-                }
-            }
-        }
-    });
-    return req.body.count;
+	const req = await fastify.elastic.count({
+		index: fastify.manager.chain + '-action-*',
+		body: {
+			"query": {
+				"bool": {
+					"must": [
+						{"term": {"creator_action_ordinal": 0}},
+						{"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}
+					]
+				}
+			}
+		}
+	});
+	return req.body.count;
 }
 
 async function getUniqueActors(fastify: FastifyInstance, dateFrom: string, dateTo: string) {
-    const req = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        size: 0,
-        body: {
-            "aggs": {
-                "unique_actors": {
-                    "cardinality": {
-                        "field": "act.authorization.actor"
-                    }
-                }
-            },
-            "query": {
-                "bool": {
-                    "filter": [{"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}]
-                }
-            }
-        }
-    });
-    return req.body.aggregations.unique_actors.value;
+	const req = await fastify.elastic.search({
+		index: fastify.manager.chain + '-action-*',
+		size: 0,
+		body: {
+			"aggs": {
+				"unique_actors": {
+					"cardinality": {
+						"field": "act.authorization.actor"
+					}
+				}
+			},
+			"query": {
+				"bool": {
+					"filter": [{"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}]
+				}
+			}
+		}
+	});
+	return req.body.aggregations.unique_actors.value;
 }
 
 async function getActionUsage(fastify: FastifyInstance, request: FastifyRequest) {
-    const query = request.query;
-    let now = new Date();
-    let expiration = 86400;
-    if (query.end_date) {
-        now = new Date(query.end_date);
-    }
-    now.setMinutes(0, 0, 0);
-    let period;
-    if (query.period === '1h') {
-        period = 3600000;
-    } else if (query.period === '24h' || query.period === '1d') {
-        period = 86400000;
-    } else {
-        return {};
-    }
+	const query: any = request.query;
+	let now = new Date();
+	let expiration = 86400;
+	if (query.end_date) {
+		now = new Date(query.end_date);
+	}
+	now.setMinutes(0, 0, 0);
+	let period;
+	if (query.period === '1h') {
+		period = 3600000;
+	} else if (query.period === '24h' || query.period === '1d') {
+		period = 86400000;
+	} else {
+		return {};
+	}
 
-    if (!query.end_date) {
-        expiration = Math.floor((period - (Date.now() - now.getTime())) / 1000);
-    }
+	if (!query.end_date) {
+		expiration = Math.floor((period - (Date.now() - now.getTime())) / 1000);
+	}
 
-    const response = {} as any;
-    const periodStart = new Date(now.getTime() - period);
-    const pStart = periodStart.toISOString();
-    const pEnd = now.toISOString();
+	const response = {} as any;
+	const periodStart = new Date(now.getTime() - period);
+	const pStart = periodStart.toISOString();
+	const pEnd = now.toISOString();
 
-    const cacheKey = `${fastify.manager.chain}_act_stats_${pStart}_${pEnd}_${query.unique_actors}`;
-    const cachedValue = await fastify.redis.get(cacheKey);
-    if (cachedValue) {
-        const cachedResponse = JSON.parse(cachedValue);
-        cachedResponse.cached = true;
-        cachedResponse.cache_exp = expiration;
-        return cachedResponse;
-    }
+	const cacheKey = `${fastify.manager.chain}_act_stats_${pStart}_${pEnd}_${query.unique_actors}`;
+	const cachedValue = await fastify.redis.get(cacheKey);
+	if (cachedValue) {
+		const cachedResponse = JSON.parse(cachedValue);
+		cachedResponse.cached = true;
+		cachedResponse.cache_exp = expiration;
+		return cachedResponse;
+	}
 
-    const firstReq = await getLastSeq(fastify, pStart);
-    const lastReq = await getLastSeq(fastify, pEnd);
-    response.action_count = lastReq - firstReq;
-    response.tx_count = await getTxCount(fastify, pStart, pEnd);
+	const firstReq = await getLastSeq(fastify, pStart);
+	const lastReq = await getLastSeq(fastify, pEnd);
+	response.action_count = lastReq - firstReq;
+	response.tx_count = await getTxCount(fastify, pStart, pEnd);
 
-    if (query.unique_actors) {
-        response.unique_actors = await getUniqueActors(fastify, pStart, pEnd);
-    }
+	if (query.unique_actors) {
+		response.unique_actors = await getUniqueActors(fastify, pStart, pEnd);
+	}
 
-    response.period = query.period;
-    response.from = pStart;
-    response.to = pEnd;
+	response.period = query.period;
+	response.from = pStart;
+	response.to = pEnd;
 
-    await fastify.redis.set(cacheKey, JSON.stringify(response), 'EX', expiration);
-    return response;
+	await fastify.redis.set(cacheKey, JSON.stringify(response), 'EX', expiration);
+	return response;
 }
 
 export function getActionUsageHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getActionUsage, fastify, request, route));
-    }
-}
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getActionUsage, fastify, request, route));
+	}
+}

api/routes/v2-stats/get_api_usage/get_api_usage.ts

@@ -0,0 +1,53 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {timedQuery} from "../../../helpers/functions";
+import {parseInt} from "lodash";
+
+async function getApiUsage(fastify: FastifyInstance, request: FastifyRequest) {
+    const response = {
+        total: {responses: {}},
+        buckets: []
+    } as any;
+    const data = await fastify.redis.keys(`stats:${fastify.manager.chain}:*`);
+    const buckets = [];
+    for (const key of data) {
+        const parts = key.split(':');
+        if (parts[2] === 'H') {
+            const bucket = {
+                timestamp: new Date(parseInt(parts[3])),
+                responses: {}
+            };
+            const sortedSet = await fastify.redis.zrange(key, 0, -1, 'WITHSCORES');
+            for (let i = 0; i < sortedSet.length; i += 2) {
+                const [status, path] = sortedSet[i].split(']');
+                if (status) {
+                    const statusCode = status.slice(1);
+                    if (!bucket.responses[statusCode]) {
+                        bucket.responses[statusCode] = {};
+                    }
+                    if (statusCode) {
+                        bucket.responses[statusCode][path] = parseInt(sortedSet[i + 1]);
+                    }
+                }
+            }
+            response.buckets.push(bucket);
+        } else {
+            if (parts[2] === 'total' && parts[4]) {
+                const statusCode = parts[3];
+                if (!response.total.responses[statusCode]) {
+                    response.total.responses[statusCode] = {};
+                }
+                const value = await fastify.redis.get(key);
+                response.total.responses[statusCode][parts[4]] = parseInt(value);
+            }
+        }
+    }
+    // reverse sorte by date
+    response.buckets.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+    return response;
+}
+
+export function getApiUsageHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getApiUsage, fastify, request, route));
+    }
+}

api/routes/v2-stats/get_api_usage/index.ts

@@ -0,0 +1,41 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getApiUsageHandler} from "./get_api_usage";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get hyperion api usage statistics',
+        summary: 'get hyperion api usage statistics',
+        tags: ['stats'],
+        response: extendResponseSchema({
+            total: {
+                type: 'object',
+                properties: {
+                    responses: {
+                        additionalProperties: true
+                    }
+                }
+            },
+            buckets: {
+                type: 'array',
+                items: {
+                    type: 'object',
+                    properties: {
+                        timestamp: {type: 'string'},
+                        responses: {
+                            additionalProperties: true
+                        }
+                    }
+                }
+            }
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getApiUsageHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-stats/get_missed_blocks/get_missed_blocks.ts

@@ -1,64 +1,64 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {timedQuery} from "../../../helpers/functions";
-import {ServerResponse} from "http";
 import {Search} from "@elastic/elasticsearch/api/requestParams";
 import {applyTimeFilter} from "../../v2-history/get_actions/functions";
 
 async function getMissedBlocks(fastify: FastifyInstance, request: FastifyRequest) {
-    const response = {
-        stats: {
-            by_producer: {}
-        },
-        events: []
-    };
-    const searchParams: Search<any> = {
-        track_total_hits: true,
-        index: fastify.manager.chain + "-logs-*",
-        size: 100,
-        body: {
-            query: {
-                bool: {
-                    must: [
-                        {term: {"type": "missed_blocks"}}
-                    ]
-                }
-            },
-            sort: {
-                "@timestamp": "desc"
-            }
-        }
-    };
+	const query: any = request.query;
+	const response = {
+		stats: {
+			by_producer: {}
+		},
+		events: []
+	};
+	const searchParams: Search<any> = {
+		track_total_hits: true,
+		index: fastify.manager.chain + "-logs-*",
+		size: 100,
+		body: {
+			query: {
+				bool: {
+					must: [
+						{term: {"type": "missed_blocks"}}
+					]
+				}
+			},
+			sort: {
+				"@timestamp": "desc"
+			}
+		}
+	};
 
-    let minBlocks = 0;
-    if (request.query.min_blocks) {
-        minBlocks = parseInt(request.query.min_blocks);
-        searchParams.body.query.bool.must.push({range: {"missed_blocks.size": {gte: minBlocks}}})
-    }
+	let minBlocks = 0;
+	if (query.min_blocks) {
+		minBlocks = parseInt(query.min_blocks);
+		searchParams.body.query.bool.must.push({range: {"missed_blocks.size": {gte: minBlocks}}})
+	}
 
-    if (request.query.producer) {
-        searchParams.body.query.bool.must.push({term: {"missed_blocks.producer": request.query.producer}});
-    }
+	if (query.producer) {
+		searchParams.body.query.bool.must.push({term: {"missed_blocks.producer": query.producer}});
+	}
 
-    applyTimeFilter(request.query, searchParams.body.query);
+	applyTimeFilter(query, searchParams.body.query);
 
-    const apiResponse = await fastify.elastic.search(searchParams);
-    apiResponse.body.hits.hits.forEach(v => {
-        const ev = v._source;
-        response.events.push({
-            "@timestamp": ev["@timestamp"],
-            ...ev.missed_blocks
-        });
-        if (!response.stats.by_producer[ev.missed_blocks.producer]) {
-            response.stats.by_producer[ev.missed_blocks.producer] = ev.missed_blocks.size;
-        } else {
-            response.stats.by_producer[ev.missed_blocks.producer] += ev.missed_blocks.size;
-        }
-    });
-    return response;
+	const apiResponse = await fastify.elastic.search(searchParams);
+	apiResponse.body.hits.hits.forEach(v => {
+		const ev = v._source;
+		response.events.push({
+			"@timestamp": ev["@timestamp"],
+			...ev.missed_blocks
+		});
+		if (!response.stats.by_producer[ev.missed_blocks.producer]) {
+			response.stats.by_producer[ev.missed_blocks.producer] = ev.missed_blocks.size;
+		} else {
+			response.stats.by_producer[ev.missed_blocks.producer] += ev.missed_blocks.size;
+		}
+	});
+	return response;
 }
 
 export function getMissedBlocksHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getMissedBlocks, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getMissedBlocks, fastify, request, route));
+	}
 }

api/routes/v2-stats/get_resource_usage/get_resource_usage.ts

@@ -1,16 +1,16 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 
 const percentiles = [1, 5, 25, 50, 75, 95, 99];
 
 async function getResourceUsage(fastify: FastifyInstance, request: FastifyRequest) {
+	const query: any = request.query;
 	const searchBody: any = {
 		query: {
 			bool: {
 				must: [
-					{term: {"act.account": request.query.code}},
-					{term: {"act.name": request.query.action}},
+					{term: {"act.account": query.code}},
+					{term: {"act.name": query.action}},
 					{term: {"action_ordinal": {"value": 1}}},
 					{
 						range: {
@@ -69,7 +69,7 @@ async function getResourceUsage(fastify: FastifyInstance, request: FastifyReques
 }
 
 export function getResourceUsageHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
 		reply.send(await timedQuery(getResourceUsage, fastify, request, route));
 	}
 }

api/routes/v2/health/health.ts

@@ -1,10 +1,10 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {connect} from "amqplib";
 import {timedQuery} from "../../../helpers/functions";
-import {getLastIndexedBlockWithTotalBlocks} from "../../../../helpers/common_functions";
+import {getFirstIndexedBlock, getLastIndexedBlockWithTotalBlocks} from "../../../../helpers/common_functions";
 
-async function checkRabbit(fastify: FastifyInstance) {
+
+async function checkRabbit(fastify: FastifyInstance): Promise<ServiceResponse<any>> {
     try {
         const connection = await connect(fastify.manager.ampqUrl);
         await connection.close();
@@ -15,7 +15,15 @@ async function checkRabbit(fastify: FastifyInstance) {
     }
 }
 
-async function checkNodeos(fastify: FastifyInstance) {
+interface NodeosService {
+    head_block_num: number;
+    head_block_time: string;
+    time_offset: number;
+    last_irreversible_block: number;
+    chain_id: string;
+}
+
+async function checkNodeos(fastify: FastifyInstance): Promise<ServiceResponse<NodeosService>> {
     const rpc = fastify.manager.nodeosJsonRPC;
     try {
         const results = await rpc.get_info();
@@ -36,14 +44,54 @@ async function checkNodeos(fastify: FastifyInstance) {
     }
 }
 
-async function checkElastic(fastify: FastifyInstance) {
+interface ESService {
+    first_indexed_block: number;
+    last_indexed_block: number;
+    total_indexed_blocks: number;
+    active_shards: string;
+    missing_blocks: number;
+    missing_pct: string;
+    head_offset: number;
+}
+
+interface ServiceResponse<T> {
+    service: string;
+    time: number;
+    status: any
+    service_data?: T;
+}
+
+async function checkElastic(fastify: FastifyInstance): Promise<ServiceResponse<ESService>> {
     try {
-        let esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
+        const esStatusCache = await fastify.redis.get(`${fastify.manager.chain}::es_status`);
+        let esStatus;
+        if (esStatusCache) {
+            esStatus = JSON.parse(esStatusCache);
+        } else {
+            esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
+            await fastify.redis.set(`${fastify.manager.chain}::es_status`, JSON.stringify(esStatus), 'EX', 30 * 60);
+        }
+        let firstIndexedBlock: number;
+        const fib = await fastify.redis.get(`${fastify.manager.chain}::fib`);
+        if (fib) {
+            firstIndexedBlock = parseInt(fib);
+        } else {
+            firstIndexedBlock = await getFirstIndexedBlock(fastify.elastic, fastify.manager.chain);
+            await fastify.redis.set(`${fastify.manager.chain}::fib`, firstIndexedBlock);
+        }
         let indexedBlocks = await getLastIndexedBlockWithTotalBlocks(fastify.elastic, fastify.manager.chain);
-        const data = {
-            last_indexed_block: indexedBlocks[0],
-            total_indexed_blocks: indexedBlocks[1],
-            active_shards: esStatus.body[0]['active_shards_percent']
+        const lastIndexedBlock = indexedBlocks[0];
+        const totalIndexed = indexedBlocks[1] - 1;
+        const missingCounter = (lastIndexedBlock - firstIndexedBlock) - totalIndexed;
+        const missingPct = (missingCounter * 100 / indexedBlocks[1]).toFixed(2) + "%";
+        const data: ESService = {
+            active_shards: esStatus.body[0]['active_shards_percent'],
+            head_offset: null,
+            first_indexed_block: firstIndexedBlock,
+            last_indexed_block: lastIndexedBlock,
+            total_indexed_blocks: totalIndexed,
+            missing_blocks: missingCounter,
+            missing_pct: missingPct
         };
         let stat = 'OK';
         esStatus.body.forEach(status => {
@@ -60,7 +108,7 @@ async function checkElastic(fastify: FastifyInstance) {
     }
 }
 
-function createHealth(name: string, status, data?: any) {
+function createHealth(name: string, status, data?: any): ServiceResponse<any> {
     let time = Date.now();
     return {
         service: name,
@@ -70,22 +118,27 @@ function createHealth(name: string, status, data?: any) {
     }
 }
 
-async function getHealthQuery(fastify: FastifyInstance, request: FastifyRequest) {
+async function getHealthQuery(fastify: FastifyInstance) {
     let response = {
         version: fastify.manager.current_version,
         version_hash: fastify.manager.getServerHash(),
         host: fastify.manager.config.api.server_name,
-        features: fastify.manager.config.features,
-        health: []
+        health: [],
+        features: fastify.manager.config.features
     };
-    response.health.push(await checkRabbit(fastify));
-    response.health.push(await checkNodeos(fastify));
-    response.health.push(await checkElastic(fastify));
+    response.health = await Promise.all([
+        checkRabbit(fastify),
+        checkNodeos(fastify),
+        checkElastic(fastify)
+    ]);
+    const es = response.health.find(value => value.service === 'Elasticsearch');
+    const nodeos = response.health.find(value => value.service === 'NodeosRPC');
+    es.service_data.head_offset = nodeos.service_data.head_block_num - es.service_data.last_indexed_block;
     return response;
 }
 
 export function healthHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getHealthQuery, fastify, request, route));
     }
 }

api/routes/v2/health/index.ts

@@ -1,18 +1,18 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {addApiRoute, getRouteName} from "../../../helpers/functions";
 import {healthHandler} from "./health";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
-        tags: ['status'],
-        summary: "API Service Health Report"
-    };
-    addApiRoute(
-        fastify,
-        'GET',
-        getRouteName(__filename),
-        healthHandler,
-        schema
-    );
-    next();
+	const schema: FastifySchema = {
+		tags: ['status'],
+		summary: "API Service Health Report"
+	};
+	addApiRoute(
+		fastify,
+		'GET',
+		getRouteName(__filename),
+		healthHandler,
+		schema
+	);
+	next();
 }

api/schemas/get_abi_snapshot.js

@@ -1,26 +0,0 @@
-exports.GET = {
-    description: 'fetch contract abi at specific block',
-    summary: 'fetch abi at specific block',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "contract": {
-                description: 'contract account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "block": {
-                description: 'target block',
-                type: 'integer',
-                minimum: 1
-            },
-            "fetch": {
-                description: 'should fetch the ABI',
-                type: 'boolean'
-            }
-        },
-        required: ["contract"]
-    }
-};

api/schemas/get_account.js

@@ -1,14 +0,0 @@
-exports.GET = {
-    description: 'get account data',
-    summary: 'get account summary',
-    tags: ['accounts', 'state'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'account name',
-                type: 'string'
-            }
-        }
-    }
-};

api/schemas/get_actions.js

@@ -1,132 +0,0 @@
-exports.GET = {
-    description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
-        ' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
-    summary: 'get root actions',
-    tags: ['actions', 'history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'notified account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "track": {
-                description: 'total results to track (count) [number or true]',
-                type: 'string'
-            },
-            "filter": {
-                description: 'code:name filter',
-                type: 'string',
-                minLength: 3
-            },
-            "skip": {
-                description: 'skip [n] actions (pagination)',
-                type: 'integer',
-                minimum: 0
-            },
-            "limit": {
-                description: 'limit of [n] actions per page',
-                type: 'integer',
-                minimum: 1
-            },
-            "sort": {
-                description: 'sort direction',
-                enum: ['desc', 'asc', '1', '-1'],
-                type: 'string'
-            },
-            "after": {
-                description: 'filter after specified date (ISO8601)',
-                type: 'string'
-            },
-            "before": {
-                description: 'filter before specified date (ISO8601)',
-                type: 'string'
-            },
-            "simple": {
-                description: 'simplified output mode',
-                type: 'boolean'
-            }
-        }
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "query_time": {
-                    type: "number"
-                },
-                "cached": {type: "boolean"},
-                "lib": {
-                    type: "number"
-                },
-                "total": {
-                    type: "object",
-                    properties: {
-                        "value": {type: "number"},
-                        "relation": {type: "string"}
-                    }
-                },
-                "simple_actions": {
-                    type: "array",
-                    items: {
-                        type: "object",
-                        properties: {
-                            "block": {type: "number"},
-                            "timestamp": {type: "string"},
-                            "irreversible": {type: "boolean"},
-                            "contract": {type: "string"},
-                            "action": {type: "string"},
-                            "actors": {type: "string"},
-                            "notified": {type: "string"},
-                            "transaction_id": {type: "string"},
-                            "data": {
-                                additionalProperties: true
-                            }
-                        }
-                    }
-                },
-                "actions": {
-                    type: "array",
-                    items: {
-                        type: 'object',
-                        properties: {
-                            "act": {
-                                type: 'object',
-                                properties: {
-                                    "account": {type: "string"},
-                                    "name": {type: "string"}
-                                },
-                                additionalProperties: true
-                            },
-                            "cpu_usage_us": {type: "number"},
-                            "net_usage_words": {type: "number"},
-                            "account_ram_deltas": {
-                                type: "array",
-                                items: {
-                                    type: "object",
-                                    properties: {
-                                        "account": {type: "string"},
-                                        "delta": {type: "number"}
-                                    },
-                                    additionalProperties: true
-                                }
-                            },
-                            "global_sequence": {type: "number"},
-                            "receiver": {type: 'string'},
-                            "@timestamp": {type: "string"},
-                            "block_num": {type: "number"},
-                            "producer": {type: "string"},
-                            "trx_id": {type: "string"},
-                            "parent": {type: "number"},
-                            "action_ordinal": {type: 'number'},
-                            "creator_action_ordinal": {type: 'number'},
-                            "notified": {type: "array", items: {type: "string"}}
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_blocks.js

@@ -1,20 +0,0 @@
-exports.GET = {
-    description: 'get block range',
-    summary: 'get block range',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "from": {
-                description: 'starting block',
-                type: 'integer',
-                minimum: 1
-            },
-            "to": {
-                description: 'last block',
-                type: 'integer',
-                minimum: 1
-            }
-        }
-    }
-};

api/schemas/get_created_accounts.js

@@ -1,44 +0,0 @@
-exports.GET = {
-    description: 'get all accounts created by one creator',
-    summary: 'get created accounts',
-    tags: ['accounts', 'history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'creator account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            }
-        }
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "query_time": {
-                    type: "number"
-                },
-                "total": {
-                    type: "object",
-                    properties: {
-                        "value": {type: "number"},
-                        "relation": {type: "string"}
-                    }
-                },
-                "accounts": {
-                    type: "array",
-                    items: {
-                        type: 'object',
-                        properties: {
-                            'name': {type: 'string'},
-                            'timestamp': {type: 'string'},
-                            'trx_id': {type: 'string'}
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_creator.js

@@ -1,41 +0,0 @@
-exports.GET = {
-    description: 'get account creator',
-    summary: 'get account creator',
-    tags: ['accounts', 'history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'created account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            }
-        }
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "account": {
-                    type: "string"
-                },
-                "creator": {
-                    type: "string"
-                },
-                "timestamp": {
-                    type: "string"
-                },
-                "block_num": {
-                    type: "integer"
-                },
-                "trx_id": {
-                    type: "string"
-                },
-                "indirect_creator": {
-                    type: "string"
-                }
-            }
-        }
-    }
-};

api/schemas/get_deltas.js

@@ -1,26 +0,0 @@
-exports.GET = {
-    description: 'get state deltas',
-    summary: 'get state deltas',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "code": {
-                description: 'contract account',
-                type: 'string'
-            },
-            "scope": {
-                description: 'table scope',
-                type: 'string'
-            },
-            "table": {
-                description: 'table name',
-                type: 'string'
-            },
-            "payer": {
-                description: 'payer account',
-                type: 'string'
-            }
-        }
-    }
-};

api/schemas/get_key_accounts.js

@@ -1,58 +0,0 @@
-module.exports = {
-    GET: {
-        description: 'get accounts by public key',
-        summary: 'get accounts by public key',
-        tags: ['accounts','state'],
-        querystring: {
-            type: 'object',
-            properties: {
-                "public_key": {
-                    description: 'public key',
-                    type: 'string'
-                },
-            },
-            required: ["public_key"]
-        },
-        response: {
-            200: {
-                type: 'object',
-                properties: {
-                    "account_names": {
-                        type: "array",
-                        items: {
-                            type: "string"
-                        }
-                    }
-                }
-            }
-        }
-    },
-    POST: {
-        description: 'get accounts by public key',
-        summary: 'get accounts by public key',
-        tags: ['accounts','state'],
-        body: {
-            type: 'object',
-            properties: {
-                "public_key": {
-                    description: 'public key',
-                    type: 'string'
-                },
-            },
-            required: ["public_key"]
-        },
-        response: {
-            200: {
-                type: 'object',
-                properties: {
-                    "account_names": {
-                        type: "array",
-                        items: {
-                            type: "string"
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_proposals.js

@@ -1,48 +0,0 @@
-exports.GET = {
-    description: 'get proposals',
-    summary: 'get proposals',
-    tags: ['state'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "proposer": {
-                description: 'filter by proposer',
-                type: 'string'
-            },
-            "proposal": {
-                description: 'filter by proposal name',
-                type: 'string'
-            },
-            "account": {
-                description: 'filter by either requested or provided account',
-                type: 'string'
-            },
-            "requested": {
-                description: 'filter by requested account',
-                type: 'string'
-            },
-            "provided": {
-                description: 'filter by provided account',
-                type: 'string'
-            },
-            "executed": {
-                description: 'filter by execution status',
-                type: 'boolean'
-            },
-            "track": {
-                description: 'total results to track (count) [number or true]',
-                type: 'string'
-            },
-            "skip": {
-                description: 'skip [n] actions (pagination)',
-                type: 'integer',
-                minimum: 0
-            },
-            "limit": {
-                description: 'limit of [n] actions per page',
-                type: 'integer',
-                minimum: 1
-            }
-        }
-    }
-};

api/schemas/get_tokens.js

@@ -1,37 +0,0 @@
-exports.GET = {
-    description: 'get tokens from account',
-    summary: 'get tokens from account',
-    tags: ['accounts', 'state'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'account',
-                type: 'string'
-            },
-        },
-        required: ["account"]
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "query_time": {type: "number"},
-                "cached": {type: "boolean"},
-                "account": {type: "string"},
-                "tokens": {
-                    type: "array",
-                    items: {
-                        type: "object",
-                        properties: {
-                            "symbol": {type: "string"},
-                            "precision": {type: "number"},
-                            "amount": {type: "number"},
-                            "contract": {type: "string"}
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_transacted_accounts.js

@@ -1,52 +0,0 @@
-exports.GET = {
-    description: 'get all account that interacted with the source account provided',
-    summary: 'get interactions based on transfers',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'source account',
-                type: 'string'
-            },
-            "symbol": {
-                description: 'token symbol',
-                type: 'string',
-                minLength: 1,
-                maxLength: 7
-            },
-            "contract": {
-                description: 'token contract',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "direction": {
-                description: 'search direction',
-                enum: ['in', 'out', 'both'],
-                type: 'string'
-            },
-            "min": {
-                description: 'minimum value',
-                type: 'number'
-            },
-            "max": {
-                description: 'maximum value',
-                type: 'number'
-            },
-            "limit": {
-                description: 'query limit',
-                type: 'number'
-            },
-            "after": {
-                description: 'filter after specified date (ISO8601) or block number',
-                type: 'string'
-            },
-            "before": {
-                description: 'filter before specified date (ISO8601) or block number',
-                type: 'string'
-            }
-        },
-        required: ["account", "direction"]
-    }
-};

api/schemas/get_transaction.js

@@ -1,15 +0,0 @@
-exports.GET = {
-    description: 'get all actions belonging to the same transaction',
-    summary: 'get transaction by id',
-    tags: ['transactions','history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "id": {
-                description: 'transaction id',
-                type: 'string'
-            }
-        },
-        required: ["id"]
-    }
-};

api/schemas/get_transfers.js

@@ -1,96 +0,0 @@
-exports.GET = {
-    description: 'get token transfers utilizing the eosio.token standard',
-    summary: 'get token transfers',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "from": {
-                description: 'source account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "to": {
-                description: 'destination account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "symbol": {
-                description: 'token symbol',
-                type: 'string',
-                minLength: 1,
-                maxLength: 7
-            },
-            "contract": {
-                description: 'token contract',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "skip": {
-                description: 'skip [n] actions (pagination)',
-                type: 'integer',
-                minimum: 0
-            },
-            "limit": {
-                description: 'limit of [n] actions per page',
-                type: 'integer',
-                minimum: 1
-            },
-            "after": {
-                description: 'filter after specified date (ISO8601)',
-                type: 'string',
-                format: 'date-time'
-            },
-            "before": {
-                description: 'filter before specified date (ISO8601)',
-                type: 'string',
-                format: 'date-time'
-            },
-        },
-        "anyOf": [
-            {required: ["from"]},
-            {required: ["to"]},
-            {required: ["symbol"]},
-            {required: ["contract"]}
-        ]
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "action_count": {
-                    type: "number"
-                },
-                "total_amount": {
-                    type: "number"
-                },
-                "actions": {
-                    type: "array",
-                    items: {
-                        type: 'object',
-                        properties: {
-                            "act": {
-                                type: 'object',
-                                properties: {
-                                    "account": {type: "string"},
-                                    "name": {type: "string"}
-                                },
-                                additionalProperties: true
-                            },
-                            "@timestamp": {type: "string"},
-                            "block_num": {type: "number"},
-                            "producer": {type: "string"},
-                            "trx_id": {type: "string"},
-                            "parent": {type: "number"},
-                            "global_sequence": {type: "number"},
-                            "notified": {type: "array", items: {type: "string"}}
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_voters.js

@@ -1,24 +0,0 @@
-exports.GET = {
-    description: 'get voters',
-    summary: 'get voters',
-    tags: ['accounts','state'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "producer": {
-                description: 'filter by voted producer (comma separated)',
-                type: 'string'
-            },
-            "skip": {
-                description: 'skip [n] actions (pagination)',
-                type: 'integer',
-                minimum: 0
-            },
-            "limit": {
-                description: 'limit of [n] actions per page',
-                type: 'integer',
-                minimum: 1
-            }
-        }
-    }
-};

api/schemas/index.js

@@ -1,16 +0,0 @@
-module.exports = {
-    getTransfersSchema: require('./get_transfers'),
-    getAbiSnapshotSchema: require('./get_abi_snapshot'),
-    getKeyAccountsSchema: require('./get_key_accounts'),
-    getActionsSchema: require('./get_actions'),
-    getTransactedAccountsSchema: require('./get_transacted_accounts'),
-    getTransactionSchema: require('./get_transaction'),
-    getCreatorSchema: require('./get_creator'),
-    getTokensSchema: require('./get_tokens'),
-    getDeltasSchema: require('./get_deltas'),
-    getCreatedAccountsSchema: require('./get_created_accounts'),
-    getVotersSchema: require('./get_voters'),
-    getProposalsSchema: require('./get_proposals'),
-    getBlocksSchema: require('./get_blocks'),
-    getAccountSchema: require('./get_account')
-};

api/server.ts

@@ -1,123 +1,170 @@
-import {hLog} from "../helpers/common_functions";
+import {hLog, waitUntilReady} from "../helpers/common_functions";
 import {ConfigurationModule} from "../modules/config";
 import {ConnectionManager} from "../connections/manager.class";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
-import {IncomingMessage, Server, ServerResponse} from "http";
-import * as Fastify from 'fastify';
-import * as Redis from 'ioredis';
+import IORedis from 'ioredis';
+import fastify from 'fastify'
 import {registerPlugins} from "./plugins";
 import {AddressInfo} from "net";
 import {registerRoutes} from "./routes";
 import {generateOpenApiConfig} from "./config/open_api";
-import {createWriteStream, existsSync, mkdirSync, writeFileSync} from "fs";
+import {createWriteStream, existsSync, mkdirSync, readFileSync} from "fs";
 import {SocketManager} from "./socketManager";
-import got from "got";
-import {join} from "path";
-import * as io from 'socket.io-client';
+import {HyperionModuleLoader} from "../modules/loader";
+import {extendedActions} from "./routes/v2-history/get_actions/definitions";
+import {io, Socket} from "socket.io-client";
+import {CacheManager} from "./helpers/cacheManager";
+
+import {bootstrap} from 'global-agent';
 
 class HyperionApiServer {
 
-    private conf: HyperionConfig;
-    private readonly manager: ConnectionManager;
-    private readonly fastify: Fastify.FastifyInstance<Server, IncomingMessage, ServerResponse>;
+    private hub: Socket;
+    private readonly fastify;
     private readonly chain: string;
-    socketManager: SocketManager;
+    private readonly conf: HyperionConfig;
+    private readonly manager: ConnectionManager;
+    private readonly cacheManager: CacheManager;
 
-    private hub: SocketIOClient.Socket;
+    socketManager: SocketManager;
+    mLoader: HyperionModuleLoader;
+    private pluginParams: any;
 
     constructor() {
+
+        const package_json = JSON.parse(readFileSync('./package.json').toString());
+        hLog(`--------- Hyperion API ${package_json.version} ---------`);
+
         const cm = new ConfigurationModule();
         this.conf = cm.config;
+
+        if (this.conf.settings.use_global_agent) {
+            bootstrap();
+        }
+
         this.chain = this.conf.settings.chain;
         process.title = `hyp-${this.chain}-api`;
         this.manager = new ConnectionManager(cm);
         this.manager.calculateServerHash();
-        this.manager.getHyperionVersion();
+        this.mLoader = new HyperionModuleLoader(cm);
+        this.cacheManager = new CacheManager(this.conf);
 
         if (!existsSync('./logs/' + this.chain)) {
             mkdirSync('./logs/' + this.chain, {recursive: true});
         }
 
         const logStream = createWriteStream('./logs/' + this.chain + '/api.access.log');
-        this.fastify = Fastify({
+
+        const loggerOpts = {
+            stream: logStream,
+            redact: ['req.headers.authorization'],
+            level: 'info',
+            prettyPrint: true,
+            serializers: {
+                res: (reply) => {
+                    return {
+                        statusCode: reply.statusCode
+                    };
+                },
+                req: (request) => {
+                    return {
+                        method: request.method,
+                        url: request.url,
+                        ip: request.headers['x-real-ip']
+                    }
+                }
+            }
+        };
+
+        this.fastify = fastify({
             ignoreTrailingSlash: false,
             trustProxy: true,
             pluginTimeout: 5000,
-            logger: this.conf.api.access_log ? {
-                stream: logStream,
-                redact: ['req.headers.authorization'],
-                level: 'info',
-                serializers: {
-                    res: (res) => {
-                        return {
-                            status: res.statusCode
-                        };
-                    },
-                    req: (req) => {
-                        return {
-                            method: req.method,
-                            url: req.url,
-                            ip: req.headers['x-real-ip']
-                        }
-                    }
-                }
-            } : false
+            logger: this.conf.api.access_log ? loggerOpts : false
         });
+
+        this.fastify.decorate('cacheManager', this.cacheManager);
+
         this.fastify.decorate('manager', this.manager);
 
-        if (this.conf.api.chain_api && this.conf.api.chain_api !== "") {
-            this.fastify.decorate('chain_api', this.conf.api.chain_api);
-        } else {
-            this.fastify.decorate('chain_api', this.manager.conn.chains[this.chain].http);
+        // import get_actions query params from custom modules
+        const extendedActionsSet: Set<string> = new Set([...extendedActions]);
+        for (const qPrefix of this.mLoader.extendedActions) {
+            extendedActionsSet.add(qPrefix);
         }
+        this.fastify.decorate('allowedActionQueryParamSet', extendedActionsSet);
 
-        if (this.conf.api.push_api && this.conf.api.push_api !== "") {
+        // define chain api url for /v1/chain/ redirects
+        let chainApiUrl: string = this.conf.api.chain_api;
+        if (chainApiUrl === null || chainApiUrl === "") {
+            chainApiUrl = this.manager.conn.chains[this.chain].http;
+        }
+        this.fastify.decorate('chain_api', chainApiUrl);
+
+        // define optional push api url for /v1/chain/push_transaction
+        if (this.conf.api.push_api) {
             this.fastify.decorate('push_api', this.conf.api.push_api);
         }
 
-        console.log(`Chain API URL: ${this.fastify.chain_api}`);
-        console.log(`Push API URL: ${this.fastify.push_api}`);
+        hLog(`Chain API URL: "${this.fastify.chain_api}" | Push API URL: "${this.fastify.push_api}"`);
 
-        const ioRedisClient = new Redis(this.manager.conn.redis);
-        const api_rate_limit = {
-            max: 1000,
-            whitelist: ['127.0.0.1'],
-            timeWindow: '1 minute',
-            redis: ioRedisClient
-        };
+        const ioRedisClient = new IORedis(this.manager.conn.redis);
+
+        const pluginParams = {
+            fastify_elasticsearch: {
+                client: this.manager.elasticsearchClient
+            },
+            fastify_redis: this.manager.conn.redis,
+            fastify_eosjs: this.manager,
+            chain_id: '',
+        } as any;
+
+        if (!this.conf.api.disable_rate_limit) {
+            let rateLimiterWhitelist = ['127.0.0.1'];
+            if (this.conf.api.rate_limit_allow && this.conf.api.rate_limit_allow.length > 0) {
+                const tempSet = new Set<string>(['127.0.0.1', ...this.conf.api.rate_limit_allow]);
+                rateLimiterWhitelist = [...tempSet];
+            }
+            let rateLimiterRPM = 1000;
+            if (this.conf.api.rate_limit_rpm) {
+                rateLimiterRPM = this.conf.api.rate_limit_rpm;
+            }
+            pluginParams.fastify_rate_limit = {
+                max: rateLimiterRPM,
+                allowList: rateLimiterWhitelist,
+                timeWindow: '1 minute',
+                redis: ioRedisClient
+            }
+        }
 
         if (this.conf.features.streaming.enable) {
             this.activateStreaming();
         }
 
-        registerPlugins(this.fastify, {
-            fastify_elasticsearch: {
-                client: this.manager.elasticsearchClient
-            },
-            fastify_oas: generateOpenApiConfig(this.manager.config),
-            fastify_rate_limit: api_rate_limit,
-            fastify_redis: this.manager.conn.redis,
-            fastify_eosjs: this.manager,
-        });
+        const docsConfig = generateOpenApiConfig(this.manager.config);
+        if (docsConfig) {
+            pluginParams.fastify_swagger = docsConfig;
+        }
 
-        this.addGenericTypeParsing();
+        this.pluginParams = pluginParams;
     }
 
     activateStreaming() {
-        console.log('Importing stream module');
+        console.log('Importing stream module...');
         import('./socketManager').then((mod) => {
             const connOpts = this.manager.conn.chains[this.chain];
-
             let _port = 57200;
             if (connOpts.WS_ROUTER_PORT) {
                 _port = connOpts.WS_ROUTER_PORT;
             }
-
             let _host = "127.0.0.1";
             if (connOpts.WS_ROUTER_HOST) {
                 _host = connOpts.WS_ROUTER_HOST;
             }
-
+            if (_host === "0.0.0.0") {
+                hLog(`[ERROR] WS Router Host is set to 0.0.0.0, please use a fixed IP address instead. Can't start streaming.`);
+                return;
+            }
             this.socketManager = new mod.SocketManager(
                 this.fastify,
                 `http://${_host}:${_port}`,
@@ -128,15 +175,15 @@ class HyperionApiServer {
     }
 
     private addGenericTypeParsing() {
-        this.fastify.addContentTypeParser('*', (req, done) => {
+        this.fastify.addContentTypeParser('*', (request, payload, done) => {
             let data = '';
-            req.on('data', chunk => {
+            payload.on('data', chunk => {
                 data += chunk;
             });
-            req.on('end', () => {
+            payload.on('end', () => {
                 done(null, data);
             });
-            req.on('error', (err) => {
+            payload.on('error', (err) => {
                 console.log('---- Content Parsing Error -----');
                 console.log(err);
             });
@@ -145,15 +192,65 @@ class HyperionApiServer {
 
     async init() {
 
-        await this.fetchChainLogo();
+        const rpc = this.manager.nodeosJsonRPC;
+        await waitUntilReady(async () => {
+            try {
+                const chain_data = await rpc.get_info();
+                if (chain_data && chain_data.chain_id) {
+                    this.pluginParams.chain_id = chain_data.chain_id;
+                    return true;
+                } else {
+                    return false;
+                }
+            } catch (e) {
+                hLog(e.message);
+                return false;
+            }
+        }, 20, 5000, () => {
+            hLog('Failed to validate chain api!');
+            process.exit(1);
+        });
+        hLog('Chain API validated!');
+
+        // Wait for Elasticsearch availability
+        await waitUntilReady(async () => {
+            try {
+                const esInfo = await this.manager.elasticsearchClient.info();
+                hLog(`Elasticsearch: ${esInfo.body.version.number} | Lucene: ${esInfo.body.version.lucene_version}`);
+                return true;
+            } catch (e) {
+                console.log(e.message);
+                return false;
+            }
+        }, 10, 5000, () => {
+            hLog('Failed to check elasticsearch version!');
+            process.exit();
+        });
+
+        hLog('Elasticsearch validated!');
+        hLog('Registering plugins...');
+
+        registerPlugins(this.fastify, this.pluginParams);
+        this.addGenericTypeParsing();
+
+        await this.mLoader.init();
+
+        // add custom plugin routes
+        for (const plugin of this.mLoader.plugins) {
+            if (plugin.hasApiRoutes) {
+                hLog(`Adding routes for plugin: ${plugin.internalPluginName}`);
+                plugin.addRoutes(this.fastify);
+                plugin.chainName = this.chain;
+            }
+        }
 
         registerRoutes(this.fastify);
 
+        // register documentation when ready
         this.fastify.ready().then(async () => {
-            await this.fastify.oas();
-            console.log(this.chain + ' api ready!');
+            await this.fastify.swagger();
         }, (err) => {
-            console.log('an error happened', err)
+            hLog('an error happened', err)
         });
 
         try {
@@ -161,28 +258,14 @@ class HyperionApiServer {
                 host: this.conf.api.server_addr,
                 port: this.conf.api.server_port
             });
-            console.log(`server listening on ${(this.fastify.server.address() as AddressInfo).port}`);
+            hLog(`${this.chain} hyperion api ready and listening on port ${(this.fastify.server.address() as AddressInfo).port}`);
             this.startHyperionHub();
         } catch (err) {
-            console.log(err);
+            hLog(err);
             process.exit(1)
         }
     }
 
-    async fetchChainLogo() {
-        try {
-            if (this.conf.api.chain_logo_url && this.conf.api.enable_explorer) {
-                console.log(`Downloading chain logo from ${this.conf.api.chain_logo_url}...`);
-                const chainLogo = await got(this.conf.api.chain_logo_url);
-                const path = join(__dirname, '..', 'hyperion-explorer', 'dist', 'assets', this.chain + '_logo.png');
-                writeFileSync(path, chainLogo.rawBody);
-                this.conf.api.chain_logo_url = 'https://' + this.conf.api.server_name + '/v2/explore/assets/' + this.chain + '_logo.png';
-            }
-        } catch (e) {
-            console.log(e);
-        }
-    }
-
     startHyperionHub() {
         if (this.conf.hub) {
             const url = this.conf.hub.inform_url;
@@ -190,17 +273,13 @@ class HyperionApiServer {
             this.hub = io(url, {
                 query: {
                     key: this.conf.hub.publisher_key,
-                    client_mode: false
+                    client_mode: 'false'
                 }
             });
             this.hub.on('connect', () => {
                 hLog(`Hyperion Hub connected!`);
                 this.emitHubApiUpdate();
             });
-            // this.hub.on('reconnect', () => {
-            //     hLog(`Reconnecting...`);
-            //     this.emitHubApiUpdate();
-            // });
         }
     }
 
@@ -211,7 +290,7 @@ class HyperionApiServer {
             location: this.conf.hub.location,
             chainId: this.manager.conn.chains[this.chain].chain_id,
             providerName: this.conf.api.provider_name,
-            explorerEnabled: this.conf.api.enable_explorer,
+            explorerEnabled: this.conf.plugins.explorer?.enabled,
             providerUrl: this.conf.api.provider_url,
             providerLogo: this.conf.api.provider_logo,
             chainLogo: this.conf.api.chain_logo_url,

api/socketManager.ts

@@ -1,7 +1,12 @@
-import {checkFilter, hLog} from '../helpers/common_functions';
-import * as sockets from 'socket.io';
-import * as IOClient from 'socket.io-client';
+import {hLog, sleep} from '../helpers/common_functions';
+import {Server, Socket} from 'socket.io';
+import {createAdapter} from 'socket.io-redis';
+import {io} from 'socket.io-client';
 import {FastifyInstance} from "fastify";
+import IORedis from "ioredis";
+import {App, TemplatedApp} from 'uWebSockets.js';
+import {streamPastActions, streamPastDeltas} from "./helpers/functions";
+import {randomUUID} from "crypto";
 
 export interface StreamDeltasRequest {
     code: string;
@@ -26,234 +31,42 @@ export interface StreamActionsRequest {
     read_until: number | string;
 }
 
-async function addBlockRangeOpts(data, search_body, fastify: FastifyInstance) {
-
-    let timeRange;
-    let blockRange;
-    let head;
-
-    if (typeof data['start_from'] === 'string' && data['start_from'] !== '') {
-        if (!timeRange) {
-            timeRange = {"@timestamp": {}};
-        }
-        timeRange["@timestamp"]['gte'] = data['start_from'];
-    }
-
-    if (typeof data['read_until'] === 'string' && data['read_until'] !== '') {
-        if (!timeRange) {
-            timeRange = {"@timestamp": {}};
-        }
-        timeRange["@timestamp"]['lte'] = data['read_until'];
-    }
-
-    if (typeof data['start_from'] === 'number' && data['start_from'] !== 0) {
-        if (!blockRange) {
-            blockRange = {"block_num": {}};
-        }
-        if (data['start_from'] < 0) {
-            if (!head) {
-                head = (await fastify.eosjs.rpc.get_info()).head_block_num;
-            }
-            blockRange["block_num"]['gte'] = head + data['start_from'];
-        } else {
-            blockRange["block_num"]['gte'] = data['start_from'];
-        }
-    }
-
-    if (typeof data['read_until'] === 'number' && data['read_until'] !== 0) {
-        if (!blockRange) {
-            blockRange = {"block_num": {}};
-        }
-        if (data['read_until'] < 0) {
-            if (!head) {
-                head = (await fastify.eosjs.rpc.get_info()).head_block_num;
-            }
-            blockRange["block_num"]['lte'] = head + data['read_until'];
-        } else {
-            blockRange["block_num"]['lte'] = data['read_until'];
-        }
-    }
-
-    if (timeRange) {
-        search_body.query.bool.must.push({
-            range: timeRange,
-        });
-    }
-    if (blockRange) {
-        search_body.query.bool.must.push({
-            range: blockRange,
-        });
-    }
-}
-
-function addTermMatch(data, search_body, field) {
-    if (data[field] !== '*' && data[field] !== '') {
-        const termQuery = {};
-        termQuery[field] = data[field];
-        search_body.query.bool.must.push({'term': termQuery});
-    }
-}
-
-const deltaQueryFields = ['code', 'table', 'scope', 'payer'];
-
-async function streamPastDeltas(fastify: FastifyInstance, socket, data) {
-    const search_body = {
-        query: {bool: {must: []}},
-        sort: {block_num: 'asc'},
-    };
-    await addBlockRangeOpts(data, search_body, fastify);
-    deltaQueryFields.forEach(f => {
-        addTermMatch(data, search_body, f);
-    });
-    const responseQueue = [];
-    let counter = 0;
-    const init_response = await fastify.elastic.search({
-        index: fastify.manager.chain + '-delta-*',
-        scroll: '30s',
-        size: 20,
-        body: search_body,
-    });
-    responseQueue.push(init_response);
-    while (responseQueue.length) {
-        const {body} = responseQueue.shift();
-        counter += body['hits']['hits'].length;
-        if (socket.connected) {
-            socket.emit('message', {
-                type: 'delta_trace',
-                mode: 'history',
-                messages: body['hits']['hits'].map(doc => doc._source),
-            });
-        } else {
-            console.log('LOST CLIENT');
-            break;
-        }
-        if (body['hits'].total.value === counter) {
-            console.log(`${counter} past deltas streamed to ${socket.id}`);
-            break;
-        }
-
-        const next_response = await fastify.elastic.scroll({
-            body: {
-                scroll_id: body['_scroll_id'],
-                scroll: '30s'
-            }
-        });
-        responseQueue.push(next_response);
-    }
-}
-
-async function streamPastActions(fastify: FastifyInstance, socket, data) {
-    const search_body = {
-        query: {bool: {must: []}},
-        sort: {global_sequence: 'asc'},
-    };
-
-    await addBlockRangeOpts(data, search_body, fastify);
-
-    if (data.account !== '') {
-        search_body.query.bool.must.push({
-            bool: {
-                should: [
-                    {term: {'notified': data.account}},
-                    {term: {'act.authorization.actor': data.account}},
-                ],
-            },
-        });
-    }
-
-    if (data.contract !== '*' && data.contract !== '') {
-        search_body.query.bool.must.push({'term': {'act.account': data.contract}});
-    }
-
-    if (data.action !== '*' && data.action !== '') {
-        search_body.query.bool.must.push({'term': {'act.name': data.action}});
-    }
-
-    const onDemandFilters = [];
-    if (data.filters.length > 0) {
-        data.filters.forEach(f => {
-            if (f.field && f.value) {
-                if (f.field.startsWith('@') && !f.field.startsWith('act.data')) {
-                    const _q = {};
-                    _q[f.field] = f.value;
-                    search_body.query.bool.must.push({'term': _q});
-                } else {
-                    onDemandFilters.push(f);
-                }
-            }
-        });
-    }
-
-    const responseQueue = [];
-    let counter = 0;
-
-    const init_response = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        scroll: '30s',
-        size: 20,
-        body: search_body,
-    });
-    responseQueue.push(init_response);
-    while (responseQueue.length) {
-        const {body} = responseQueue.shift();
-        const enqueuedMessages = [];
-        counter += body['hits']['hits'].length;
-        for (const doc of body['hits']['hits']) {
-            let allow = false;
-            if (onDemandFilters.length > 0) {
-                allow = onDemandFilters.every(filter => {
-                    return checkFilter(filter, doc._source);
-                });
-            } else {
-                allow = true;
-            }
-            if (allow) {
-                enqueuedMessages.push(doc._source);
-            }
-        }
-        if (socket.connected) {
-            socket.emit('message', {type: 'action_trace', mode: 'history', messages: enqueuedMessages});
-        } else {
-            console.log('LOST CLIENT');
-            break;
-        }
-        if (body['hits'].total.value === counter) {
-            console.log(`${counter} past actions streamed to ${socket.id}`);
-            break;
-        }
-        if (init_response.body.hits.total.value < 1000) {
-            const next_response = await fastify.elastic.scroll({
-                body: {scroll_id: body['_scroll_id'], scroll: '30s'}
-            });
-            responseQueue.push(next_response);
-        } else {
-            console.log('Request too large!');
-            socket.emit('message', {type: 'action_trace', mode: 'history', messages: []});
-        }
-    }
-}
 
 export class SocketManager {
 
-    private io;
+    private io: Server;
     private relay;
     relay_restored = true;
     relay_down = false;
     private readonly url;
     private readonly server: FastifyInstance;
+    private readonly uwsApp: TemplatedApp;
+    private readonly chainId: string;
+    private currentBlockNum: number;
 
-    constructor(fastify: FastifyInstance, url, redisOpts) {
+    constructor(fastify: FastifyInstance, url, redisOptions) {
         this.server = fastify;
         this.url = url;
+        this.uwsApp = App({});
 
-        this.io = sockets(fastify.server, {
-            transports: ['websocket', 'polling'],
+        // WS Server for public access
+        this.io = new Server({
+            transports: ['websocket'],
+            path: '/stream'
         });
 
-        this.io.on('connection', (socket) => {
+        this.io.attachApp(this.uwsApp);
+
+        this.chainId = this.server.manager.conn.chains[this.server.manager.chain].chain_id
+        hLog(`[SocketManager] chain_id: ${this.chainId}`);
+        const pubClient = new IORedis(redisOptions);
+        const subClient = pubClient.duplicate();
+        this.io.adapter(createAdapter({pubClient, subClient, key: this.chainId}));
+
+        this.io.on('connection', (socket: Socket) => {
 
             if (socket.handshake.headers['x-forwarded-for']) {
-                console.log(`[socket] ${socket.id} connected via ${socket.handshake.headers['x-forwarded-for']}`);
+                hLog(`[socket] ${socket.id} connected via ${socket.handshake.headers['x-forwarded-for']}`);
             }
 
             socket.emit('message', {
@@ -264,17 +77,49 @@ export class SocketManager {
             if (this.relay) {
                 this.relay.emit('event', {
                     type: 'client_count',
-                    counter: Object.keys(this.io.sockets.connected).length,
+                    counter: this.io.sockets.sockets.size,
                 });
             }
 
             socket.on('delta_stream_request', async (data: StreamDeltasRequest, callback) => {
                 if (typeof callback === 'function' && data) {
                     try {
+                        // generate random uuid
+                        socket.data.reqUUID = randomUUID();
+                        const lastHistoryBlock = await new Promise<number>((resolve) => {
+                            // start sending realtime data
+                            this.emitToRelay(data, 'delta_request', socket, (emissionResult) => {
+                                callback(emissionResult);
+                                resolve(emissionResult.currentBlockNum);
+                            });
+                        });
+                        // push history data
                         if (data.start_from) {
-                            await streamPastDeltas(this.server, socket, data);
+                            data.read_until = lastHistoryBlock;
+                            console.log('Performing primary scroll request...');
+                            let ltb = 0;
+                            const hStreamResult = await streamPastDeltas(this.server, socket, data);
+                            if (hStreamResult.status === false) {
+                                return;
+                            } else {
+                                ltb = hStreamResult.lastTransmittedBlock;
+                                let attempts = 0;
+                                await sleep(500);
+                                while (ltb > 0 && lastHistoryBlock > ltb && attempts < 3) {
+                                    attempts++;
+                                    console.log(`Performing fill request from ${ltb}...`);
+                                    data.start_from = hStreamResult.lastTransmittedBlock + 1;
+                                    data.read_until = lastHistoryBlock;
+                                    const r = await streamPastDeltas(this.server, socket, data);
+                                    if (r.status === false) {
+                                        console.log(r);
+                                        return;
+                                    } else {
+                                        ltb = r.lastTransmittedBlock;
+                                    }
+                                }
+                            }
                         }
-                        this.emitToRelay(data, 'delta_request', socket, callback);
                     } catch (e) {
                         console.log(e);
                     }
@@ -284,10 +129,42 @@ export class SocketManager {
             socket.on('action_stream_request', async (data: StreamActionsRequest, callback) => {
                 if (typeof callback === 'function' && data) {
                     try {
+                        // generate random uuid
+                        socket.data.reqUUID = randomUUID();
+                        const lastHistoryBlock = await new Promise<number>((resolve) => {
+                            // start sending realtime data
+                            this.emitToRelay(data, 'action_request', socket, (emissionResult) => {
+                                callback(emissionResult);
+                                resolve(emissionResult.currentBlockNum);
+                            });
+                        });
+                        // push history data
                         if (data.start_from) {
-                            await streamPastActions(this.server, socket, data);
+                            data.read_until = lastHistoryBlock;
+                            console.log('Performing primary scroll request...');
+                            let ltb = 0;
+                            const hStreamResult = await streamPastActions(this.server, socket, data);
+                            if (hStreamResult.status === false) {
+                                return;
+                            } else {
+                                ltb = hStreamResult.lastTransmittedBlock;
+                                let attempts = 0;
+                                await sleep(500);
+                                while (ltb > 0 && lastHistoryBlock > ltb && attempts < 3) {
+                                    attempts++;
+                                    console.log(`Performing fill request from ${hStreamResult.lastTransmittedBlock}...`);
+                                    data.start_from = hStreamResult.lastTransmittedBlock + 1;
+                                    data.read_until = lastHistoryBlock;
+                                    const r = await streamPastActions(this.server, socket, data);
+                                    if (r.status === false) {
+                                        console.log(r);
+                                        return;
+                                    } else {
+                                        ltb = r.lastTransmittedBlock;
+                                    }
+                                }
+                            }
                         }
-                        this.emitToRelay(data, 'action_request', socket, callback);
                     } catch (e) {
                         console.log(e);
                     }
@@ -295,7 +172,7 @@ export class SocketManager {
             });
 
             socket.on('disconnect', (reason) => {
-                console.log(`[socket] ${socket.id} disconnected - ${reason}`);
+                hLog(`[socket] ${socket.id} disconnected - ${reason}`);
                 this.relay.emit('event', {
                     type: 'client_disconnected',
                     id: socket.id,
@@ -303,16 +180,28 @@ export class SocketManager {
                 });
             });
         });
-        console.log('Websocket manager loaded!');
+
+        try {
+            const port = this.server.manager.config.api.stream_port || 1234;
+            this.uwsApp.listen(port, () => {
+                hLog(`Socket.IO via uWS started on port ${port}`);
+            });
+        } catch (e) {
+            hLog(e.message);
+        }
+
+        hLog('Websocket manager loaded!');
     }
 
+    /*
+    WS Relay will connect to the indexer
+     */
     startRelay() {
-        console.log(`starting relay - ${this.url}`);
-
-        this.relay = IOClient(this.url, {path: '/router'});
+        hLog(`starting relay - ${this.url}`);
+        this.relay = io(this.url, {path: '/router'});
 
         this.relay.on('connect', () => {
-            console.log('Relay Connected!');
+            hLog('Relay Connected!');
             if (this.relay_down) {
                 this.relay_restored = true;
                 this.relay_down = false;
@@ -321,7 +210,7 @@ export class SocketManager {
         });
 
         this.relay.on('disconnect', () => {
-            console.log('Relay disconnected!');
+            hLog('Relay disconnected!');
             this.io.emit('status', 'relay_down');
             this.relay_down = true;
             this.relay_restored = false;
@@ -335,26 +224,49 @@ export class SocketManager {
             this.emitToClient(traceData, 'action_trace');
         });
 
-        // Relay LIB info to clients;
-        this.relay.on('lib_update', (data) => {
-            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-                this.io.emit('lib_update', data);
+        this.relay.on('block', (blockData) => {
+            try {
+                // const decodedBlock = JSON.parse(blockData.content.toString());
+                // console.log(blockData.serverTime, blockData.blockNum, decodedBlock);
+                this.currentBlockNum = blockData.blockNum;
+            } catch (e) {
+                hLog(`Failed to decode incoming live block ${blockData.blockNum}: ${e.message}`);
             }
         });
 
-        // Relay LIB info to clients;
-        this.relay.on('fork_event', (data) => {
-            console.log(data);
-            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-                this.io.emit('fork_event', data);
+        this.addRelayForwarding('lib_update');
+
+        this.addRelayForwarding('fork_event');
+
+        // // Relay LIB info to clients;
+        // this.relay.on('lib_update', (data) => {
+        //     if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+        //         this.io.emit('lib_update', data);
+        //     }
+        // });
+        //
+        // // Relay fork info to clients;
+        // this.relay.on('fork_event', (data) => {
+        //     if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+        //         this.io.emit('fork_event', data);
+        //     }
+        // });
+    }
+
+    // Relay events to clients
+    addRelayForwarding(event: string) {
+        this.relay.on(event, (data: any) => {
+            if (data.chain_id && this.chainId === data.chain_id) {
+                this.io.emit(event, data);
             }
         });
     }
 
     emitToClient(traceData, type) {
-        if (this.io.sockets.connected[traceData.client]) {
-            this.io.sockets.connected[traceData.client].emit('message', {
+        if (this.io.sockets.sockets.has(traceData.client)) {
+            this.io.sockets.sockets.get(traceData.client).emit('message', {
                 type: type,
+                reqUUID: traceData.req,
                 mode: 'live',
                 message: traceData.message,
             });
@@ -364,10 +276,13 @@ export class SocketManager {
     emitToRelay(data, type, socket, callback) {
         if (this.relay.connected) {
             this.relay.emit('event', {
+                reqUUID: socket.data.reqUUID,
                 type: type,
                 client_socket: socket.id,
                 request: data,
             }, (response) => {
+                response['reqUUID'] = socket.data.reqUUID;
+                response['currentBlockNum'] = this.currentBlockNum;
                 callback(response);
             });
         } else {

auto_setup.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-git clone https://github.com/eosrio/hyperion-history-api.git  || exit
-cd hyperion-history-api || exit
-bash ./install_env.sh

chains/example.config.json

@@ -1,8 +1,14 @@
 {
   "api": {
+    "enabled": true,
+    "pm2_scaling": 1,
+    "node_max_old_space_size": 1024,
     "chain_name": "EXAMPLE Chain",
     "server_addr": "127.0.0.1",
     "server_port": 7000,
+    "stream_port": 1234,
+    "stream_scroll_limit": -1,
+    "stream_scroll_batch": 500,
     "server_name": "127.0.0.1:7000",
     "provider_name": "Example Provider",
     "provider_url": "https://example.com",
@@ -19,19 +25,48 @@
       "get_trx_actions": 200
     },
     "access_log": false,
-    "enable_explorer": false,
     "chain_api_error_log": false,
     "custom_core_token": "",
     "enable_export_action": false,
+    "disable_rate_limit": false,
+    "rate_limit_rpm": 1000,
+    "rate_limit_allow": [],
     "disable_tx_cache": false,
-    "tx_cache_expiration_sec": 3600
+    "tx_cache_expiration_sec": 3600,
+    "v1_chain_cache": [
+      {
+        "path": "get_block",
+        "ttl": 3000
+      },
+      {
+        "path": "get_info",
+        "ttl": 500
+      }
+    ]
+  },
+  "indexer": {
+    "enabled": true,
+    "node_max_old_space_size": 4096,
+    "start_on": 0,
+    "stop_on": 0,
+    "rewrite": false,
+    "purge_queues": false,
+    "live_reader": false,
+    "live_only_mode": false,
+    "abi_scan_mode": true,
+    "fetch_block": true,
+    "fetch_traces": true,
+    "disable_reading": false,
+    "disable_indexing": false,
+    "process_deltas": true,
+    "disable_delta_rm": true
   },
   "settings": {
     "preview": false,
     "chain": "eos",
     "eosio_alias": "eosio",
-    "parser": "1.8",
-    "auto_stop": 300,
+    "parser": "3.2",
+    "auto_stop": 0,
     "index_version": "v1",
     "debug": false,
     "bp_logs": false,
@@ -39,12 +74,14 @@
     "ipc_debug_rate": 60000,
     "allow_custom_abi": false,
     "rate_monitoring": true,
-    "max_ws_payload_kb": 256,
+    "max_ws_payload_mb": 256,
     "ds_profiling": false,
     "auto_mode_switch": false,
     "hot_warm_policy": false,
     "custom_policy": "",
-    "bypass_index_map": false
+    "use_global_agent": false,
+    "index_partition_size": 10000000,
+    "es_replicas": 0
   },
   "blacklists": {
     "actions": [],
@@ -63,29 +100,16 @@
     "ds_pool_size": 1,
     "indexing_queues": 1,
     "ad_idx_queues": 1,
+    "dyn_idx_queues": 1,
     "max_autoscale": 4,
     "batch_size": 5000,
     "resume_trigger": 5000,
     "auto_scale_trigger": 20000,
     "block_queue_limit": 10000,
     "max_queue_limit": 100000,
-    "routing_mode": "heatmap",
+    "routing_mode": "round_robin",
     "polling_interval": 10000
   },
-  "indexer": {
-    "start_on": 0,
-    "stop_on": 0,
-    "rewrite": false,
-    "purge_queues": false,
-    "live_reader": false,
-    "live_only_mode": false,
-    "abi_scan_mode": true,
-    "fetch_block": true,
-    "fetch_traces": true,
-    "disable_reading": false,
-    "disable_indexing": false,
-    "process_deltas": true
-  },
   "features": {
     "streaming": {
       "enable": false,
@@ -98,7 +122,7 @@
       "voters": true
     },
     "index_deltas": true,
-    "index_transfer_memo": false,
+    "index_transfer_memo": true,
     "index_all_deltas": true,
     "deferred_trx": false,
     "failed_trx": false,
@@ -109,5 +133,6 @@
     "read": 50,
     "block": 100,
     "index": 500
-  }
+  },
+  "plugins": {}
 }

connections/amqp.ts

@@ -9,33 +9,32 @@ export async function createConnection(config): Promise<Connection> {
 		debugLog("[AMQP] connection established");
 		return conn;
 	} catch (e) {
-		console.log("[AMQP] failed to connect!");
-		console.log(e.message);
+		hLog("[AMQP] failed to connect!");
+		hLog(e.message);
 		await new Promise(resolve => setTimeout(resolve, 5000));
 		return await createConnection(config);
 	}
 }
 
 export function getAmpqUrl(config): string {
-	let frameMaxValue = '0x10000';
-	if (config.frameMax) {
-		frameMaxValue = config.frameMax;
-	}
-	const u = encodeURIComponent(config.user);
-	const p = encodeURIComponent(config.pass);
-	const v = encodeURIComponent(config.vhost)
-	return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
+    let frameMaxValue = '0x10000';
+    if (config.frameMax) {
+        frameMaxValue = config.frameMax;
+    }
+    const u = encodeURIComponent(config.user);
+    const p = encodeURIComponent(config.pass);
+    const v = encodeURIComponent(config.vhost);
+    return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
 }
 
-
 async function createChannels(connection) {
 	try {
 		const channel = await connection.createChannel();
 		const confirmChannel = await connection.createConfirmChannel();
 		return [channel, confirmChannel];
 	} catch (e) {
-		console.log("[AMQP] failed to create channels");
-		console.error(e);
+		hLog("[AMQP] failed to create channels");
+		hLog(e);
 		return null;
 	}
 }
@@ -70,7 +69,7 @@ export async function amqpConnect(onReconnect, config, onClose) {
 export async function checkQueueSize(q_name, config) {
 	try {
 		const v = encodeURIComponent(config.vhost);
-		const apiUrl = `http://${config.api}/api/queues/${v}/${encodeURIComponent(q_name)}`;
+		const apiUrl = `${config.protocol}://${config.api}/api/queues/${v}/${encodeURIComponent(q_name)}`;
 		const opts = {
 			username: config.user,
 			password: config.pass
@@ -78,13 +77,13 @@ export async function checkQueueSize(q_name, config) {
 		const data = await got.get(apiUrl, opts).json() as any;
 		return data.messages;
 	} catch (e) {
-		hLog('[WARNING] Checking queue size failed, HTTP API is not ready!');
-		if (e instanceof HTTPError) {
-			if(e.response.body) {
+		hLog(`[WARNING] Checking queue size failed! - ${e.message}`);
+		if (e.response && e.response.body) {
+			if (e instanceof HTTPError) {
 				hLog(e.response.body);
+			} else {
+				hLog(JSON.stringify(e.response.body, null, 2));
 			}
-		} else {
-			hLog(JSON.stringify(e, null, 2));
 		}
 		return 0;
 	}

connections/manager.class.ts

@@ -1,5 +1,5 @@
 import {ConfigurationModule} from "../modules/config";
-import {JsonRpc} from "eosjs/dist";
+import {JsonRpc} from "eosjs";
 import got from "got";
 import {Client} from '@elastic/elasticsearch'
 import {HyperionConnections} from "../interfaces/hyperionConnections";
@@ -19,12 +19,15 @@ export class ConnectionManager {
     last_commit_hash: string;
     current_version: string;
 
-    private esIngestClients: Client[];
-    private esIngestClient: Client;
+    esIngestClients: Client[];
+    esIngestClient: Client;
 
     constructor(private cm: ConfigurationModule) {
         this.config = cm.config;
         this.conn = cm.connections;
+        if (!this.conn.amqp.protocol) {
+            this.conn.amqp.protocol = 'http';
+        }
         this.chain = this.config.settings.chain;
         this.esIngestClients = [];
         this.prepareESClient();
@@ -39,7 +42,8 @@ export class ConnectionManager {
     async purgeQueues() {
         hLog(`Purging all ${this.chain} queues!`);
         const apiUrl = `http://${this.conn.amqp.api}`;
-        const getAllQueuesFromVHost = apiUrl + `/api/queues/%2F${this.conn.amqp.vhost}`;
+        const vHost = encodeURIComponent(this.conn.amqp.vhost);
+        const getAllQueuesFromVHost = apiUrl + `/api/queues/${vHost}`;
         const opts = {
             username: this.conn.amqp.user,
             password: this.conn.amqp.pass
@@ -61,7 +65,7 @@ export class ConnectionManager {
                     const msg_count = parseInt(queue.messages);
                     if (msg_count > 0) {
                         try {
-                            await got.delete(apiUrl + `/api/queues/%2F${this.conn.amqp.vhost}/${queue.name}/contents`, opts);
+                            await got.delete(apiUrl + `/api/queues/${vHost}/${queue.name}/contents`, opts);
                             hLog(`${queue.messages} messages deleted on queue ${queue.name}`);
                         } catch (e) {
                             console.log(e.message);
@@ -75,21 +79,19 @@ export class ConnectionManager {
     }
 
     prepareESClient() {
-        let es_url;
         const _es = this.conn.elasticsearch;
         if (!_es.protocol) {
             _es.protocol = 'http';
         }
-        if (_es.user !== '') {
-            es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${_es.host}`;
-        } else {
-            es_url = `${_es.protocol}://${_es.host}`
-        }
         this.esIngestClient = new Client({
-            node: es_url,
-            ssl: {
+            node: `${_es.protocol}://${_es.host}`,
+            auth: {
+                username: _es.user,
+                password: _es.pass
+            },
+            ssl: _es.protocol === 'https' ? {
                 rejectUnauthorized: false
-            }
+            } : undefined
         });
     }
 
@@ -105,19 +107,17 @@ export class ConnectionManager {
         if (_es.ingest_nodes) {
             if (_es.ingest_nodes.length > 0) {
                 for (const node of _es.ingest_nodes) {
-                    let es_url;
-                    if (_es.user !== '') {
-                        es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${node}`;
-                    } else {
-                        es_url = `${_es.protocol}://${node}`
-                    }
-                    this.esIngestClients.push(new Client({
-                        node: es_url,
+                    this.esIngestClient = new Client({
+                        node: `${_es.protocol}://${_es.host}`,
+                        auth: {
+                            username: _es.user,
+                            password: _es.pass
+                        },
                         pingTimeout: 100,
-                        ssl: {
+                        ssl: _es.protocol === 'https' ? {
                             rejectUnauthorized: false
-                        }
-                    }));
+                        } : undefined
+                    });
                 }
             }
         }
@@ -140,7 +140,7 @@ export class ConnectionManager {
     }
 
     get shipClient(): StateHistorySocket {
-        return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_kb);
+        return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_mb);
     }
 
     get ampqUrl() {
@@ -149,8 +149,15 @@ export class ConnectionManager {
 
     calculateServerHash() {
         exec('git rev-parse HEAD', (err, stdout) => {
-            console.log('Last commit hash on this branch is:', stdout);
-            this.last_commit_hash = stdout.trim();
+            if (err) {
+                // hLog(`\n ${err.message}\n >>> Failed to check last commit hash. Version hash will be "custom"`);
+                hLog(`Failed to check last commit hash. Version hash will be "custom"`);
+                this.last_commit_hash = 'custom';
+            } else {
+                hLog('Last commit hash on this branch is:', stdout);
+                this.last_commit_hash = stdout.trim();
+            }
+            this.getHyperionVersion();
         });
     }
 
@@ -159,6 +166,9 @@ export class ConnectionManager {
     }
 
     getHyperionVersion() {
-        this.current_version = require('../package.json').version
+        this.current_version = require('../package.json').version;
+        if (this.last_commit_hash === 'custom') {
+            this.current_version = this.current_version + '-dirty';
+        }
     }
 }

connections/state-history.ts

@@ -1,48 +1,57 @@
 import {debugLog, hLog} from "../helpers/common_functions";
 
-const WebSocket = require('ws');
+import WebSocket from 'ws';
 
 export class StateHistorySocket {
-    private ws;
-    private readonly shipUrl;
-    private readonly max_payload_kb;
+	private ws;
+	private readonly shipUrl;
+	private readonly max_payload_mb;
+	retryOnDisconnect = true;
+	connected = false;
 
-    constructor(ship_url, max_payload_kb) {
-        this.shipUrl = ship_url;
-        if (max_payload_kb) {
-            this.max_payload_kb = max_payload_kb;
-        } else {
-            this.max_payload_kb = 256;
-        }
-    }
+	constructor(ship_url, max_payload_mb) {
+		this.shipUrl = ship_url;
+		if (max_payload_mb) {
+			this.max_payload_mb = max_payload_mb;
+		} else {
+			this.max_payload_mb = 256;
+		}
+	}
 
-    connect(onMessage, onDisconnect, onError, onConnected) {
-        debugLog(`Connecting to ${this.shipUrl}...`);
-        this.ws = new WebSocket(this.shipUrl, null, {
-            perMessageDeflate: false,
-            maxPayload: this.max_payload_kb * 1024 * 1024,
-        });
-        this.ws.on('open', () => {
-            hLog('Websocket connected!');
-            if (onConnected) {
-                onConnected();
-            }
-        });
-        this.ws.on('message', onMessage);
-        this.ws.on('close', () => {
-            hLog('Websocket disconnected!');
-            onDisconnect();
-        });
-        this.ws.on('error', (err) => {
-            hLog(`${this.shipUrl} :: ${err.message}`);
-        });
-    }
+	connect(onMessage, onDisconnect, onError, onConnected) {
+		debugLog(`Connecting to ${this.shipUrl}...`);
+		this.ws = new WebSocket(this.shipUrl, {
+			perMessageDeflate: false,
+			maxPayload: this.max_payload_mb * 1024 * 1024,
+		});
+		this.ws.on('open', () => {
+			this.connected = true;
+			hLog('Websocket connected!');
+			if (onConnected) {
+				onConnected();
+			}
+		});
+		this.ws.on('message', (data) => onMessage(data));
+		this.ws.on('close', () => {
+			this.connected = false;
+			hLog('Websocket disconnected!');
+			if(this.retryOnDisconnect) {
+				onDisconnect();
+			}
+		});
+		this.ws.on('error', (err) => {
+			hLog(`${this.shipUrl} :: ${err.message}`);
+		});
+	}
 
-    close() {
-        this.ws.close();
-    }
+	close(graceful: boolean) {
+		if(graceful) {
+			this.retryOnDisconnect = false;
+		}
+		this.ws.close();
+	}
 
-    send(payload) {
-        this.ws.send(payload);
-    }
+	send(payload) {
+		this.ws.send(payload);
+	}
 }

definitions/abi_def.js

@@ -1,260 +1,264 @@
 const AbiDefinitions = {
-    version: "eosio::abi/1.1",
-    structs: [
+  version: 'eosio::abi/1.1',
+  structs: [
+    {
+      name: 'extensions_entry',
+      base: '',
+      fields: [
         {
-            name: "extensions_entry",
-            base: "",
-            fields: [
-                {
-                    name: "tag",
-                    type: "uint16"
-                },
-                {
-                    name: "value",
-                    type: "bytes"
-                }
-            ]
+          name: 'tag',
+          type: 'uint16',
         },
         {
-            name: "type_def",
-            base: "",
-            fields: [
-                {
-                    name: "new_type_name",
-                    type: "string"
-                },
-                {
-                    name: "type",
-                    type: "string"
-                }
-            ]
+          name: 'value',
+          type: 'bytes',
+        },
+      ],
+    },
+    {
+      name: 'type_def',
+      base: '',
+      fields: [
+        {
+          name: 'new_type_name',
+          type: 'string',
         },
         {
-            name: "field_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "string"
-                },
-                {
-                    name: "type",
-                    type: "string"
-                }
-            ]
+          name: 'type',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'field_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'string',
         },
         {
-            name: "struct_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "string"
-                },
-                {
-                    name: "base",
-                    type: "string"
-                },
-                {
-                    name: "fields",
-                    type: "field_def[]"
-                }
-            ]
+          name: 'type',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'struct_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'string',
         },
         {
-            name: "action_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "name"
-                },
-                {
-                    name: "type",
-                    type: "string"
-                },
-                {
-                    name: "ricardian_contract",
-                    type: "string"
-                }
-            ]
+          name: 'base',
+          type: 'string',
         },
         {
-            name: "table_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "name"
-                },
-                {
-                    name: "index_type",
-                    type: "string"
-                },
-                {
-                    name: "key_names",
-                    type: "string[]"
-                },
-                {
-                    name: "key_types",
-                    type: "string[]"
-                },
-                {
-                    name: "type",
-                    type: "string"
-                }
-            ]
+          name: 'fields',
+          type: 'field_def[]',
+        },
+      ],
+    },
+    {
+      name: 'action_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'name',
         },
         {
-            name: "clause_pair",
-            base: "",
-            fields: [
-                {
-                    name: "id",
-                    type: "string"
-                },
-                {
-                    name: "body",
-                    type: "string"
-                }
-            ]
+          name: 'type',
+          type: 'string',
         },
         {
-            name: "error_message",
-            base: "",
-            fields: [
-                {
-                    name: "error_code",
-                    type: "uint64"
-                },
-                {
-                    name: "error_msg",
-                    type: "string"
-                }
-            ]
+          name: 'ricardian_contract',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'table_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'name',
         },
         {
-            name: "variant_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "string"
-                },
-                {
-                    name: "types",
-                    type: "string[]"
-                }
-            ]
+          name: 'index_type',
+          type: 'string',
         },
         {
-            name: "abi_def",
-            base: "",
-            fields: [
-                {
-                    name: "version",
-                    type: "string"
-                },
-                {
-                    name: "types",
-                    type: "type_def[]"
-                },
-                {
-                    name: "structs",
-                    type: "struct_def[]"
-                },
-                {
-                    name: "actions",
-                    type: "action_def[]"
-                },
-                {
-                    name: "tables",
-                    type: "table_def[]"
-                },
-                {
-                    name: "ricardian_clauses",
-                    type: "clause_pair[]"
-                },
-                {
-                    name: "error_messages",
-                    type: "error_message[]"
-                },
-                {
-                    name: "abi_extensions",
-                    type: "extensions_entry[]"
-                },
-                {
-                    name: "variants",
-                    type: "variant_def[]$"
-                }
-            ]
-        }
-    ]
+          name: 'key_names',
+          type: 'string[]',
+        },
+        {
+          name: 'key_types',
+          type: 'string[]',
+        },
+        {
+          name: 'type',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'clause_pair',
+      base: '',
+      fields: [
+        {
+          name: 'id',
+          type: 'string',
+        },
+        {
+          name: 'body',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'error_message',
+      base: '',
+      fields: [
+        {
+          name: 'error_code',
+          type: 'uint64',
+        },
+        {
+          name: 'error_msg',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'variant_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'string',
+        },
+        {
+          name: 'types',
+          type: 'string[]',
+        },
+      ],
+    },
+    {
+      name: 'abi_def',
+      base: '',
+      fields: [
+        {
+          name: 'version',
+          type: 'string',
+        },
+        {
+          name: 'types',
+          type: 'type_def[]',
+        },
+        {
+          name: 'structs',
+          type: 'struct_def[]',
+        },
+        {
+          name: 'actions',
+          type: 'action_def[]',
+        },
+        {
+          name: 'tables',
+          type: 'table_def[]',
+        },
+        {
+          name: 'ricardian_clauses',
+          type: 'clause_pair[]',
+        },
+        {
+          name: 'error_messages',
+          type: 'error_message[]',
+        },
+        {
+          name: 'abi_extensions',
+          type: 'extensions_entry[]',
+        },
+        {
+          name: 'variants',
+          type: 'variant_def[]$',
+        },
+      ],
+    },
+  ],
 };
 
 const RexAbi = {
-    version: "eosio::abi/1.1",
-    types: [],
-    structs: [
+  version: 'eosio::abi/1.1',
+  types: [],
+  structs: [
+    {
+      name: 'buyresult',
+      base: '',
+      fields: [
         {
-            name: "buyresult",
-            base: "",
-            fields: [{
-                name: "rex_received",
-                type: "asset"
-            }
-            ]
+          name: 'rex_received',
+          type: 'asset',
+        },
+      ],
+    }, {
+      name: 'orderresult',
+      base: '',
+      fields: [
+        {
+          name: 'owner',
+          type: 'name',
         }, {
-            name: "orderresult",
-            base: "",
-            fields: [{
-                name: "owner",
-                type: "name"
-            }, {
-                name: "proceeds",
-                type: "asset"
-            }
-            ]
-        }, {
-            name: "rentresult",
-            base: "",
-            fields: [{
-                name: "rented_tokens",
-                type: "asset"
-            }
-            ]
+          name: 'proceeds',
+          type: 'asset',
         },
+      ],
+    }, {
+      name: 'rentresult',
+      base: '',
+      fields: [
         {
-            name: "sellresult",
-            base: "",
-            fields: [{
-                name: "proceeds",
-                type: "asset"
-            }]
-        }
-    ],
-    actions: [
-        {
-            name: "buyresult",
-            type: "buyresult",
-            ricardian_contract: ""
+          name: 'rented_tokens',
+          type: 'asset',
         },
+      ],
+    },
+    {
+      name: 'sellresult',
+      base: '',
+      fields: [
         {
-            name: "orderresult",
-            type: "orderresult",
-            ricardian_contract: ""
-        },
-        {
-            name: "rentresult",
-            type: "rentresult",
-            ricardian_contract: ""
-        },
-        {
-            name: "sellresult",
-            type: "sellresult",
-            ricardian_contract: ""
-        }
-    ]
+          name: 'proceeds',
+          type: 'asset',
+        }],
+    },
+  ],
+  actions: [
+    {
+      name: 'buyresult',
+      type: 'buyresult',
+      ricardian_contract: '',
+    },
+    {
+      name: 'orderresult',
+      type: 'orderresult',
+      ricardian_contract: '',
+    },
+    {
+      name: 'rentresult',
+      type: 'rentresult',
+      ricardian_contract: '',
+    },
+    {
+      name: 'sellresult',
+      type: 'sellresult',
+      ricardian_contract: '',
+    },
+  ],
 };
 
 module.exports = {AbiDefinitions, RexAbi};

definitions/ecosystem_settings.js

@@ -1,38 +1,51 @@
-function addIndexer(chainName) {
-  return {
-    script: './launcher.js',
-    name: chainName + '-indexer',
-    namespace: chainName,
-    interpreter: 'node',
-    interpreter_args: ['--max-old-space-size=4096', '--trace-deprecation'],
-    autorestart: false,
-    kill_timeout: 3600,
-    watch: false,
-    time: true,
-    env: {
-      CONFIG_JSON: 'chains/' + chainName + '.config.json',
-      TRACE_LOGS: 'false',
-    },
-  };
+function interpreterArgs(heap) {
+    const arr = ['--trace-deprecation', '--trace-warnings'];
+    if (heap) {
+        arr.push('--max-old-space-size=' + heap);
+    } else {
+        arr.push('--max-old-space-size=2048');
+    }
+    if (process.env.INSPECT) {
+        arr.push('--inspect');
+    }
+    return arr;
 }
 
-function addApiServer(chainName, threads) {
-  return {
-    script: './api/server.js',
-    name: chainName + '-api',
-    namespace: chainName,
-    node_args: ['--trace-deprecation'],
-    exec_mode: 'cluster',
-    merge_logs: true,
-    instances: threads,
-    autorestart: true,
-    exp_backoff_restart_delay: 100,
-    watch: false,
-    time: true,
-    env: {
-      CONFIG_JSON: 'chains/' + chainName + '.config.json',
-    },
-  };
+function addIndexer(chainName, heap) {
+    return {
+        script: './launcher.js',
+        name: chainName + '-indexer',
+        namespace: chainName,
+        interpreter: 'node',
+        interpreter_args: interpreterArgs(heap),
+        autorestart: false,
+        kill_timeout: 3600,
+        watch: false,
+        time: true,
+        env: {
+            CONFIG_JSON: 'chains/' + chainName + '.config.json',
+            TRACE_LOGS: 'false',
+        },
+    };
+}
+
+function addApiServer(chainName, threads, heap) {
+    return {
+        script: './api/server.js',
+        name: chainName + '-api',
+        namespace: chainName,
+        node_args: interpreterArgs(heap),
+        exec_mode: 'cluster',
+        merge_logs: true,
+        instances: threads,
+        autorestart: true,
+        exp_backoff_restart_delay: 100,
+        watch: false,
+        time: true,
+        env: {
+            CONFIG_JSON: 'chains/' + chainName + '.config.json',
+        },
+    };
 }
 
 module.exports = {addIndexer, addApiServer};

definitions/index-lifecycle-policies.ts

@@ -1,5 +1,6 @@
 import {IlmPutLifecycle} from "@elastic/elasticsearch/api/requestParams";
 
+// noinspection JSUnusedGlobalSymbols
 export const ILPs: IlmPutLifecycle[] = [
     {
         policy: "hyperion-rollover",

definitions/index-templates.ts

@@ -1,9 +1,7 @@
 import {ConfigurationModule} from "../modules/config";
 
 const shards = 2;
-const replicas = 0;
 const refresh = "1s";
-let defaultLifecyclePolicy = "200G";
 
 export * from './index-lifecycle-policies';
 
@@ -15,12 +13,10 @@ const compression = "best_compression";
 const cm = new ConfigurationModule();
 const chain = cm.config.settings.chain;
 
-if (cm.config.settings.hot_warm_policy) {
-    defaultLifecyclePolicy = "hyperion-rollover";
-}
-
-if(cm.config.settings.custom_policy) {
-    defaultLifecyclePolicy = cm.config.settings.custom_policy;
+// update number of replicas if set to larger than 0
+let replicas = 0;
+if (cm.config.settings.es_replicas) {
+    replicas = cm.config.settings.es_replicas;
 }
 
 const defaultIndexSettings = {
@@ -34,10 +30,6 @@ const defaultIndexSettings = {
 
 const actionSettings = {
     index: {
-        lifecycle: {
-            "name": defaultLifecyclePolicy,
-            "rollover_alias": chain + "-action"
-        },
         codec: compression,
         refresh_interval: refresh,
         number_of_shards: shards * 2,
@@ -49,10 +41,25 @@ const actionSettings = {
     }
 };
 
+// actionSettings.index["lifecycle"] = {
+//     "name": defaultLifecyclePolicy,
+//     "rollover_alias": chain + "-action"
+// };
+
 if (cm.config.settings.hot_warm_policy) {
     actionSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
 }
 
+const transferProps = {
+    "properties": {
+        "from": {"type": "keyword"},
+        "to": {"type": "keyword"},
+        "amount": {"type": "float"},
+        "symbol": {"type": "keyword"},
+        "memo": {"type": "text"}
+    }
+};
+
 export const action = {
     order: 0,
     index_patterns: [
@@ -72,6 +79,7 @@ export const action = {
             "act.name": {"type": "keyword"},
             "act.data": {"enabled": false},
             "block_num": {"type": "long"},
+            "block_id": {"type": "keyword"},
             "action_ordinal": {"type": "long"},
             "creator_action_ordinal": {"type": "long"},
             "cpu_usage_us": {"type": "integer"},
@@ -80,7 +88,6 @@ export const action = {
             "abi_sequence": {"type": "integer"},
             "trx_id": {"type": "keyword"},
             "producer": {"type": "keyword"},
-            "notified": {"type": "keyword"},
             "signatures": {"enabled": false},
             "inline_count": {"type": "short"},
             "max_inline": {"type": "short"},
@@ -118,15 +125,7 @@ export const action = {
             },
 
             // *::transfer
-            "@transfer": {
-                "properties": {
-                    "from": {"type": "keyword"},
-                    "to": {"type": "keyword"},
-                    "amount": {"type": "float"},
-                    "symbol": {"type": "keyword"},
-                    "memo": {"type": "text"}
-                }
-            },
+            "@transfer": transferProps,
 
             // eosio::unstaketorex
             "@unstaketorex": {
@@ -184,26 +183,96 @@ export const action = {
                     "unstake_net_quantity": {"type": "float"},
                     "amount": {"type": "float"}
                 }
-            }
+            },
+
+            // eosio::undelegatebw
+            // "@exec": {
+            //     "properties": {
+            //         "executer": {"type": "keyword"},
+            //         "coopname": {"type": "keyword"},
+            //         "decision_id": {"type": "long"},
+            //     }
+            // },
+
+            // "@statement": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "username": {"type": "keyword"},
+            //         "action": {"type": "keyword"},
+            //         "decision_id": {"type": "long"},
+            //         "batch_id": {"type": "long"},
+            //         "statement": {"enabled": false},
+            //     }
+            // },
+
+            // "@act": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "username": {"type": "keyword"},
+            //         "action": {"type": "keyword"},
+            //         "decision_id": {"type": "long"},
+            //         "batch_id": {"type": "long"},
+            //         "act": {"enabled": false},
+            //     }
+            // },
+
+            // "@decision": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "username": {"type": "keyword"},
+            //         "action": {"type": "keyword"},
+            //         "decision_id": {"type": "long"},
+            //         "batch_id": {"type": "long"},
+            //         "decision": {"enabled": false},
+            //     }
+            // },
+            // "@batch": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "action": {"type": "keyword"},
+            //         "batch_id": {"type": "long"},
+            //     }
+            // },
+
+
+            // // eosio::votefor
+            // "@votefor": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "member": {"type": "keyword"},
+            //         "decision_id": {"type": "long"}
+            //     }
+            // },
+
+            // "@voteagainst": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "member": {"type": "keyword"},
+            //         "decision_id": {"type": "long"}
+            //     }
+            // }
         }
     }
 };
 
 const deltaSettings = {
-    "index": {
-        "lifecycle": {
-            "name": defaultLifecyclePolicy,
-            "rollover_alias": chain + "-delta"
-        },
-        "codec": compression,
-        "number_of_shards": shards * 2,
-        "refresh_interval": refresh,
-        "number_of_replicas": replicas,
-        "sort.field": ["block_num", "scope", "primary_key"],
-        "sort.order": ["desc", "asc", "asc"]
+    index: {
+        codec: compression,
+        number_of_shards: shards * 2,
+        refresh_interval: refresh,
+        number_of_replicas: replicas,
+        sort: {
+            field: ["block_num", "scope", "primary_key"],
+            order: ["desc", "asc", "asc"]
+        }
     }
 };
 
+// deltaSettings.index["lifecycle"] = {
+//     "name": defaultLifecyclePolicy,
+//     "rollover_alias": chain + "-delta"
+// };
+
 if (cm.config.settings.hot_warm_policy) {
     deltaSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
 }
@@ -216,6 +285,7 @@ export const delta = {
 
             // base fields
             "@timestamp": {"type": "date"},
+            "present": {"type": "byte"},
             "ds_error": {"type": "boolean"},
             "block_id": {"type": "keyword"},
             "block_num": {"type": "long"},
@@ -298,7 +368,7 @@ export const permissionLink = {
         "properties": {
             "block_num": {"type": "long"},
             "@timestamp": {"type": "date"},
-            "present": {"type": "boolean"},
+            "present": {"type": "byte"},
             "account": {"type": "keyword"},
             "code": {"type": "keyword"},
             "action": {"type": "keyword"},
@@ -313,7 +383,7 @@ export const permission = {
     "mappings": {
         "properties": {
             "block_num": {"type": "long"},
-            "present": {"type": "boolean"},
+            "present": {"type": "byte"},
             "owner": {"type": "keyword"},
             "name": {"type": "keyword"},
             "parent": {"type": "keyword"},
@@ -368,6 +438,26 @@ export const failedTransaction = {
     }
 };
 
+export const schedule = {
+    "index_patterns": [chain + "-schedule-*"],
+    "settings": {
+        ...defaultIndexSettings,
+        sort: {
+            field: "version",
+            order: "desc"
+        }
+    },
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "version": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "producers.name": {"type": "keyword"},
+            "producers.keys": {"type": "keyword"},
+        }
+    }
+};
+
 export const resourceUsage = {
     "index_patterns": [chain + "-userres-*"],
     "settings": defaultIndexSettings,
@@ -437,7 +527,7 @@ export const tableProposals = {
     "mappings": {
         "properties": {
             "block_num": {"type": "long"},
-            "present": {"type": "boolean"},
+            "present": {"type": "byte"},
             "proposal_name": {"type": "keyword"},
             "requested_approvals": {"type": "object"},
             "provided_approvals": {"type": "object"},
@@ -461,7 +551,7 @@ export const tableAccounts = {
     "mappings": {
         "properties": {
             "block_num": {"type": "long"},
-            "present": {"type": "boolean"},
+            "present": {"type": "byte"},
             "code": {"type": "keyword"},
             "scope": {"type": "keyword"},
             "amount": {"type": "float"},
@@ -470,6 +560,7 @@ export const tableAccounts = {
     }
 };
 
+// noinspection JSUnusedGlobalSymbols
 export const tableDelBand = {
     "index_patterns": [chain + "-table-delband-*"],
     "settings": {

docker/elasticsearch/config/elasticsearch.yml

@@ -0,0 +1,5 @@
+cluster.name: docker-cluster
+network.host: 0.0.0.0
+bootstrap.memory_lock: true
+discovery.type: single-node
+node.name: es01

docker/elasticsearch/config/jvm.options.d/heap.options

@@ -0,0 +1,2 @@
+-Xms8g
+-Xmx8g

docker/eosio/Dockerfile

@@ -1,11 +1,5 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get upgrade -y && apt-get autoremove && apt-get install -y wget netcat
-
-RUN wget -nv https://github.com/eosio/eos/releases/download/v2.0.5/eosio_2.0.5-1-ubuntu-18.04_amd64.deb
-RUN apt-get install -y ./eosio_2.0.5-1-ubuntu-18.04_amd64.deb
-
-RUN useradd -m -s /bin/bash eosio
-USER eosio
-
-EXPOSE 8080
+FROM ubuntu:20.04
+LABEL version="2.1.0" description="EOSIO 2.1.0"
+RUN apt-get update && apt-get install -y wget
+RUN wget -nv https://github.com/EOSIO/eos/releases/download/v2.1.0/eosio_2.1.0-1-ubuntu-20.04_amd64.deb
+RUN apt-get install -y ./eosio_2.1.0-1-ubuntu-20.04_amd64.deb && rm -f ./eosio_2.1.0-1-ubuntu-20.04_amd64.deb

docker/eosio/config/config.ini

@@ -1,28 +1,14 @@
-# Enable block production, even if the chain is stale. (eosio::producer_plugin)
+### PRODUCER PLUGIN ###
 enable-stale-production = true
-
-# ID of producer controlled by this node (e.g. inita; may specify multiple times) (eosio::producer_plugin)
 producer-name = eosio
 
-# print contract's output to console (eosio::chain_plugin)
-contracts-console = true
-
-# The local IP and port to listen for incoming http connections; set blank to disable. (eosio::http_plugin)
-http-server-address = 0.0.0.0:8888
-
-# If set to false, then any incoming "Host" header is considered valid (eosio::http_plugin)
-http-validate-host = false
-
-# enable trace history (eosio::state_history_plugin)
+### STATE HISTORY PLUGIN ###
+plugin = eosio::state_history_plugin
+state-history-endpoint = 0.0.0.0:8080
 trace-history = true
-
-# enable chain state history (eosio::state_history_plugin)
 chain-state-history = true
 
-# the endpoint upon which to listen for incoming connections. Caution: only expose this port to your internal network. (eosio::state_history_plugin)
-state-history-endpoint = 0.0.0.0:8080
-
-# Plugin(s) to enable, may be specified multiple times
-plugin = eosio::producer_api_plugin
+### CHAIN API PLUGIN ###
 plugin = eosio::chain_api_plugin
-plugin = eosio::state_history_plugin
+http-server-address = 0.0.0.0:8888
+http-validate-host = false

docker/hyperion/Dockerfile

@@ -1,15 +1,6 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get upgrade -y && apt-get autoremove && apt-get install -y build-essential git curl netcat
-
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-RUN apt-get install -y nodejs && npm install pm2 -g && git clone https://github.com/eosrio/hyperion-history-api.git
-
-RUN useradd -m -s /bin/bash hyperion && chown -R hyperion:hyperion /hyperion-history-api
-USER hyperion
+FROM node:16.5
+RUN npm install pm2 -g
+RUN git clone https://github.com/eosrio/hyperion-history-api.git
 WORKDIR /hyperion-history-api
-
-RUN npm install
-
-EXPOSE 7001
-
+RUN git checkout dev-3.3
+RUN npm install --production