rollback fastify-elasticsearch

API logs need timestamp

.gitignore

@@ -79,8 +79,6 @@ api/**/*.js
 api/**/*.js.map
 api/tests
 
-hyperion-explorer/
-
 hyperion-launcher.js
 hyperion-launcher.js.map
 
@@ -107,3 +105,10 @@ modules/**/*.js.map
 
 addons/**/*.js
 addons/**/*.js.map
+
+plugins/repos
+
+docker/redis/data
+docker/elasticsearch/data
+docker/eosio/data
+docker/rabbitmq/data

DEPRECATED_install_env.sh

@@ -1,14 +1,20 @@
 #!/usr/bin/env bash
-#set -eo pipefail
+set -eo pipefail
 
 #terminal colors
-export COLOR_NC=$(tput sgr0) #No Color
-export COLOR_RED=$(tput setaf 1)
-export COLOR_GREEN=$(tput setaf 2)
-export COLOR_YELLOW=$(tput setaf 3)
-export COLOR_BLUE=$(tput setaf 4)
+COLOR_NC=$(tput sgr0)
+export COLOR_NC
+COLOR_RED=$(tput setaf 1)
+export COLOR_RED
+COLOR_GREEN=$(tput setaf 2)
+export COLOR_GREEN
+COLOR_YELLOW=$(tput setaf 3)
+export COLOR_YELLOW
+COLOR_BLUE=$(tput setaf 4)
+export COLOR_BLUE
 
 #variables
+INITIAL_PARAMS="$*"
 GLOBAL="y"
 RABBIT_USER="hyperion"
 RABBIT_PASSWORD="123456"
@@ -16,97 +22,117 @@ NODE=false
 ELASTIC=false
 REDIS=false
 RABBIT=false
-KIBANA=false
 RAM=0
 
-check_ram(){
+help_function() {
+    echo -e "\nAutomated shell script that installs all dependencies and then configure Hyperion."
+    echo -e "\nUsage: ./install.env <option>\n"
+    echo -e "Options:"
+    echo -e "  --help ==> Show this menu"
+    echo -e "  --version ==> Show Hyperion current version\n"
+    exit 0
+}
+
+version (){
+  echo -e "\nv3.0.0\n"
+  exit 0
+
+}
+
+arg_checker() {
+    if [ "${INITIAL_PARAMS}" == "--help" ]; then
+        help_function
+
+    elif [ "${INITIAL_PARAMS}" == "--version" ]; then
+        version
+    fi
+}
+
+check_ram() {
   RAM=$(free --giga | awk '/Mem/ {print $2}')
 }
-#first check if is installed, after check version
-check_dependecies(){
+
+# first check if is installed, after check version
+check_dependencies() {
   echo -e "\n\n${COLOR_BLUE}Checking dependencies...${COLOR_NC}\n\n"
-  CMD=$(if(dpkg --compare-versions 2> /dev/null $(dpkg -s  nodejs 2> /dev/null | awk '/Version/ {print $2}') ge "5"); then echo true; else echo false; fi)
-  if("$CMD" = true); then
+  CMD=$(if (dpkg --compare-versions $(dpkg -s nodejs 2>/dev/null | awk '/Version/ {print $2}') ge "5" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
     NODE=true
-    echo -e "\n\n${COLOR_BLUE}Nodejs compatible version already installed ${COLOR_NC}\n\n"
-  elif("$CMD" = false); then
+    echo -e "\n\n${COLOR_BLUE}Nodejs compatible version already installed ${COLOR_NC}"
+  elif ("$CMD" = false); then
     echo -e "\n\n${COLOR_RED}Nodejs installed version is < 13. Please, update and try again. ${COLOR_NC}\n\n"
     exit 1
   else
-    echo -e "\n\n${COLOR_BLUE}Nodejs not installed ${COLOR_NC}\n\n"
+    echo -e "\n\n${COLOR_BLUE}Nodejs not installed ${COLOR_NC}"
   fi
-  CMD=$(if(dpkg --compare-versions 2> /dev/null $(dpkg -s  elasticsearch 2> /dev/null | awk '/Version/ {print $2}') ge "7.6"); then echo true; else echo false; fi)
-  if("$CMD" = true); then
+  CMD=$(if (dpkg --compare-versions $(dpkg -s elasticsearch 2>/dev/null | awk '/Version/ {print $2}') ge "7.6" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
     ELASTIC=true
-    echo -e "\n\n${COLOR_BLUE}Elasticsearch compatible version already installed ${COLOR_NC}\n\n"
+    echo -e "\n${COLOR_BLUE}Elasticsearch compatible version already installed ${COLOR_NC}"
   elif ("$CMD" = false); then
     echo -e "\n\n${COLOR_RED}Elasticsearch installed version is < 7.6. Please, update and try again. ${COLOR_NC}\n\n"
     exit 1
   else
-    echo -e "\n\n${COLOR_BLUE}Elasticsearch not installed ${COLOR_NC}\n\n"
+    echo -e "\n${COLOR_BLUE}Elasticsearch not installed ${COLOR_NC}"
   fi
-  CMD=$(if(dpkg --compare-versions 2> /dev/null $(dpkg -s  redis-server 2> /dev/null | awk '/Version/ {print $2}') ge "5"); then echo true; else echo false; fi)
-  if("$CMD" = true); then
+  CMD=$(if (dpkg --compare-versions $(dpkg -s redis-server 2>/dev/null | awk '/Version/ {print $2}') ge "5" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
     REDIS=true
-    echo -e "\n\n${COLOR_BLUE}Redis compatible version already installed ${COLOR_NC}\n\n"
-  elif("$CMD" = false); then
+    echo -e "\n${COLOR_BLUE}Redis compatible version already installed ${COLOR_NC}"
+  elif ("$CMD" = false); then
     echo -e "\n\n${COLOR_RED}Redis installed version is < 5. Please, update and try again. ${COLOR_NC}\n\n"
     exit 1
   else
-    echo -e "\n\n${COLOR_BLUE}Redis not installed ${COLOR_NC}\n\n"
+    echo -e "\n${COLOR_BLUE}Redis not installed ${COLOR_NC}"
   fi
-  CMD=$(if(dpkg --compare-versions 2> /dev/null $(dpkg -s  rabbitmq-server 2> /dev/null | awk '/Version/ {print $2}') ge "3.8"); then echo true; else echo false; fi)
-  if("$CMD" = true); then
+  CMD=$(if (dpkg --compare-versions $(dpkg -s rabbitmq-server 2>/dev/null | awk '/Version/ {print $2}') ge "3.8" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
     RABBIT=true
-    echo -e "\n\n${COLOR_BLUE}RabbitMQ compatible version already installed ${COLOR_NC}\n\n"
-  elif("$CMD" = false); then
+    echo -e "\n${COLOR_BLUE}RabbitMQ compatible version already installed ${COLOR_NC}"
+  elif ("$CMD" = false); then
     echo -e "\n\n${COLOR_RED}RabbitMQ installed version is < 3.8. Please, update and try again. ${COLOR_NC}\n\n"
     exit 1
   else
-    echo -e "\n\n${COLOR_BLUE}RabbitMQ not installed ${COLOR_NC}\n\n"
+    echo -e "\n${COLOR_BLUE}RabbitMQ not installed ${COLOR_NC}"
   fi
-  CMD=$(if(dpkg --compare-versions 2> /dev/null $(dpkg -s  kibana 2> /dev/null | awk '/Version/ {print $2}') ge "7.6"); then echo true; else echo false; fi)
-  if("$CMD" = true); then
+  CMD=$(if (dpkg --compare-versions $(dpkg -s kibana 2>/dev/null | awk '/Version/ {print $2}') ge "7.6" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
     RABBIT=true
-    echo -e "\n\n${COLOR_BLUE}Kibana compatible version already installed ${COLOR_NC}\n\n"
-  elif("$CMD" = false); then
+    echo -e "\n${COLOR_BLUE}Kibana compatible version already installed ${COLOR_NC}"
+  elif ("$CMD" = false); then
     echo -e "\n\n${COLOR_RED}Kibana installed version is < 7.6. Please, update and try again. ${COLOR_NC}\n\n"
     exit 1
   else
-    echo -e "\n\n${COLOR_BLUE}Kibana not installed ${COLOR_NC}\n\n"
+    echo -e "\n${COLOR_BLUE}Kibana not installed ${COLOR_NC}\n\n"
   fi
 }
 
 # Make a directory for global installations
-configure_npm(){
+configure_npm() {
   mkdir ~/.npm-global
-  npm config set prefix '~/.npm-global'
-  #export path for the current session
-  export PATH=~/.npm-global/bin:$PATH
-  #make PATH persistent
-  echo "export PATH=~/.npm-global/bin:$PATH" >> ~/.profile
+  npm config set prefix "$HOME/.npm-global"
+  # export path for the current session
+  export PATH=$HOME/.npm-global/bin:$PATH
+  # make PATH persistent
+  echo "export PATH=~/.npm-global/bin:$PATH" >>"$HOME/.profile"
+  # shellcheck source=/dev/null
   source ~/.profile
 }
 
-install_keys_sources(){
+install_keys_sources() {
   echo -e "\n\n${COLOR_BLUE}Configuring keys and sources...${COLOR_NC}\n\n"
 
-  #export APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
-
   PPA="https://artifacts.elastic.co/packages/7.x/apt stable main"
   if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
     wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
     echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
   fi
 
-  KEY=$(apt-key list 2> /dev/null | grep erlang)
-  if [[ ! $KEY ]]; then
-    wget -O- https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc | sudo apt-key add -
-  fi
-
-  KEY=$(apt-key list 2> /dev/null | grep rabbitmq)
-  if [[ ! $KEY ]]; then
-    wget -O - "https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey" | sudo apt-key add -
+  PPA="http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang"
+  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
+    curl -fsSL https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc | sudo apt-key add -
+    echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" | sudo tee /etc/apt/sources.list.d/bintray.rabbitmq.list
+    curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | sudo bash
   fi
 
   PPA="https://deb.nodesource.com/node_13.x bionic main"
@@ -117,32 +143,28 @@ install_keys_sources(){
   sudo apt update -y
 }
 
-install_dep(){
-  echo -e "\n\n${COLOR_BLUE}Installing build essentials...${COLOR_NC}\n\n"
-  sudo apt install -y build-essential curl
+install_dep() {
+ echo -e "\n\n${COLOR_BLUE}Installing dependencies...${COLOR_NC}\n\n"
+ # sudo apt install -y build-essential curl
+ sudo apt install -y curl
 }
 
-install_node(){
+install_node() {
   echo -e "\n\n${COLOR_BLUE}Installing nodejs...${COLOR_NC}\n\n"
   sudo apt install -y nodejs
   read -p "Do you want to create a directory for npm global installations [Y/n] : " GLOBAL
   GLOBAL=${GLOBAL:-y}
   if [ "$GLOBAL" = "y" ]; then
     configure_npm
-    npm install -g node-gyp
-    npm install -g typescript
-  else
-    sudo npm install -g node-gyp
-    sudo npm install -g typescript
   fi
 }
 
-install_build_hyperion(){
-  echo -e "\n\n${COLOR_BLUE}Installing and building hyperion...${COLOR_NC}\n\n"
+install_build_hyperion() {
+  echo -e "\n\n${COLOR_BLUE}Installing packages and building hyperion...${COLOR_NC}\n\n"
   npm install
 }
 
-install_pm2(){
+install_pm2() {
   echo -e "\n\n${COLOR_BLUE}Installing pm2...${COLOR_NC}\n\n"
   if [ "$GLOBAL" = "y" ]; then
     npm install pm2@latest -g
@@ -151,21 +173,20 @@ install_pm2(){
   fi
 }
 
-install_redis(){
+install_redis() {
   echo -e "\n\n${COLOR_BLUE}Installing redis...${COLOR_NC}\n\n"
   sudo apt install -y redis-server
   sudo sed -ie 's/supervised no/supervised systemd/' /etc/redis/redis.conf
   sudo systemctl restart redis.service
 }
 
-install_earlang(){
-  echo -e "\n\n${COLOR_BLUE}Installing earlang...${COLOR_NC}\n\n"
-  echo "deb https://packages.erlang-solutions.com/ubuntu bionic contrib" | sudo tee /etc/apt/sources.list.d/rabbitmq.list
-  sudo apt update
+install_erlang() {
+  echo -e "\n\n${COLOR_BLUE}Installing erlang...${COLOR_NC}\n\n"
   sudo apt -y install erlang
 }
+
 #ask user for rabbit credentials
-rabbit_credentials(){
+rabbit_credentials() {
   read -p "Enter rabbitmq user [hyperion]: " RABBIT_USER
   RABBIT_USER=${RABBIT_USER:-hyperion}
 
@@ -174,9 +195,8 @@ rabbit_credentials(){
 
 }
 
-install_rabittmq(){
+install_rabittmq() {
   echo -e "\n\n${COLOR_BLUE}Installing rabbit-mq...${COLOR_NC}\n\n"
-  curl -s "https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh" | sudo bash
   sudo apt install -y rabbitmq-server
   #enable web gui
   sudo rabbitmq-plugins enable rabbitmq_management
@@ -186,7 +206,7 @@ install_rabittmq(){
   sudo rabbitmqctl set_permissions -p /hyperion ${RABBIT_USER} ".*" ".*" ".*"
 }
 
-install_elastic(){
+install_elastic() {
   echo -e "\n\n${COLOR_BLUE}Installing elastic...${COLOR_NC}\n\n"
   sudo apt install -y elasticsearch
 
@@ -196,48 +216,72 @@ install_elastic(){
   sudo sed -ie 's/#cluster.name: my-application/cluster.name: myCluster/; s/#bootstrap.memory_lock: true/bootstrap.memory_lock: true/' /etc/elasticsearch/elasticsearch.yml
   # set jvm options based on system RAM
   check_ram
-  if [ "$RAM" -lt 20 ]; then
-    let RAM=$RAM/2
+  if [ "$RAM" -lt 32 ]; then
+    (( RAM=RAM/2 ))
     sudo sed -ie 's/-Xms1g/-Xms'"$RAM"'g/; s/-Xmx1g/-Xmx'"$RAM"'g/' /etc/elasticsearch/jvm.options
   else
     sudo sed -ie 's/-Xms1g/-Xms16g/; s/-Xmx1g/-Xmx16g/' /etc/elasticsearch/jvm.options
   fi
+
+  sudo bash -c 'echo "xpack.security.enabled: true" >> /etc/elasticsearch/elasticsearch.yml'
+
   sudo mkdir -p /etc/systemd/system/elasticsearch.service.d/
   echo -e "[Service]\nLimitMEMLOCK=infinity" | sudo tee /etc/systemd/system/elasticsearch.service.d/override.conf
   sudo systemctl daemon-reload
   sudo service elasticsearch start
   sudo systemctl enable elasticsearch
 
+  echo -e "\n\n${COLOR_BLUE}Generating Elasticsearch cluster passwords...${COLOR_NC}\n\n"
+
+  echo "y" | sudo /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto >elastic_pass.txt
+
 }
 
-install_kibana(){
-  echo -e "\n\n${COLOR_BLUE}Installing kibana...${COLOR_NC}\n\n"
+install_kibana() {
+  echo -e "\n\n${COLOR_BLUE}Installing and configuring Kibana...${COLOR_NC}\n\n"
   sudo apt install -y kibana
+
+  KIBANA_PASSWORD=$(awk <elastic_pass.txt '/PASSWORD kibana =/ {print $4}')
+
+  sudo sed -ie 's/#elasticsearch.password: "pass"/elasticsearch.password: '"$KIBANA_PASSWORD"'/; s/#elasticsearch.username: "kibana"/elasticsearch.username: "kibana"/' /etc/kibana/kibana.yml
+
   sudo systemctl enable kibana
   sudo systemctl start kibana
 }
 
+#******************
+# End of functions
+#******************
+arg_checker
+
 echo -e "\n\n${COLOR_BLUE}*** STARTING HYPERION HISTORY API CONFIGURATION ***${COLOR_NC}\n\n"
 
-check_dependecies
-if [ "$RABBIT" = false ] ; then
+check_dependencies
+
+if [ "$RABBIT" = false ]; then
   rabbit_credentials
 fi
+
 install_dep
+
 install_keys_sources
-if [ "$NODE" = false ] ; then
+if [ "$NODE" = false ]; then
   install_node
 fi
+
 install_pm2
+
 install_build_hyperion
-if [ "$RABBIT" = false ] ; then
-  install_earlang
+
+if [ "$RABBIT" = false ]; then
   install_rabittmq
 fi
-if [ "$REDIS" = false ] ; then
+
+if [ "$REDIS" = false ]; then
   install_redis
 fi
-if [ "$ELASTIC" = false ] ; then
+
+if [ "$ELASTIC" = false ]; then
   install_elastic
   install_kibana
 fi
@@ -255,4 +299,5 @@ echo -e "\n${COLOR_YELLOW}Please, check your installation: ${COLOR_NC}"
 echo -e "${COLOR_YELLOW}- Elastic: http://localhost:9200 ${COLOR_NC}"
 echo -e "${COLOR_YELLOW}- Kibana: http://localhost:5601 ${COLOR_NC}"
 echo -e "${COLOR_YELLOW}- RabbitMQ: http://localhost:15672 ${COLOR_NC}"
+echo -e "${COLOR_YELLOW}- Refer to the elastic_pas.txt file for the elasticsearch passwords. ${COLOR_NC}"
 echo -e "${COLOR_YELLOW}- Make pm2 auto-boot at server restart: \$ pm2 startup ${COLOR_NC}\n"

INSTALL.md

@@ -1,195 +0,0 @@
-### nodeos config.ini
-```
-state-history-dir = "state-history"
-trace-history = true
-chain-state-history = true
-state-history-endpoint = 127.0.0.1:8080
-plugin = eosio::state_history_plugin
-```
-
-### NodeJS
-
-```bash
-curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -
-sudo apt-get install -y nodejs
-```
-
-### PM2
-
-```bash
-sudo npm install pm2@latest -g
-sudo pm2 startup
-```
-
-### Elasticsearch Installation
-
-* Follow instructions on https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html (Ubuntu/Debian)
-
-Edit `/etc/elasticsearch/elasticsearch.yml`
-
-```
-cluster.name: myCluster
-bootstrap.memory_lock: true
-```
-
-Edit `/etc/elasticsearch/jvm.options`
-```
-# Set your heap size, avoid allocating more than 31GB, even if you have enought RAM.
-# Test on your specific machine by changing -Xmx32g in the following command:
-# java -Xmx32g -XX:+UseCompressedOops -XX:+PrintFlagsFinal Oops | grep Oops
--Xms16g
--Xmx16g
-```
-
-Disable swap on /etc/fstab
-
-Allow memlock:
-run `sudo systemctl edit elasticsearch` and add the following lines
-
-
-```
-[Service]
-LimitMEMLOCK=infinity
-```
-
-Start elasticsearch and check the logs (verify if the memory lock was successful)
-
-```bash
-sudo service elasticsearch start
-sudo less /var/log/elasticsearch/myCluster.log
-sudo systemctl enable elasticsearch
-```
-
-Test the REST API `curl http://localhost:9200`
-
-```
-{
-  "name" : "ip-172-31-5-121",
-  "cluster_name" : "hyperion",
-  "cluster_uuid" : "....",
-  "version" : {
-    "number" : "7.1.0",
-    "build_flavor" : "default",
-    "build_type" : "deb",
-    "build_hash" : "606a173",
-    "build_date" : "2019-05-16T00:43:15.323135Z",
-    "build_snapshot" : false,
-    "lucene_version" : "8.0.0",
-    "minimum_wire_compatibility_version" : "6.8.0",
-    "minimum_index_compatibility_version" : "6.0.0-beta1"
-  },
-  "tagline" : "You Know, for Search"
-}
-```
-
-### Kibana Installation
-
-```bash
-wget https://artifacts.elastic.co/downloads/kibana/kibana-7.4.0-amd64.deb
-sudo apt install ./kibana-7.4.0-amd64.deb
-sudo systemctl enable kibana
-```
-
-Open and test Kibana on `http://localhost:5601`
-
-### RabbitMQ Installation
-
-`https://www.rabbitmq.com/install-debian.html#installation-methods`
-
-```bash
-sudo apt install rabbitmq-server
-sudo rabbitmq-plugins enable rabbitmq_management
-sudo rabbitmqctl add_vhost /hyperion
-sudo rabbitmqctl add_user my_user my_password
-sudo rabbitmqctl set_user_tags my_user administrator
-sudo rabbitmqctl set_permissions -p /hyperion my_user ".*" ".*" ".*"
-```
-
-Check access to the WebUI `http://localhost:15672`
-
-### Redis Installation
-
-```bash
-sudo apt install redis-server
-```
-
-Edit `/etc/redis/redis.conf` and change supervised to `systemd`
-
-```bash
-sudo systemctl restart redis.service
-```
-
-### Hyperion Indexer
-
-```bash
-git clone https://github.com/eosrio/Hyperion-History-API.git
-cd Hyperion-History-API
-npm install
-cp example-ecosystem.config.js ecosystem.config.js
-```
-
-Edit `ecosystem.config.js`
-
-
-### Setup Indices and Aliases
-
-Load templates first by starting the Hyperion Indexer in preview mode `PREVIEW: 'true'`
-
-Indices and aliases are created automatically using the `CREATE_INDICES` option (set it to your version suffix e.g, v1, v2, v3)
-If you want to create them manually, use the commands bellow on the kibana dev console
-```
-PUT mainnet-action-v1-000001
-PUT mainnet-abi-v1-000001
-PUT mainnet-block-v1-000001
-
-POST _aliases
-{
-  "actions": [
-    {
-      "add": {
-        "index": "mainnet-abi-v1-000001",
-        "alias": "mainnet-abi"
-      }
-    },
-    {
-      "add": {
-        "index": "mainnet-action-v1-000001",
-        "alias": "mainnet-action"
-      }
-    },
-    {
-      "add": {
-        "index": "mainnet-block-v1-000001",
-        "alias": "mainnet-block"
-      }
-    }
-  ]
-}
-```
-
-Before indexing actions into elasticsearch its required to do a ABI scan pass
-
-Start with
-```
-ABI_CACHE_MODE: 'true',
-FETCH_BLOCK: 'false',
-FETCH_TRACES: 'false',
-INDEX_DELTAS: 'false',
-INDEX_ALL_DELTAS: 'false',
-```
-
-When indexing is finished, change the settings back and restart the indexer. In case you do not have much contract updates, you do not need to run a full pass.
-
-Tune your configs to your specific hardware using the following settings:
-```
-BATCH_SIZE
-READERS
-DESERIALIZERS
-DS_MULT
-ES_IDX_QUEUES
-ES_AD_IDX_QUEUES
-READ_PREFETCH
-BLOCK_PREFETCH
-INDEX_PREFETCH
-```
-

README.md

@@ -1,177 +1,54 @@
-# Hyperion History API - Private
+# Hyperion History API
+
+<img height="64" src="https://eosrio.io/hyperion.png">
+<br/>
 Scalable Full History API Solution for EOSIO based blockchains
 
 Made with ♥ by [EOS Rio](https://eosrio.io/)
 
-### Introducing an storage-optimized action format for EOSIO
+Official documentation: https://hyperion.docs.eosrio.io/
 
-The original *history_plugin* bundled with eosio, that provided the v1 api, stored inline action traces nested inside their root actions. This led to an excessive amount of data being stored and also transferred whenever a user requested the action history for a given account. Also inline actions are used as a "event" mechanism to notify parties on a transaction. Based on those Hyperion implements some changes
+### 1. Overview
 
-1. actions are stored in a flattened format
-2. a parent field is added to the inline actions to point to the parent global sequence
-3. if the inline action data is identical to the parent it is considered a notification and thus removed from the database
-4. no blocks or transaction data is stored, all information can be reconstructed from actions
+Hyperion is a full history solution for indexing, storing and retrieving EOSIO blockchain`s historical data. EOSIO protocol is highly scalable reaching up to tens of thousands of transactions per second demanding high performance indexing and optimized storage and querying solutions. Hyperion is developed to tackle those challenges providing open source software to be operated by block producers, infrastructure providers and dApp developers.
 
-With those changes the API format focus on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers. 
+Focused on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers, Hyperion implements an improved data structure
+actions are stored in a flattened format
+a parent field is added to the inline actions to point to the parent global sequence
+if the inline action data is identical to the parent it is considered a notification and thus removed from the database
+no blocks or transaction data is stored, all information can be reconstructed from actions
 
-#### Action Data Structure (work in progress)
 
- - `@timestamp` - block time
- - `global_sequence` - unique action global_sequence, used as index id
- - `parent` - points to the parent action (in the case of an inline action) or equal to 0 if root level
- - `block_num` - block number where the action was processed
- - `trx_id` - transaction id
- - `producer` - block producer
- - `act`
-    - `account` - contract account
-    - `name` - contract method name
-    - `authorization` - array of signers
-        - `actor` - signing actor
-        - `permission` - signing permission
-    - `data` - action data input object
- - `account_ram_deltas` - array of ram deltas and payers
-    - `account`
-    - `delta`
- - `notified` - array of accounts that were notified (via inline action events)
+### 2. Architecture
+The following components are required in order to have a fully functional Hyperion API deployment, 
+for small use cases its fine to run all components on a single machine. But for larger chains and 
+production environments we recommend setting them up into different servers under a high-speed local network.
 
-## Dependencies
+#### 2.1 - Elasticsearch Cluster
+The ES cluster is responsible for storing all indexed data.
+Direct access to the Hyperion API and Indexer must be provided. We recommend nodes in the 
+cluster to have at least 32GB of RAM and 8 cpu cores. SSD/NVME drives are recommended for 
+maximum indexing throughput. For production environments a multi-node cluster is highly recommended.
 
-This setup has only been tested with Ubuntu 18.04, but should work with other OS versions too
+#### 2.2 - Hyperion Indexer
+The Indexer is a Node.js based app that process data from the state history plugin and allows it to be indexed.
+The PM2 process manager is used to launch and operate the indexer. The configuration flexibility is very extensive, 
+so system recommendations will depend on the use case and data load. It will require access to at least one ES node, 
+RabbitMQ and the state history node.
 
- - [Elasticsearch 7.5.X](https://www.elastic.co/downloads/elasticsearch)
- - [RabbitMQ](https://www.rabbitmq.com/install-debian.html)
- - [Redis](https://redis.io/topics/quickstart)
- - [Node.js v13](https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions)
- - [PM2](http://pm2.keymetrics.io/docs/usage/quick-start/)
- - Nodeos w/ state_history_plugin and chain_api_plugin
-  
-  > The indexer requires redis, pm2 and node.js to be on the same machine. Other dependencies might be installed on other machines, preferably over a very high speed and low latency network. Indexing speed will vary greatly depending on this configuration.
-  
-## Setup Instructions
+#### 2.3 - Hyperion API
+Parallelizable API server that provides the V2 and V1 (legacy history plugin) endpoints.
+It is launched by PM2 and can also operate in cluster mode. It requires direct access to 
+at least one ES node for the queries and all other services for full healthcheck
 
-Install, configure and test all dependencies above before continuing
+#### 2.4 - RabbitMQ
+Use as messaging queue and data transport between the indexer stages
 
-Read the step-by-step instructions here - [INSTALL.md](https://github.com/eosrio/Hyperion-History-API/blob/master/INSTALL.md)
+#### 2.5 - EOSIO State History
+Nodeos plugin used to collect action traces and state deltas. Provides data via websocket to the indexer
 
-#### 1. Clone & Install packages
-```bash
-git clone https://github.com/eosrio/Hyperion-History-API.git
-cd Hyperion-History-API
-npm install
-```
+### 3. How to use
 
-#### 2. Edit configs
-```
-cp example-ecosystem.config.js ecosystem.config.js
-nano ecosystem.config.js
+#### 3.1 [For Providers](https://eosrio.github.io/hyperion-docs/quickstart/)
 
-# Enter connection details here (chain name must match on the ecosystem file)
-cp example-connections.json connections.json
-nano connections.json
-```
-
-connections.json Reference
-```
-{
-  "amqp": {
-    "host": "127.0.0.1:5672", // RabbitMQ Server
-    "api": "127.0.0.1:15672", // RabbitMQ API Endpoint
-    "user": "username",
-    "pass": "password",
-    "vhost": "hyperion" // RabbitMQ vhost
-  },
-  "elasticsearch": {
-    "host": "127.0.0.1:9200",              // Elasticsearch HTTP API Endpoint
-    "ingest_nodes": ["127.0.0.1:9200"],    // List of ES ingest nodes
-    "user": "elastic",
-    "pass": "password"
-  },
-  "redis": {
-    "host": "127.0.0.1",
-    "port": "6379"
-  },
-  "chains": {
-    "eos": { // Chain name (must match on the ecosystem file)
-      "http": "http://127.0.0.1:8888", // Nodeos Chain API Endpoint
-      "ship": "ws://127.0.0.1:8080" // Nodeos State History Endpoint
-    },
-    "other_chain": {...}
-  }
-}
-```
-
-ecosystem.config.js Reference
-```
-CHAIN: 'eos',                          // chain prefix for indexing
-ABI_CACHE_MODE: 'false',               // only cache historical ABIs to redis
-DEBUG: 'false',                        // debug mode - display extra logs for debugging
-LIVE_READER: 'true',                   // enable continuous reading after reaching the head block
-FETCH_DELTAS: 'false',                 // read table deltas
-CREATE_INDICES: 'v1',                  // index suffix to be created, set to false to use existing aliases
-START_ON: 0,                           // start indexing on block (0=disable)
-STOP_ON: 0,                            // stop indexing on block  (0=disable)
-AUTO_STOP: 0,                          // automatically stop Indexer after X seconds if no more blocks are being processed (0=disable)
-REWRITE: 'false',                      // force rewrite the target replay range
-PURGE_QUEUES: 'false',                 // clear rabbitmq queues before starting the indexer
-BATCH_SIZE: 2000,                      // parallel reader batch size in blocks
-QUEUE_THRESH: 8000,                    // queue size limit on rabbitmq
-LIVE_ONLY: 'false',                    // only reads realtime data serially
-FETCH_BLOCK: 'true',                   // Request full blocks from the state history plugin
-FETCH_TRACES: 'true',                  // Request traces from the state history plugin
-PREVIEW: 'false',                      // preview mode - prints worker map and exit
-DISABLE_READING: 'false',              // completely disable block reading, for lagged queue processing
-READERS: 3,                            // parallel state history readers
-DESERIALIZERS: 4,                      // deserialization queues
-DS_MULT: 4,                            // deserialization threads per queue
-ES_IDX_QUEUES: 4,              // elastic indexers per queue
-ES_AD_IDX_QUEUES: 2,                      // multiplier for action indexing queues
-READ_PREFETCH: 50,                     // Stage 1 prefecth size
-BLOCK_PREFETCH: 5,                     // Stage 2 prefecth size
-INDEX_PREFETCH: 500,                   // Stage 3 prefetch size
-ENABLE_INDEXING: 'true',               // enable elasticsearch indexing
-INDEX_DELTAS: 'true',                  // index common table deltas (see delta on definitions/mappings)
-INDEX_ALL_DELTAS: 'false'              // index all table deltas (WARNING)
-```
- 
- #### 3. Starting
- 
- `./run.sh Indexer`
- or
- ```
- pm2 start --only Indexer --update-env
- pm2 logs Indexer
- ```
- 
- #### 4. Stopping
- 
- Stop reading and wait for queues to flush
- ```
- pm2 trigger Indexer stop
- ```
- 
- Force stop
- ```
- pm2 stop Indexer
- ```
- 
- #### 5. Starting the API node
- 
- `./run.sh API`
- or
- ```
- pm2 start --only API --update-env
- pm2 logs API
- ```
- 
-## API Reference
-
-Documentation is automatically generated by Swagger/OpenAPI.
-
-Example: [OpenAPI Docs](https://eos.hyperion.eosrio.io/v2/docs)
-
-### Roadmap
-
-- Table deltas storage & queries (in progress)
-- Real-time streaming support (in progress)
-- Plugin system (in progress)
-- Control GUI
+#### 3.2 [For Developers](https://eosrio.github.io/hyperion-docs/howtouse/)

api/helpers/functions.ts

@@ -2,181 +2,432 @@ import {createHash} from "crypto";
 import * as _ from "lodash";
 import {FastifyInstance, FastifyReply, FastifyRequest, HTTPMethod, RouteSchema} from "fastify";
 import {ServerResponse} from "http";
+import got from "got";
 
 export function extendResponseSchema(responseProps: any) {
-    const props = {
-        query_time_ms: {type: "number"},
-        cached: {type: "boolean"},
-        lib: {type: "number"},
-        total: {
-            type: "object",
-            properties: {
-                value: {type: "number"},
-                relation: {type: "string"}
-            }
-        }
-    };
-    for (const p in responseProps) {
-        if (responseProps.hasOwnProperty(p)) {
-            props[p] = responseProps[p];
-        }
-    }
-    return {
-        200: {
-            type: 'object',
-            properties: props
-        }
-    };
+	const props = {
+		query_time_ms: {type: "number"},
+		cached: {type: "boolean"},
+		hot_only: {type: "boolean"},
+		lib: {type: "number"},
+		total: {
+			type: "object",
+			properties: {
+				value: {type: "number"},
+				relation: {type: "string"}
+			}
+		}
+	};
+	for (const p in responseProps) {
+		if (responseProps.hasOwnProperty(p)) {
+			props[p] = responseProps[p];
+		}
+	}
+	return {
+		200: {
+			type: 'object',
+			properties: props
+		}
+	};
 }
 
-export function extendQueryStringSchema(queryParams: any) {
-    const params = {
-        limit: {
-            description: 'limit of [n] results per page',
-            type: 'integer',
-            minimum: 1
-        },
-        skip: {
-            description: 'skip [n] results',
-            type: 'integer',
-            minimum: 0
-        }
-    };
-    for (const p in queryParams) {
-        if (queryParams.hasOwnProperty(p)) {
-            params[p] = queryParams[p];
-        }
-    }
-    return {
-        type: 'object',
-        properties: params
-    }
+export function extendQueryStringSchema(queryParams: any, required?: string[]) {
+	const params = {
+		limit: {
+			description: 'limit of [n] results per page',
+			type: 'integer',
+			minimum: 1
+		},
+		skip: {
+			description: 'skip [n] results',
+			type: 'integer',
+			minimum: 0
+		}
+	};
+	for (const p in queryParams) {
+		if (queryParams.hasOwnProperty(p)) {
+			params[p] = queryParams[p];
+		}
+	}
+	const schema = {
+		type: 'object',
+		properties: params
+	}
+	if (required && required.length > 0) {
+		schema["required"] = required;
+	}
+	return schema;
 }
 
 export async function getCacheByHash(redis, key, chain) {
-    const hash = createHash('sha256');
-    const query_hash = hash.update(chain + "-" + key).digest('hex');
-    return [await redis.get(query_hash), query_hash];
+	const hash = createHash('sha256');
+	const query_hash = hash.update(chain + "-" + key).digest('hex');
+	return [await redis.get(query_hash), query_hash];
 }
 
 export function mergeActionMeta(action) {
-    const name = action.act.name;
-    if (action['@' + name]) {
-        action['act']['data'] = _.merge(action['@' + name], action['act']['data']);
-        delete action['@' + name];
-    }
-    action['timestamp'] = action['@timestamp'];
-    // delete action['@timestamp'];
+	const name = action.act.name;
+	if (action['@' + name]) {
+		action['act']['data'] = _.merge(action['@' + name], action['act']['data']);
+		delete action['@' + name];
+	}
+	action['timestamp'] = action['@timestamp'];
+	// delete action['@timestamp'];
 }
 
 export function mergeDeltaMeta(delta: any) {
-    const name = delta.table;
-    if (delta["@" + name]) {
-        delta['data'] = _.merge(delta['@' + name], delta['data']);
-        delete delta['@' + name];
-    }
-    delta['timestamp'] = delta['@timestamp'];
-    delete delta['@timestamp'];
-    return delta;
+	const name = delta.table;
+	if (delta["@" + name]) {
+		delta['data'] = _.merge(delta['@' + name], delta['data']);
+		delete delta['@' + name];
+	}
+	delta['timestamp'] = delta['@timestamp'];
+	delete delta['@timestamp'];
+	return delta;
 }
 
-export function setCacheByHash(fastify, hash, response) {
-    if (fastify.manager.config.api.enable_caching) {
-        fastify.redis
-            .set(hash,
-                JSON.stringify(response),
-                'EX',
-                fastify.manager.config.api.cache_life)
-            .catch(console.log);
-    }
+export function setCacheByHash(fastify, hash, response, expiration?: number) {
+	if (fastify.manager.config.api.enable_caching) {
+		let exp;
+		if (expiration) {
+			exp = expiration;
+		} else {
+			exp = fastify.manager.config.api.cache_life;
+		}
+		fastify.redis.set(hash, JSON.stringify(response), 'EX', exp).catch(console.log);
+	}
 }
 
 export function getRouteName(filename: string) {
-    const arr = filename.split("/");
-    return arr[arr.length - 2];
+	const arr = filename.split("/");
+	return arr[arr.length - 2];
 }
 
 export function addApiRoute(
-    fastifyInstance: FastifyInstance,
-    method: HTTPMethod | HTTPMethod[],
-    routeName: string,
-    routeBuilder: (fastify: FastifyInstance, route: string) => (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => Promise<void>, schema: RouteSchema) {
-    fastifyInstance.route({
-        url: '/' + routeName,
-        method,
-        handler: routeBuilder(fastifyInstance, routeName),
-        schema
-    });
+	fastifyInstance: FastifyInstance,
+	method: HTTPMethod | HTTPMethod[],
+	routeName: string,
+	routeBuilder: (fastify: FastifyInstance, route: string) => (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => Promise<void>, schema: RouteSchema) {
+	fastifyInstance.route({
+		url: '/' + routeName,
+		method,
+		handler: routeBuilder(fastifyInstance, routeName),
+		schema
+	});
 }
 
 export async function getCachedResponse(server: FastifyInstance, route: string, key: any) {
-    const chain = server.manager.chain;
-    let resp, hash;
-    if (server.manager.config.api.enable_caching) {
-        const keystring = JSON.stringify(key);
-        [resp, hash] = await getCacheByHash(server.redis, route + keystring, chain);
-        if (resp) {
-            resp = JSON.parse(resp);
-            resp['cached'] = true;
-            return [resp, hash];
-        } else {
-            return [null, hash];
-        }
-    } else {
-        return [null, null];
-    }
+	const chain = server.manager.chain;
+	let resp, hash;
+	if (server.manager.config.api.enable_caching) {
+		[resp, hash] = await getCacheByHash(server.redis, route + JSON.stringify(key), chain);
+		if (resp) {
+			resp = JSON.parse(resp);
+			resp['cached'] = true;
+			return [resp, hash];
+		} else {
+			return [null, hash];
+		}
+	} else {
+		return [null, null];
+	}
 }
 
 export function getTrackTotalHits(query) {
-    let trackTotalHits: number | boolean = 10000;
-    if (query?.track) {
-        if (query.track === 'true') {
-            trackTotalHits = true;
-        } else if (query.track === 'false') {
-            trackTotalHits = false;
-        } else {
-            trackTotalHits = parseInt(query.track, 10);
-            if (isNaN(trackTotalHits)) {
-                throw new Error('failed to parse track param');
-            }
-        }
-    }
-    return trackTotalHits;
+	let trackTotalHits: number | boolean = 10000;
+	if (query?.track) {
+		if (query.track === 'true') {
+			trackTotalHits = true;
+		} else if (query.track === 'false') {
+			trackTotalHits = false;
+		} else {
+			const parsed = parseInt(query.track, 10);
+			if (parsed > 0) {
+				trackTotalHits = parsed;
+			} else {
+				throw new Error('failed to parse track param');
+			}
+		}
+	}
+	return trackTotalHits;
 }
 
 function bigint2Milliseconds(input: bigint) {
-    return parseFloat((parseInt(input.toString()) / 1000000).toFixed(3));
+	return parseFloat((parseInt(input.toString()) / 1000000).toFixed(3));
+}
+
+const defaultRouteCacheMap = {
+	get_resource_usage: 3600,
+	get_creator: 3600 * 24
 }
 
 export async function timedQuery(
-    queryFunction: (fastify: FastifyInstance, request: FastifyRequest) => Promise<any>,
-    fastify: FastifyInstance, request: FastifyRequest, route: string): Promise<any> {
+	queryFunction: (fastify: FastifyInstance, request: FastifyRequest) => Promise<any>,
+	fastify: FastifyInstance, request: FastifyRequest, route: string): Promise<any> {
 
-    // get reference time in nanoseconds
-    const t0 = process.hrtime.bigint();
+	// get reference time in nanoseconds
+	const t0 = process.hrtime.bigint();
 
-    // check for cached data, return the response hash if caching is enabled
-    const [cachedResponse, hash] = await getCachedResponse(fastify, route, request.query);
+	// check for cached data, return the response hash if caching is enabled
+	const [cachedResponse, hash] = await getCachedResponse(
+		fastify,
+		route,
+		request.req.method === 'POST' ? request.body : request.query
+	);
 
-    if (cachedResponse) {
-        // add cached query time
-        cachedResponse['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
-        return cachedResponse;
-    }
+	if (cachedResponse && !request.query.ignoreCache) {
+		// add cached query time
+		cachedResponse['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
+		return cachedResponse;
+	}
 
-    // call query function
-    const response = await queryFunction(fastify, request);
+	// call query function
+	const response = await queryFunction(fastify, request);
 
-    // save response to cash
-    if (hash) {
-        setCacheByHash(fastify, hash, response);
-    }
+	// save response to cash
+	if (hash) {
+		let EX = null;
+		if (defaultRouteCacheMap[route]) {
+			EX = defaultRouteCacheMap[route];
+		}
+		setCacheByHash(fastify, hash, response, EX);
+	}
 
-    // add normal query time
-    if (response) {
-        response['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
-        return response;
-    } else {
-        return {};
-    }
+	// add normal query time
+	if (response) {
+		response['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
+		return response;
+	} else {
+		return {};
+	}
+}
+
+export function chainApiHandler(fastify: FastifyInstance) {
+	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+		await handleChainApiRedirect(request, reply, fastify);
+	}
+}
+
+export async function handleChainApiRedirect(
+	request: FastifyRequest,
+	reply: FastifyReply<ServerResponse>,
+	fastify: FastifyInstance
+) {
+	const urlParts = request.req.url.split("?");
+	let reqUrl = fastify.chain_api + urlParts[0];
+
+	if (urlParts[0] === '/v1/chain/push_transaction' && fastify.push_api && fastify.push_api !== "") {
+		reqUrl = fastify.push_api + urlParts[0];
+	}
+
+	const opts = {};
+
+	if (request.req.method === 'POST') {
+		if (request.body) {
+			if (typeof request.body === 'string') {
+				opts['body'] = request.body;
+			} else if (typeof request.body === 'object') {
+				opts['body'] = JSON.stringify(request.body);
+			}
+		} else {
+			opts['body'] = "";
+		}
+	} else if (request.req.method === 'GET') {
+		opts['json'] = request.query;
+	}
+
+	try {
+		const apiResponse = await got.post(reqUrl, opts);
+		reply.headers({"Content-Type": "application/json"});
+		if (request.req.method === 'HEAD') {
+			reply.headers({"Content-Length": apiResponse.body.length});
+			reply.send("");
+		} else {
+			reply.send(apiResponse.body);
+		}
+	} catch (error) {
+		if (error.response) {
+			reply.status(error.response.statusCode).send(error.response.body);
+		} else {
+			console.log(error);
+			reply.status(500).send();
+		}
+		if (fastify.manager.config.api.chain_api_error_log) {
+			try {
+				if (error.response) {
+					const error_msg = JSON.parse(error.response.body).error.details[0].message;
+					console.log(`endpoint: ${request.req.url} | status: ${error.response.statusCode} | error: ${error_msg}`);
+				} else {
+					console.log(error);
+				}
+				// if (request.req.url === '/v1/chain/push_transaction') {
+				//     const packedTrx = JSON.parse(opts['body']).packed_trx;
+				//     const trxBuffer = Buffer.from(packedTrx, 'hex');
+				//     const trxData = await fastify.eosjs.api.deserializeTransactionWithActions(trxBuffer);
+				//     console.log(trxData);
+				// }
+			} catch (e) {
+				console.log(e);
+			}
+		}
+	}
+}
+
+export function addChainApiRoute(fastify: FastifyInstance, routeName, description, props?, required?) {
+	const baseSchema = {
+		description: description,
+		summary: description,
+		tags: ['chain']
+	};
+	addApiRoute(
+		fastify,
+		['GET', 'HEAD'],
+		routeName,
+		chainApiHandler,
+		{
+			...baseSchema,
+			querystring: props ? {
+				type: 'object',
+				properties: props,
+				required: required
+			} : undefined
+		}
+	);
+	addApiRoute(
+		fastify,
+		'POST',
+		routeName,
+		chainApiHandler,
+		{
+			...baseSchema,
+			body: props ? {
+				type: ['object', 'string'],
+				properties: props,
+				required: required
+			} : undefined
+		}
+	);
+}
+
+export function addSharedSchemas(fastify: FastifyInstance) {
+	fastify.addSchema({
+		$id: "WholeNumber",
+		description: "A whole number",
+		anyOf: [
+			{
+				type: "string",
+				pattern: "^\\d+$"
+			},
+			{
+				type: "integer"
+			}
+		],
+	});
+
+	fastify.addSchema({
+		$id: "Symbol",
+		type: "string",
+		description: "A symbol composed of capital letters between 1-7.",
+		pattern: "^([A-Z]{1,7})$",
+		title: "Symbol"
+	});
+
+	fastify.addSchema({
+		$id: "Signature",
+		type: "string",
+		description: "String representation of an EOSIO compatible cryptographic signature",
+		pattern: "^SIG_([RK]1|WA)_[1-9A-HJ-NP-Za-km-z]+$",
+		title: "Signature"
+	})
+
+	fastify.addSchema({
+		$id: "AccountName",
+		"anyOf": [
+			{
+				"type": "string",
+				"description": "String representation of privileged EOSIO name type",
+				"pattern": "^(eosio[\\.][a-z1-5]{1,6})([a-j]{1})?$",
+				"title": "NamePrivileged"
+			},
+			{
+				"type": "string",
+				"description": "String representation of basic EOSIO name type, must be 12 characters and contain only a-z and 0-5",
+				"pattern": "^([a-z]{1}[a-z1-5]{11})([a-j]{1})?$",
+				"title": "NameBasic"
+			},
+			{
+				"type": "string",
+				"description": "String representation of EOSIO bid name type, 1-12 characters and only a-z and 0-5 are allowed",
+				"pattern": "^([a-z1-5]{1,12})([a-j]{1})?$",
+				"title": "NameBid"
+			},
+			{
+				"type": "string",
+				"description": "String representation of EOSIO name type",
+				"pattern": "^([a-z1-5]{1}[a-z1-5\\.]{0,10}[a-z1-5]{1})([a-j]{1})?$",
+				"title": "NameCatchAll"
+			}
+		],
+		"title": "Name"
+	});
+
+	fastify.addSchema({
+		$id: "Expiration",
+		"description": "Time that transaction must be confirmed by.",
+		"type": "string",
+		"pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$",
+		"title": "DateTime"
+	});
+
+	fastify.addSchema({
+		$id: "BlockExtensions",
+		"type": "array",
+		"items": {
+			"anyOf": [{"type": "integer"}, {"type": "string"}]
+		},
+		"title": "Extension"
+	})
+
+	fastify.addSchema({
+		$id: "ActionItems",
+		"type": "object",
+		"additionalProperties": false,
+		"minProperties": 5,
+		"required": [
+			"account",
+			"name",
+			"authorization",
+			"data",
+			"hex_data"
+		],
+		"properties": {
+			"account": 'AccountName#',
+			"name": 'AccountName#',
+			"authorization": {
+				"type": "array",
+				"items": {
+					"type": "object",
+					"additionalProperties": false,
+					"minProperties": 2,
+					"required": [
+						"actor",
+						"permission"
+					],
+					"properties": {
+						"actor": 'AccountName#',
+						"permission": 'AccountName#'
+					},
+					"title": "Authority"
+				}
+			},
+			"data": {
+				"type": "object",
+				"additionalProperties": true
+			},
+			"hex_data": {
+				"type": "string"
+			}
+		},
+		"title": "Action"
+	});
 }

api/routes.ts

@@ -2,15 +2,17 @@ import * as fastify_static from "fastify-static";
 import {join} from "path";
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {ServerResponse} from "http";
-import {createReadStream} from "fs";
+import {createReadStream, existsSync, readFileSync, unlinkSync} from "fs";
 import * as AutoLoad from "fastify-autoload";
+import {addSharedSchemas, handleChainApiRedirect} from "./helpers/functions";
+import got from "got";
 
 function addRedirect(server: FastifyInstance, url: string, redirectTo: string) {
     server.route({
         url,
         method: 'GET',
         schema: {hide: true},
-        handler: async (request, reply) => {
+        handler: async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
             reply.redirect(redirectTo);
         }
     });
@@ -26,31 +28,159 @@ function addRoute(server: FastifyInstance, handlersPath: string, prefix: string)
 
 export function registerRoutes(server: FastifyInstance) {
 
+    // build internal map of routes
+    const routeSet = new Set<string>();
+    server.decorate('routeSet', routeSet);
+    const ignoreList = [
+        '/v2',
+        '/v2/history',
+        '/v2/state',
+        '/v1/chain/*',
+        '/v1/chain',
+    ];
+    server.addHook('onRoute', opts => {
+        if (!ignoreList.includes(opts.url)) {
+            if (opts.url.startsWith('/v') && !opts.url.startsWith('/v2/explore') && !opts.url.startsWith('/v2/docs')) {
+                routeSet.add(opts.url);
+            }
+        }
+    });
+
     // Register fastify api routes
     addRoute(server, 'v2', '/v2');
     addRoute(server, 'v2-history', '/v2/history');
     addRoute(server, 'v2-state', '/v2/state');
+    addRoute(server, 'v2-stats', '/v2/stats');
+
+    // legacy routes
     addRoute(server, 'v1-history', '/v1/history');
-    // addRoute(server,'v1-chain', '/v1/chain');
+    addRoute(server, 'v1-trace', '/v1/trace_api');
+
+    addSharedSchemas(server);
+
+    // chain api redirects
+    addRoute(server, 'v1-chain', '/v1/chain');
+    server.route({
+        url: '/v1/chain/*',
+        method: ["GET", "POST"],
+        schema: {
+            summary: "Wildcard chain api handler",
+            tags: ["chain"]
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+            await handleChainApiRedirect(request, reply, server);
+        }
+    });
+
+    // /v1/node/get_supported_apis
+    server.route({
+        url: '/v1/node/get_supported_apis',
+        method: ["GET"],
+        schema: {
+            summary: "Get list of supported APIs",
+            tags: ["node"]
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+            const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
+            if (data.apis && data.apis.length > 0) {
+                const apiSet = new Set(server.routeSet);
+                data.apis.forEach((a) => apiSet.add(a));
+                reply.send({apis: [...apiSet]});
+            } else {
+                reply.send({apis: [...server.routeSet], error: 'nodeos did not send any data'});
+            }
+        }
+    });
+
+    server.addHook('onError', (request, reply, error, done) => {
+        console.log(`[${request.req.headers['x-real-ip']}] ${request.req.method} ${request.req.url} failed with error: ${error.message}`);
+        done();
+    });
+
+    // server.addHook('onResponse', (request, reply, done) => {
+    //     if (reply.res.statusCode !== 200) {
+    //         console.log(`${request.req.url} - code: ${reply.res.statusCode}`);
+    //     }
+    //     done();
+    // });
 
     // Serve integrated explorer
-    if(server.manager.config.api.enable_explorer) {
+    if (server.manager.config.api.enable_explorer) {
+
+        server.register(require('fastify-compress'), {global: false});
+
+        try {
+            const _data = readFileSync(join(__dirname, '..', 'hyperion-explorer', 'src', 'manifest.webmanifest'));
+            const tempPath = join(__dirname, '..', 'hyperion-explorer', 'dist', 'manifest.webmanifest');
+            if (existsSync(tempPath)) {
+                unlinkSync(tempPath);
+            }
+            const baseManifest = JSON.parse(_data.toString());
+            baseManifest.name = `Hyperion Explorer - ${server.manager.config.api.chain_name}`;
+            baseManifest.short_name = baseManifest.name;
+            server.get('/v2/explore/manifest.webmanifest', (request, reply) => {
+                reply.send(baseManifest);
+            });
+        } catch (e) {
+            console.log(e);
+        }
+
         server.register(fastify_static, {
             root: join(__dirname, '..', 'hyperion-explorer', 'dist'),
             redirect: true,
-            wildcard: true,
-            prefix: '/v2/explore'
+            wildcard: false,
+            prefix: '/v2/explore',
+            setHeaders: (res: ServerResponse, path) => {
+                if (path.endsWith('/ngsw-worker.js')) {
+                    res.setHeader('Service-Worker-Allowed', '/');
+                }
+            }
         });
+
+        server.get(
+            '/v2/explore/**/*',
+            {
+                schema: {
+                    tags: ['internal']
+                }
+            },
+            (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+                reply.sendFile('index.html', join(__dirname, '..', 'hyperion-explorer', 'dist'));
+            }
+        );
+
+        server.get(
+            '/v2/explorer_metadata',
+            {
+                schema: {
+                    tags: ['internal']
+                }
+            },
+            (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+                reply.send({
+                    logo: server.manager.config.api.chain_logo_url,
+                    provider: server.manager.config.api.provider_name,
+                    provider_url: server.manager.config.api.provider_url,
+                    chain_name: server.manager.config.api.chain_name,
+                    chain_id: server.manager.conn.chains[server.manager.chain].chain_id,
+                    custom_core_token: server.manager.config.api.custom_core_token
+                });
+            });
     }
 
-    // steam client lib
-    server.get('/stream-client.js', (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        const stream = createReadStream('./client_bundle.js');
-        reply.type('application/javascript').send(stream);
-    });
+    if (server.manager.config.features.streaming) {
+        // steam client lib
+        server.get('/stream-client.js', {schema: {tags: ['internal']}}, (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+            const stream = createReadStream('./client_bundle.js');
+            reply.type('application/javascript').send(stream);
+        });
+    }
 
     // Redirect routes to documentation
     addRedirect(server, '/v2', '/v2/docs');
     addRedirect(server, '/v2/history', '/v2/docs/index.html#/history');
     addRedirect(server, '/v2/state', '/v2/docs/index.html#/state');
+    addRedirect(server, '/v1/chain', '/v2/docs/index.html#/chain');
+    addRedirect(server, '/explorer', '/v2/explore');
+    addRedirect(server, '/explore', '/v2/explore');
 }

api/routes/blockStore.js

@@ -1,227 +0,0 @@
-const async = require('async');
-const {Serialize} = require('eosjs');
-const {ConnectionManager} = require('../../connections/manager');
-let abi, types;
-let connected = false;
-let tables = new Map();
-const textEncoder = new TextEncoder();
-const textDecoder = new TextDecoder();
-const ws_opts = {compress: true, binary: true, mask: true, fin: true};
-const maxMessagesInFlight = 1;
-const blockReadingQueue = async.cargo(async.ensureAsync(processIncomingBlockArray), maxMessagesInFlight);
-
-class BlockStore {
-    localBlocks;
-    pendingBlocks;
-    pendingRequests;
-    constructor() {
-        this.pendingRequests = [];
-        this.localBlocks = new Map();
-        this.pendingBlocks = new Map();
-    }
-
-    addBlock(num, data) {
-        if (this.pendingBlocks.has(num)) {
-            this.localBlocks.set(num, data);
-            this.pendingBlocks.get(num)['callback']();
-            this.pendingBlocks.delete(num);
-        }
-    }
-
-    requestRange(from_block, to_block, onBlock, onCompletion) {
-        // console.log(`Requesting blocks from ${from_block} to ${to_block}`);
-        const range = to_block - from_block;
-        // console.log(`Range size: ${range}`);
-        let requested = range;
-        for (let i = 0; i < range; i++) {
-            const blk_num = from_block + i;
-            // console.log('Looking on local cache for block ' + blk_num);
-            if (this.localBlocks.has(blk_num)) {
-                onBlock(this.localBlocks.get(blk_num), true);
-                requested--;
-            } else {
-                this.pendingBlocks.set(blk_num, {
-                    callback: () => {
-                        onBlock(this.localBlocks.get(blk_num), false);
-                    }
-                });
-            }
-        }
-        if (requested > 0) {
-            requestBlockRange(from_block, to_block);
-            const checker = setInterval(() => {
-                if (this.pendingBlocks.size === 0) {
-                    onCompletion();
-                    clearInterval(checker);
-                }
-            }, 30);
-        } else {
-            onCompletion();
-        }
-    }
-}
-
-const blockStore = new BlockStore();
-
-const baseRequest = {
-    max_messages_in_flight: maxMessagesInFlight,
-    have_positions: [],
-    irreversible_only: false,
-    fetch_block: true,
-    fetch_traces: true,
-    fetch_deltas: false
-};
-
-const manager = new ConnectionManager();
-const shipClient = manager.shipClient;
-startWS();
-
-function createSerialBuffer(array_data) {
-    return new Serialize.SerialBuffer({textEncoder, textDecoder, array: array_data});
-}
-
-
-function serialize(type, value) {
-    const buffer = createSerialBuffer();
-    Serialize.getType(types, type).serialize(buffer, value);
-    return buffer.asUint8Array();
-}
-
-function deserialize(type, array) {
-    const buffer = new Serialize.SerialBuffer({textEncoder, textDecoder, array});
-    const sState = new Serialize.SerializerState({bytesAsUint8Array: true});
-    return Serialize.getType(types, type).deserialize(buffer, sState);
-}
-
-function processIncomingBlockArray(payload, cb) {
-    processIncomingBlocks(payload).then(() => {
-        cb();
-    }).catch((err) => {
-        console.log('FATAL ERROR READING BLOCKS', err);
-        process.exit(1);
-    })
-}
-
-function handleLostConnection() {
-    console.log('connection lost');
-    connected = false;
-    startWS();
-}
-
-function ackBlocks(num) {
-    shipClient.send(serialize('request',
-        ['get_blocks_ack_request_v0', {num_messages: num}]
-    ), ws_opts, (err) => {
-        if (err) console.warn(err);
-    });
-}
-
-async function processIncomingBlocks(block_array) {
-    if (abi) {
-        ackBlocks(block_array.length);
-        for (const block of block_array) {
-            try {
-                await onMessage(block);
-            } catch (e) {
-                console.log(e);
-            }
-        }
-    } else {
-        await onMessage(block_array[0]);
-    }
-    return true;
-}
-
-function processFirstABI(data) {
-    abi = JSON.parse(data);
-    types = Serialize.getTypesFromAbi(Serialize.createInitialTypes(), abi);
-    abi.tables.map(table => tables.set(table.name, table.type));
-}
-
-function deserializeType(obj) {
-    return deserialize(obj[0], obj[1], textEncoder, textDecoder, types);
-}
-
-function onMessage(data) {
-    if (abi) {
-        const res = deserialize('result', data)[1];
-        if (res['this_block']) {
-            const blk_num = res['this_block']['block_num'];
-            const block_data = {
-                block: null,
-                traces: [],
-                deltas: []
-            };
-            if (res.block && res.block.length) {
-                block_data.block = deserialize('signed_block', res.block, textEncoder, textDecoder, types);
-                if (block_data.block === null) {
-                    console.log(res);
-                } else {
-                    block_data.block.id = res['this_block']['block_id'];
-                    block_data.block.block_num = blk_num;
-                    block_data.block.transactions = [];
-
-                    if (res['traces'] && res['traces'].length) {
-                        const trx_traces = deserialize('transaction_trace[]', res['traces'], textEncoder, textDecoder, types).map(trace => trace[1]);
-                        for (const trace of trx_traces) {
-                            trace.partial = trace.partial[1];
-                            trace.actions = [];
-                            for (const action_trace of trace.action_traces) {
-                                // console.log(action_trace);
-                                // action_trace.receipt = action_trace.receipt[1];
-                                const action = action_trace[1];
-                                action.receipt = action.receipt[1];
-                                action['act']['data'] = Buffer.from(action['act']['data']).toString('hex');
-                                trace.actions.push(action);
-                            }
-                            delete trace.action_traces;
-                            if (trace.actions[0]['act']['account'] === 'eosio' && trace.actions[0]['act']['name'] === 'onblock') {
-                                // ingore onblock
-                            } else {
-                                block_data.block.transactions.push(trace);
-                            }
-                        }
-                    }
-
-                }
-            }
-
-            // if (res['deltas'] && res['deltas'].length) {
-            //     block_data.deltas = deserialize('table_delta[]', res['deltas'], textEncoder, textDecoder, types);
-            // }
-            blockStore.addBlock(blk_num, block_data.block);
-        }
-    } else {
-        processFirstABI(data);
-        return 1;
-    }
-}
-
-function send(req_data) {
-    shipClient.send(serialize('request', req_data, textEncoder, textDecoder, types));
-}
-
-function requestBlockRange(start, finish) {
-    const request = baseRequest;
-    request.start_block_num = start;
-    request.end_block_num = finish;
-    // console.log(request);
-    if (connected) {
-        send(['get_blocks_request_v0', request]);
-    } else {
-        startWS(() => {
-            send(['get_blocks_request_v0', request]);
-        });
-    }
-}
-
-function startWS(onConnect) {
-    if (onConnect) {
-        shipClient.connect(blockReadingQueue.push, handleLostConnection, null, onConnect);
-    } else {
-        shipClient.connect(blockReadingQueue.push, handleLostConnection);
-    }
-    connected = true;
-}
-
-module.exports = {blockStore};

api/routes/history/getAbiSnapshot.js

@@ -1,72 +0,0 @@
-const {getCacheByHash} = require("../../helpers/functions");
-const {getAbiSnapshotSchema} = require("../../schemas");
-
-const enable_caching = process.env.ENABLE_CACHING === 'true';
-let cache_life = 30;
-if (process.env.CACHE_LIFE) {
-    cache_life = parseInt(process.env.CACHE_LIFE);
-}
-
-async function getAbiSnapshot(fastify, request) {
-    const t0 = Date.now();
-    const {redis, elastic} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, JSON.stringify(request.query));
-    if (cachedResponse) {
-        return JSON.parse(cachedResponse);
-    }
-
-    const response = {
-        query_time: 0,
-        block_num: null
-    };
-
-    const code = request.query.contract;
-    const block = request.query.block;
-    const should_fetch = request.query.fetch;
-
-    if (should_fetch) {
-        console.log(should_fetch);
-    }
-
-    const mustArray = [];
-
-    mustArray.push({"term": {"account": code}});
-
-    if (block) {
-        mustArray.push({"range": {"block": {"lte": parseInt(block)}}});
-    }
-
-    const results = await elastic.search({
-        index: process.env.CHAIN + '-abi',
-        size: 1,
-        body: {
-            query: {bool: {must: mustArray}},
-            sort: [{block: {order: "desc"}}]
-        }
-    });
-    if (results['body']['hits']['hits'].length > 0) {
-        if (should_fetch) {
-            response['abi'] = JSON.parse(results['body']['hits']['hits'][0]['_source']['abi']);
-        } else {
-            response['present'] = true;
-        }
-        response.block_num = results['body']['hits']['hits'][0]['_source']['block'];
-    } else {
-        response['present'] = false;
-        response['error'] = 'abi not found for ' + code + ' until block ' + block;
-    }
-    response['query_time'] = Date.now() - t0;
-    if (enable_caching) {
-        redis.set(hash, JSON.stringify(response), 'EX', cache_life);
-    }
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get('/get_abi_snapshot', {
-        schema: getAbiSnapshotSchema.GET
-    }, async (request, reply) => {
-        reply.send(await getAbiSnapshot(fastify, request));
-    });
-    next()
-};

api/routes/history/getActions.js

@@ -1,378 +0,0 @@
-const prettyjson = require("prettyjson");
-const {getActionsSchema} = require("../../schemas");
-const {getCacheByHash, mergeActionMeta} = require("../../helpers/functions");
-
-const maxActions = 1000;
-const route = '/get_actions';
-
-const terms = [
-    "notified.keyword",
-    "act.authorization.actor"
-];
-
-const extendedActions = new Set([
-    "transfer",
-    "newaccount",
-    "updateauth",
-    "buyram",
-    "buyrambytes"
-]);
-
-const primaryTerms = [
-    "notified",
-    "block_num",
-    "global_sequence",
-    "producer",
-    "@timestamp",
-    "creator_action_ordinal",
-    "action_ordinal",
-    "cpu_usage_us",
-    "net_usage_words"
-];
-
-const enable_caching = process.env.ENABLE_CACHING === 'true';
-let cache_life = 30;
-if (process.env.CACHE_LIFE) {
-    cache_life = parseInt(process.env.CACHE_LIFE);
-}
-
-function getTrackTotalHits(query) {
-    let trackTotalHits = 15000;
-    if (query.track) {
-        if (query.track === 'true') {
-            trackTotalHits = true;
-        } else if (query.track === 'false') {
-            trackTotalHits = false;
-        } else {
-            trackTotalHits = parseInt(query.track, 10);
-            if (trackTotalHits !== trackTotalHits) {
-                throw new Error('failed to parse track param');
-            }
-        }
-    }
-    return trackTotalHits;
-}
-
-function addSortedBy(query, queryBody, sort_direction) {
-    if (query['sortedBy']) {
-        const opts = query['sortedBy'].split(":");
-        const sortedByObj = {};
-        sortedByObj[opts[0]] = opts[1];
-        queryBody['sort'] = sortedByObj;
-    } else {
-        queryBody['sort'] = {
-            "global_sequence": sort_direction
-        };
-    }
-}
-
-function processMultiVars(queryStruct, parts, field) {
-    const must = [];
-    const mustNot = [];
-
-    parts.forEach(part => {
-        if (part.startsWith("!")) {
-            mustNot.push(part.replace("!", ""));
-        } else {
-            must.push(part);
-        }
-    });
-
-    if (must.length > 1) {
-        queryStruct.bool.must.push({
-            bool: {
-                should: must.map(elem => {
-                    const _q = {};
-                    _q[field] = elem;
-                    return {term: _q}
-                })
-            }
-        });
-    } else if (must.length === 1) {
-        const mustQuery = {};
-        mustQuery[field] = must[0];
-        queryStruct.bool.must.push({term: mustQuery});
-    }
-
-    if (mustNot.length > 1) {
-        queryStruct.bool.must_not.push({
-            bool: {
-                should: mustNot.map(elem => {
-                    const _q = {};
-                    _q[field] = elem;
-                    return {term: _q}
-                })
-            }
-        });
-    } else if (mustNot.length === 1) {
-        const mustNotQuery = {};
-        mustNotQuery[field] = mustNot[0].replace("!", "");
-        queryStruct.bool.must_not.push({term: mustNotQuery});
-    }
-}
-
-function addRangeQuery(queryStruct, prop, pkey, query) {
-    const _termQuery = {};
-    const parts = query[prop].split("-");
-    _termQuery[pkey] = {
-        "gte": parts[0],
-        "lte": parts[1]
-    };
-    queryStruct.bool.must.push({range: _termQuery});
-}
-
-function applyTimeFilter(query, queryStruct) {
-    if (query['after'] || query['before']) {
-        let _lte = "now";
-        let _gte = 0;
-        if (query['before']) {
-            _lte = query['before'];
-            if (!_lte.endsWith("Z")) {
-                _lte += "Z";
-            }
-        }
-        if (query['after']) {
-            _gte = query['after'];
-            if (!_gte.endsWith("Z")) {
-                _gte += "Z";
-            }
-        }
-        if (!queryStruct.bool['filter']) {
-            queryStruct.bool['filter'] = [];
-        }
-        queryStruct.bool['filter'].push({
-            range: {
-                "@timestamp": {
-                    "gte": _gte,
-                    "lte": _lte
-                }
-            }
-        });
-    }
-}
-
-function applyGenericFilters(query, queryStruct) {
-    for (const prop in query) {
-        if (Object.prototype.hasOwnProperty.call(query, prop)) {
-            const pair = prop.split(".");
-            if (pair.length > 1 || primaryTerms.includes(pair[0])) {
-                let pkey;
-                if (pair.length > 1) {
-                    pkey = extendedActions.has(pair[0]) ? "@" + prop : prop;
-                } else {
-                    pkey = prop;
-                }
-                if (query[prop].indexOf("-") !== -1) {
-                    addRangeQuery(queryStruct, prop, pkey, query);
-                } else {
-                    const _termQuery = {};
-                    const parts = query[prop].split(",");
-                    if (parts.length > 1) {
-                        processMultiVars(queryStruct, parts, prop);
-                    } else if (parts.length === 1) {
-                        const andParts = parts[0].split(" ");
-                        if (andParts.length > 1) {
-                            andParts.forEach(value => {
-                                const _q = {};
-                                console.log(value);
-                                _q[pkey] = value;
-                                queryStruct.bool.must.push({term: _q});
-                            });
-                        } else {
-                            if (parts[0].startsWith("!")) {
-                                _termQuery[pkey] = parts[0].replace("!", "");
-                                queryStruct.bool.must_not.push({term: _termQuery});
-                            } else {
-                                _termQuery[pkey] = parts[0];
-                                queryStruct.bool.must.push({term: _termQuery});
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-function makeShouldArray(query) {
-    const should_array = [];
-    for (const entry of terms) {
-        const tObj = {term: {}};
-        tObj.term[entry] = query.account;
-        should_array.push(tObj);
-    }
-    return should_array;
-}
-
-function applyCodeActionFilters(query, queryStruct) {
-    let filterObj = [];
-    if (query.filter) {
-        for (const filter of query.filter.split(',')) {
-            if (filter !== '*:*') {
-                const _arr = [];
-                const parts = filter.split(':');
-                if (parts.length === 2) {
-                    [code, method] = parts;
-                    if (code && code !== "*") {
-                        _arr.push({'term': {'act.account': code}});
-                    }
-                    if (method && method !== "*") {
-                        _arr.push({'term': {'act.name': method}});
-                    }
-                }
-                if (_arr.length > 0) {
-                    filterObj.push({bool: {must: _arr}});
-                }
-            }
-        }
-        if (filterObj.length > 0) {
-            queryStruct.bool['should'] = filterObj;
-            queryStruct.bool['minimum_should_match'] = 1;
-        }
-    }
-}
-
-function getSkipLimit(query) {
-    let skip, limit;
-    skip = parseInt(query.skip, 10);
-    if (skip < 0) {
-        throw new Error('invalid skip parameter');
-    }
-    limit = parseInt(query.limit, 10);
-    if (limit < 1) {
-        throw new Error('invalid limit parameter');
-    }
-    return {skip, limit};
-}
-
-function getSortDir(query) {
-    let sort_direction = 'desc';
-    if (query.sort) {
-        if (query.sort === 'asc' || query.sort === '1') {
-            sort_direction = 'asc';
-        } else if (query.sort === 'desc' || query.sort === '-1') {
-            sort_direction = 'desc'
-        } else {
-            throw new Error('invalid sort direction');
-        }
-    }
-    return sort_direction;
-}
-
-function applyAccountFilters(query, queryStruct) {
-    if (query.account) {
-        queryStruct.bool.must.push({"bool": {should: makeShouldArray(query)}});
-    }
-}
-
-async function getActions(fastify, request) {
-    const t0 = Date.now();
-    const {redis, elastic, eosjs} = fastify;
-    const query = request.query;
-    let cachedResponse, hash;
-    if (enable_caching) {
-        [cachedResponse, hash] = await getCacheByHash(redis, route + JSON.stringify(query));
-        if (cachedResponse) {
-            cachedResponse = JSON.parse(cachedResponse);
-            cachedResponse['query_time'] = Date.now() - t0;
-            cachedResponse['cached'] = true;
-            return cachedResponse;
-        }
-    }
-
-    const queryStruct = {
-        "bool": {
-            must: [],
-            must_not: [],
-            boost: 1.0
-        }
-    };
-
-    const {skip, limit} = getSkipLimit(query);
-
-    const sort_direction = getSortDir(query);
-
-    applyAccountFilters(query, queryStruct);
-
-    applyGenericFilters(query, queryStruct);
-
-    applyTimeFilter(query, queryStruct);
-
-    applyCodeActionFilters(query, queryStruct);
-
-    // allow precise counting of total hits
-    const trackTotalHits = getTrackTotalHits(query);
-
-    // Prepare query body
-    const query_body = {
-        "track_total_hits": trackTotalHits,
-        "query": queryStruct
-    };
-
-    // Include sorting
-    addSortedBy(query, query_body, sort_direction);
-
-    // console.log(prettyjson.render(queryStruct));
-
-    // Perform search
-    const pResults = await Promise.all([eosjs.rpc.get_info(), elastic['search']({
-        "index": process.env.CHAIN + '-action-*',
-        "from": skip || 0,
-        "size": (limit > maxActions ? maxActions : limit) || 10,
-        "body": query_body
-    })]);
-
-    const results = pResults[1]['body']['hits'];
-    const response = {
-        query_time: null,
-        cached: false,
-        lib: pResults[0].last_irreversible_block_num,
-        total: results['total']
-    };
-
-    if (query.simple) {
-        response['simple_actions'] = [];
-    } else {
-        response['actions'] = [];
-    }
-
-    if (results['hits'].length > 0) {
-        const actions = results['hits'];
-        for (let action of actions) {
-            action = action._source;
-            mergeActionMeta(action);
-            if (query.simple) {
-                response.simple_actions.push({
-                    block: action['block_num'],
-                    irreversible: action['block_num'] < pResults[0].last_irreversible_block_num,
-                    timestamp: action['@timestamp'],
-                    transaction_id: action['trx_id'],
-                    actors: action['act']['authorization'].map(a => `${a.actor}@${a.permission}`).join(","),
-                    notified: action['notified'].join(','),
-                    contract: action['act']['account'],
-                    action: action['act']['name'],
-                    data: action['act']['data']
-                });
-            } else {
-                response.actions.push(action);
-            }
-        }
-    }
-
-    response['query_time'] = Date.now() - t0;
-
-    if (enable_caching) {
-        redis.set(hash, JSON.stringify(response), 'EX', cache_life);
-    }
-
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get('/get_actions', {
-        schema: getActionsSchema.GET
-    }, async (request) => {
-        return await getActions(fastify, request);
-    });
-    next()
-};

api/routes/history/getBlocks.js

@@ -1,49 +0,0 @@
-const {getBlocksSchema} = require("../../schemas");
-const {blockStore} = require('../blockStore');
-
-async function getBlocks(fastify, request, reply) {
-    const t0 = Date.now();
-    let from_block, to_block;
-    const {eosjs} = fastify;
-    if (request.query.from) {
-        from_block = parseInt(request.query.from);
-        if (request.query.to) {
-            to_block = parseInt(request.query.to);
-        } else {
-            to_block = from_block + 1;
-        }
-    } else {
-        const head = (await eosjs.rpc.get_info()).head_block_num;
-        from_block = head;
-        to_block = head + 1;
-    }
-
-    if (to_block > from_block) {
-        if (to_block - from_block > 20) {
-            reply.status(403).send('request over limit');
-        } else {
-            const response = {query_time: null, cached: false, blocks: []};
-            console.log('Local Block store size:', blockStore.localBlocks.size);
-            blockStore.requestRange(from_block, to_block, (block, cached) => {
-                if (cached) {
-                    response.cached = true;
-                }
-                response.blocks.push(block);
-            }, () => {
-                response['query_time'] = Date.now() - t0;
-                reply.send(response);
-            });
-        }
-    } else {
-        reply.status(403).send('invalid block range');
-    }
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get('/get_blocks', {
-        schema: getBlocksSchema.GET
-    }, async (request, reply) => {
-        await getBlocks(fastify, request, reply);
-    });
-    next();
-};

api/routes/history/getCreator.js

@@ -1,108 +0,0 @@
-const {getCreatorSchema} = require("../../schemas");
-const {getCacheByHash} = require("../../helpers/functions");
-
-const route = '/get_creator';
-
-async function getActionByGS(client, gs) {
-    console.log(gs);
-    const results = await client['search']({
-        index: process.env.CHAIN + '-action-*',
-        body: {
-            query: {
-                bool: {
-                    must: [
-                        {term: {"global_sequence": gs}}
-                    ]
-                }
-            }
-        }
-    });
-    return results['body']['hits']['hits'][0]['_source'];
-}
-
-async function getCreator(fastify, request) {
-    const {redis, elastic, eosjs} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, route + JSON.stringify(request.query) + 'v3');
-    if (cachedResponse) {
-        return cachedResponse;
-    }
-    const newact = request.query.account;
-    const response = {
-        account: newact,
-        creator: '',
-        timestamp: '',
-        block_num: 0,
-        trx_id: '',
-    };
-    let account_data = null;
-    try {
-        account_data = await eosjs.rpc.get_account(newact);
-    } catch (e) {
-        const err = new Error();
-        err.statusCode = 404;
-        err.message = 'account not found';
-        throw err;
-    }
-    response.timestamp = account_data['created'];
-    const queryBody = {
-        size: 1000,
-        query: {
-            bool: {
-                must: [
-                    {term: {"act.name": "newaccount"}},
-                    {term: {"act.account": "eosio"}},
-                    {term: {"@timestamp": account_data['created']}}
-                ]
-            }
-        }
-    };
-    const results = await elastic['search']({
-        "index": process.env.CHAIN + '-action-*',
-        "body": queryBody
-    });
-    for (const action of results['body']['hits']['hits']) {
-        const actData = action._source.act.data;
-        let valid = false;
-        response.block_num = action._source.block_num;
-        if (actData.newact === newact) {
-            response.creator = actData.creator;
-            response['trx_id'] = action._source['trx_id'];
-            valid = true;
-        } else {
-            if (action._source['@newaccount']) {
-                if (action._source['@newaccount']['newact'] === newact) {
-                    response.creator = actData.creator;
-                    response['trx_id'] = action._source['trx_id'];
-                    valid = true;
-                }
-            }
-        }
-
-        // if (action._source.parent !== 0 && valid) {
-        //
-        //     // Find indirect creator by global seq
-        //     const creationAction = await getActionByGS(elastic, action._source.parent);
-        //
-        //     if (creationAction.act.name === 'transfer') {
-        //         response['indirect_creator'] = creationAction['@transfer']['from'];
-        //         response['trx_id'] = creationAction['trx_id'];
-        //     } else {
-        //         response['indirect_creator'] = creationAction.act.authorization[0].actor;
-        //         response['trx_id'] = creationAction['trx_id'];
-        //     }
-        // }
-
-    }
-
-    redis.set(hash, JSON.stringify(response), 'EX', 600);
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get(route, {
-        schema: getCreatorSchema.GET
-    }, async (request) => {
-        return await getCreator(fastify, request);
-    });
-    next()
-};

api/routes/history/getDeltas.js

@@ -1,99 +0,0 @@
-const {getDeltasSchema} = require("../../schemas");
-const {getCacheByHash} = require("../../helpers/functions");
-
-const route = '/get_deltas';
-const maxDeltas = 200;
-
-async function getDeltas(fastify, request) {
-    const t0 = Date.now();
-    console.log(request.query);
-    const {redis, elastic} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, route + JSON.stringify(request.query) + 'v2');
-
-    // if (cachedResponse) {
-    //     return cachedResponse;
-    // }
-
-    let skip, limit;
-    let sort_direction = 'desc';
-    const mustArray = [];
-    for (const param in request.query) {
-        if (Object.prototype.hasOwnProperty.call(request.query, param)) {
-            console.log(param, request.query[param]);
-            const value = request.query[param];
-            switch (param) {
-                case 'limit': {
-                    limit = parseInt(value, 10);
-                    if (limit < 1) {
-                        return 'invalid limit parameter';
-                    }
-                    break;
-                }
-                case 'skip': {
-                    skip = parseInt(value, 10);
-                    if (skip < 0) {
-                        return 'invalid skip parameter';
-                    }
-                    break;
-                }
-                case 'sort': {
-                    if (value === 'asc' || value === '1') {
-                        sort_direction = 'asc';
-                    } else if (value === 'desc' || value === '-1') {
-                        sort_direction = 'desc'
-                    } else {
-                        return 'invalid sort direction';
-                    }
-                    break;
-                }
-                default: {
-                    const values = request.query[param].split(",");
-                    const terms = {};
-                    terms[param] = values;
-                    const shouldArray = {terms: terms};
-                    const boolStruct = {bool: {should: [shouldArray]}};
-                    mustArray.push(boolStruct);
-                    break;
-                }
-            }
-        }
-    }
-
-
-    let prefix = process.env.CHAIN;
-    if (process.env.CHAIN === 'mainnet') {
-        prefix = 'eos';
-    }
-
-    const results = await elastic.search({
-        "index": prefix + '-delta-*',
-        "from": skip || 0,
-        "size": (limit > maxDeltas ? maxDeltas : limit) || 10,
-        "body": {
-            query: {bool: {must: mustArray}},
-            sort: {
-                "block_num": sort_direction
-            }
-        }
-    });
-    const response = {
-        query_time: null,
-        total: results['body']['hits']['total'],
-        deltas: results['body']['hits']['hits'].map((d) => {
-            delete d._source.present;
-            return d._source;
-        })
-    };
-    response['query_time'] = Date.now() - t0;
-    // redis.set(hash, JSON.stringify(response));
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get(route, {
-        schema: getDeltasSchema.GET
-    }, async (request) => {
-        return await getDeltas(fastify, request);
-    });
-    next()
-};

api/routes/history/getTransaction.js

@@ -1,55 +0,0 @@
-const {getTransactionSchema} = require("../../schemas");
-const _ = require('lodash');
-const {getCacheByHash} = require("../../helpers/functions");
-
-async function getTransaction(fastify, request) {
-    const {redis, elastic, eosjs} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, JSON.stringify(request.query));
-    if (cachedResponse) {
-        return cachedResponse;
-    }
-    const pResults = await Promise.all([eosjs.rpc.get_info(), elastic['search']({
-        "index": process.env.CHAIN + '-action-*',
-        "body": {
-            "query": {
-                "bool": {
-                    must: [
-                        {term: {"trx_id": request.query.id.toLowerCase()}}
-                    ]
-                }
-            },
-            "sort": {
-                "global_sequence": "asc"
-            }
-        }
-    })]);
-    const results = pResults[1];
-    const response = {
-        "trx_id": request.query.id,
-        "lib": pResults[0].last_irreversible_block_num,
-        "actions": []
-    };
-    const hits = results['body']['hits']['hits'];
-    if (hits.length > 0) {
-        for (let action of hits) {
-            action = action._source;
-            const name = action.act.name;
-            if (action['@' + name]) {
-                action['act']['data'] = _.merge(action['@' + name], action['act']['data']);
-                delete action['@' + name];
-            }
-            response.actions.push(action);
-        }
-    }
-    redis.set(hash, JSON.stringify(response), 'EX', 30);
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get('/get_transaction', {
-        schema: getTransactionSchema.GET
-    }, async (request, reply) => {
-        reply.send(await getTransaction(fastify, request));
-    });
-    next()
-};

api/routes/state/getAccount.js

@@ -1,65 +0,0 @@
-const {getAccountSchema} = require("../../schemas");
-const {getCacheByHash} = require("../../helpers/functions");
-const route = '/get_account';
-const got = require('got');
-
-const localApi = `http://${process.env.SERVER_ADDR}:${process.env.SERVER_PORT}/v2`;
-const getTokensApi = localApi + '/state/get_tokens';
-const getActionsApi = localApi + '/history/get_actions';
-
-const enable_caching = process.env.ENABLE_CACHING === 'true';
-let cache_life = 30;
-if (process.env.CACHE_LIFE) {
-    cache_life = parseInt(process.env.CACHE_LIFE);
-}
-
-async function getAccount(fastify, request) {
-    const t0 = Date.now();
-    const {redis, elastic, eosjs} = fastify;
-
-    // let cachedResponse, hash;
-    // if(enable_caching) {
-    //     [cachedResponse, hash] = await getCacheByHash(redis, route + JSON.stringify(request.query));
-    //     if (cachedResponse) {
-    //         cachedResponse = JSON.parse(cachedResponse);
-    //         cachedResponse['query_time'] = Date.now() - t0;
-    //         cachedResponse['cached'] = true;
-    //         return cachedResponse;
-    //     }
-    // }
-
-    const response = {
-        query_time: null,
-        cached: false,
-        account: null,
-        actions: null,
-        tokens: null,
-        links: null
-    };
-
-    const account = request.query.account;
-    const reqQueue = [];
-    reqQueue.push(eosjs.rpc.get_account(account));
-
-    // fetch recent actions
-    reqQueue.push(got.get(`${getActionsApi}?account=${account}&limit=10`));
-
-    // fetch account tokens
-    reqQueue.push(got.get(`${getTokensApi}?account=${account}`));
-
-    const results = await Promise.all(reqQueue);
-    response.account = results[0];
-    response.actions = JSON.parse(results[1].body).actions;
-    response.tokens = JSON.parse(results[2].body).tokens;
-    response['query_time'] = Date.now() - t0;
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get(route, {
-        schema: getAccountSchema.GET
-    }, async (request) => {
-        return await getAccount(fastify, request);
-    });
-    next()
-};

api/routes/state/getKeyAccounts.js

@@ -1,97 +0,0 @@
-const {getKeyAccountsSchema} = require("../../schemas");
-const {getCacheByHash} = require("../../helpers/functions");
-const numeric = require('eosjs/dist/eosjs-numeric');
-
-function invalidKey() {
-    const err = new Error();
-    err.statusCode = 400;
-    err.message = 'invalid public key';
-    throw err;
-}
-
-async function getKeyAccounts(fastify, public_Key) {
-
-    const {redis, elastic} = fastify;
-    let publicKey;
-
-    if (public_Key.startsWith("PUB_")) {
-        publicKey = public_Key;
-    } else if (public_Key.startsWith("EOS")) {
-        try {
-            publicKey = numeric.convertLegacyPublicKey(public_Key);
-        } catch (e) {
-            console.log(e);
-            invalidKey();
-        }
-    } else {
-        invalidKey();
-    }
-
-    const [cachedResponse, hash] = await getCacheByHash(redis, JSON.stringify(publicKey));
-    if (cachedResponse) {
-        return cachedResponse;
-    }
-    const _body = {
-        query: {
-            bool: {
-                should: [
-                    {term: {"@updateauth.auth.keys.key.keyword": publicKey}},
-                    {term: {"@newaccount.active.keys.key.keyword": publicKey}},
-                    {term: {"@newaccount.owner.keys.key.keyword": publicKey}}
-                ],
-                minimum_should_match: 1
-            }
-        },
-        sort: [{"global_sequence": {"order": "desc"}}]
-    };
-
-    const results = await elastic.search({
-        index: process.env.CHAIN + '-action-*',
-        body: _body
-    });
-
-    const response = {
-        account_names: []
-    };
-    if (results['body']['hits']['hits'].length > 0) {
-        response.account_names = results['body']['hits']['hits'].map((v) => {
-            if (v._source.act.name === 'newaccount') {
-                if (v._source['@newaccount'].newact) {
-                    return v._source['@newaccount'].newact;
-                } else if (v._source.act.data.newact) {
-                    return v._source.act.data.newact;
-                } else {
-                    return null;
-                }
-            } else if (v._source.act.name === 'updateauth') {
-                return v._source.act.data.account;
-            } else {
-                return null;
-            }
-        });
-    }
-    if (response.account_names.length > 0) {
-        response.account_names = [...(new Set(response.account_names))];
-        redis.set(hash, JSON.stringify(response), 'EX', 30);
-        return response;
-    } else {
-        const err = new Error();
-        err.statusCode = 404;
-        err.message = 'no accounts associated with ' + public_Key;
-        throw err;
-    }
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get('/get_key_accounts', {
-        schema: getKeyAccountsSchema.GET
-    }, async (request) => {
-        return getKeyAccounts(fastify, request.query.public_key);
-    });
-    fastify.post('/get_key_accounts', {
-        schema: getKeyAccountsSchema.POST
-    }, async (request) => {
-        return getKeyAccounts(fastify, request.body.public_key);
-    });
-    next();
-};

api/routes/state/getProposals.js

@@ -1,146 +0,0 @@
-const {getProposalsSchema} = require("../../schemas");
-const {getCacheByHash} = require("../../helpers/functions");
-const route = '/get_proposals';
-
-const maxActions = 1000;
-
-const enable_caching = process.env.ENABLE_CACHING === 'true';
-let cache_life = 30;
-if (process.env.CACHE_LIFE) {
-    cache_life = parseInt(process.env.CACHE_LIFE);
-}
-
-function getTrackOpt(request) {
-    let trackTotalHits = 10000;
-    if (request.query.track) {
-        if (request.query.track === 'true') {
-            trackTotalHits = true;
-        } else if (request.query.track === 'false') {
-            trackTotalHits = false;
-        } else {
-            trackTotalHits = parseInt(request.query.track, 10);
-            if (trackTotalHits !== trackTotalHits) {
-                throw new Error('failed to parse track param');
-            }
-        }
-    }
-    return trackTotalHits;
-}
-
-async function getProposals(fastify, request) {
-
-    const t0 = Date.now();
-    const {redis, elastic, eosjs} = fastify;
-    let cachedResponse, hash;
-    if (enable_caching) {
-        [cachedResponse, hash] = await getCacheByHash(redis, route + JSON.stringify(request.query));
-        if (cachedResponse) {
-            cachedResponse = JSON.parse(cachedResponse);
-            cachedResponse['query_time'] = Date.now() - t0;
-            cachedResponse['cached'] = true;
-            return cachedResponse;
-        }
-    }
-
-    // Pagination
-    let skip, limit;
-    skip = parseInt(request.query.skip, 10);
-    if (skip < 0) {
-        return 'invalid skip parameter';
-    }
-    limit = parseInt(request.query.limit, 10);
-    if (limit < 1) {
-        return 'invalid limit parameter';
-    }
-
-    let queryStruct = {
-        "bool": {
-            "must": []
-        }
-    };
-
-    // Filter by account
-    if (request.query.account) {
-        queryStruct.bool.must.push({
-            "bool": {
-                "should": [
-                    {"term": {"requested_approvals.actor": request.query.account}},
-                    {"term": {"provided_approvals.actor": request.query.account}}
-                ]
-            }
-        });
-    }
-
-    // Filter by proposer account
-    if (request.query.proposer) {
-        queryStruct.bool.must.push({"term": {"proposer": request.query.proposer}});
-    }
-
-    // Filter by proposal name
-    if (request.query.proposal) {
-        queryStruct.bool.must.push({"term": {"proposal_name": request.query.proposal}});
-    }
-
-    // Filter by execution status
-    if (typeof request.query.executed !== 'undefined') {
-        queryStruct.bool.must.push({"term": {"executed": request.query.executed}});
-    }
-
-    // Filter by requested actors
-    if (request.query.requested) {
-        queryStruct.bool.must.push({"term": {"requested_approvals.actor": request.query.requested}});
-    }
-
-    // Filter by provided actors
-    if (request.query.provided) {
-        queryStruct.bool.must.push({"term": {"provided_approvals.actor": request.query.provided}});
-    }
-
-    // If no filter switch to full match
-    if (queryStruct.bool.must.length === 0) {
-        queryStruct = {
-            "match_all": {}
-        };
-    }
-
-    console.log(JSON.stringify(queryStruct));
-
-    const results = await elastic.search({
-        "index": process.env.CHAIN + '-table-proposals-*',
-        "from": skip || 0,
-        "size": (limit > maxActions ? maxActions : limit) || 10,
-        "body": {
-            "track_total_hits": getTrackOpt(request),
-            "query": queryStruct,
-            "sort": [{"block_num": "desc"}]
-        }
-    });
-
-    const response = {
-        query_time: null,
-        cached: false,
-        total: results['body']['hits']['total'],
-        proposals: []
-    };
-
-    const hits = results['body']['hits']['hits'];
-    for (const hit of hits) {
-        const prop = hit._source;
-        response.proposals.push(prop);
-    }
-
-    response['query_time'] = Date.now() - t0;
-    if (enable_caching) {
-        redis.set(hash, JSON.stringify(response), 'EX', cache_life);
-    }
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get(route, {
-        schema: getProposalsSchema.GET
-    }, async (request) => {
-        return await getProposals(fastify, request);
-    });
-    next();
-};

api/routes/state/getTokens.js

@@ -1,94 +0,0 @@
-const {getTokensSchema} = require("../../schemas");
-const {getCacheByHash} = require("../../helpers/functions");
-const route = '/get_tokens';
-
-const enable_caching = process.env.ENABLE_CACHING === 'true';
-let cache_life = 30;
-if(process.env.CACHE_LIFE) {
-    cache_life = parseInt(process.env.CACHE_LIFE);
-}
-
-async function getTokens(fastify, request) {
-    const t0 = Date.now();
-    const {redis, elastic, eosjs} = fastify;
-
-    let cachedResponse, hash;
-    if(enable_caching) {
-        [cachedResponse, hash] = await getCacheByHash(redis, route + JSON.stringify(request.query));
-        if (cachedResponse) {
-            cachedResponse = JSON.parse(cachedResponse);
-            cachedResponse['query_time'] = Date.now() - t0;
-            cachedResponse['cached'] = true;
-            return cachedResponse;
-        }
-    }
-
-    const response = {
-        query_time: null,
-        cached: false,
-        'account': request.query.account,
-        'tokens': []
-    };
-
-    const results = await elastic.search({
-        "index": process.env.CHAIN + '-action-*',
-        "body": {
-            size: 0,
-            query: {
-                bool: {
-                    // must_not: {term: {"act.account": "eosio.token"}},
-                    filter: [
-                        {term: {"notified": request.query.account}},
-                        {terms: {"act.name": ["transfer", "issue"]}}
-                    ]
-                }
-            },
-            aggs: {
-                tokens: {
-                    terms: {
-                        field: "act.account",
-                        size: 1000
-                    }
-                }
-            }
-        }
-    });
-
-    for (const bucket of results['body']['aggregations']['tokens']['buckets']) {
-        let token_data;
-        try {
-            token_data = await eosjs.rpc.get_currency_balance(bucket['key'], request.query.account);
-        } catch (e) {
-            console.log(`get_currency_balance error - contract:${bucket['key']} - account:${request.query.account}`);
-            continue;
-        }
-        for (const entry of token_data) {
-            let precision = 0;
-            const [amount, symbol] = entry.split(" ");
-            const amount_arr = amount.split(".");
-            if (amount_arr.length === 2) {
-                precision = amount_arr[1].length;
-            }
-            response.tokens.push({
-                symbol: symbol,
-                precision: precision,
-                amount: parseFloat(amount),
-                contract: bucket['key']
-            });
-        }
-    }
-    response['query_time'] = Date.now() - t0;
-    if(enable_caching) {
-        redis.set(hash, JSON.stringify(response), 'EX', cache_life);
-    }
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get(route, {
-        schema: getTokensSchema.GET
-    }, async (request) => {
-        return await getTokens(fastify, request);
-    });
-    next()
-};

api/routes/state/getVoters.js

@@ -1,90 +0,0 @@
-const {getVotersSchema} = require("../../schemas");
-const {getCacheByHash} = require("../../helpers/functions");
-const route = '/get_voters';
-
-const maxActions = 1000;
-
-async function getVoters(fastify, request) {
-
-    const t0 = Date.now();
-    const {redis, elastic} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, route + JSON.stringify(request.query));
-    if (cachedResponse) {
-        return cachedResponse;
-    }
-
-    let skip, limit;
-
-    skip = parseInt(request.query.skip, 10);
-    if (skip < 0) {
-        return 'invalid skip parameter';
-    }
-    limit = parseInt(request.query.limit, 10);
-    if (limit < 1) {
-        return 'invalid limit parameter';
-    }
-
-    const response = {
-        query_time: null,
-        voter_count: 0,
-        'voters': []
-    };
-    let queryStruct = {
-        "bool": {
-            "must": []
-        }
-    };
-
-    if (request.query.producer) {
-        for (const bp of request.query.producer.split(",")) {
-            queryStruct.bool.must.push({"term": {"producers": bp}});
-        }
-    }
-
-    if (request.query.proxy === 'true') {
-        queryStruct.bool.must.push({"term": {"is_proxy": true}});
-    }
-
-    if (queryStruct.bool.must.length === 0) {
-        queryStruct = {
-            "match_all": {}
-        };
-    }
-
-    let prefix = process.env.CHAIN;
-    if (process.env.CHAIN === 'mainnet') {
-        prefix = 'eos';
-    }
-
-    const results = await elastic.search({
-        "index": prefix + '-table-voters-*',
-        "from": skip || 0,
-        "size": (limit > maxActions ? maxActions : limit) || 10,
-        "body": {
-            "query": queryStruct,
-            "sort": [{"last_vote_weight": "desc"}]
-        }
-    });
-    const hits = results['body']['hits']['hits'];
-    for (const hit of hits) {
-        const voter = hit._source;
-        response.voters.push({
-            account: voter.voter,
-            weight: voter.last_vote_weight,
-            last_vote: voter.block_num
-        });
-    }
-    response.voter_count = results['body']['hits']['total']['value'];
-    response['query_time'] = Date.now() - t0;
-    redis.set(hash, JSON.stringify(response), 'EX', 30);
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get(route, {
-        schema: getVotersSchema.GET
-    }, async (request) => {
-        return await getVoters(fastify, request);
-    });
-    next();
-};

api/routes/v1-chain/abi_bin_to_json/index.ts

@@ -0,0 +1,21 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Returns an object containing rows from the specified table.',
+        {
+            "code": 'AccountName#',
+            "action": 'AccountName#',
+            "binargs": {
+                "type": "string",
+                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
+                "title": "Hex"
+            }
+        },
+        ["code", "action", "binargs"]
+    );
+    next();
+}

api/routes/v1-chain/abi_json_to_bin/index.ts

@@ -0,0 +1,19 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Convert JSON object to binary',
+        {
+            "binargs": {
+                "type": "string",
+                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
+                "title": "Hex"
+            }
+        },
+        ["binargs"]
+    );
+    next();
+}

api/routes/v1-chain/get_abi/index.ts

@@ -0,0 +1,13 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves the ABI for a contract based on its account name',
+        {"account_name": 'AccountName#'},
+        ["account_name"]
+    );
+    next();
+}

api/routes/v1-chain/get_account/index.ts

@@ -0,0 +1,13 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Returns an object containing various details about a specific account on the blockchain.',
+        {"account_name": 'AccountName#'},
+        ["account_name"]
+    );
+    next();
+}

api/routes/v1-chain/get_activated_protocol_features/index.ts

@@ -0,0 +1,33 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retreives the activated protocol features for producer node',
+        {
+            "lower_bound": {
+                "type": "integer",
+                "description": "Lower bound"
+            },
+            "upper_bound": {
+                "type": "integer",
+                "description": "Upper bound"
+            },
+            "limit": {
+                "type": "integer",
+                "description": "The limit, default is 10"
+            },
+            "search_by_block_num": {
+                "type": "boolean",
+                "description": "Flag to indicate it is has to search by block number"
+            },
+            "reverse": {
+                "type": "boolean",
+                "description": "Flag to indicate it has to search in reverse"
+            }
+        }
+    );
+    next();
+}

api/routes/v1-chain/get_block.js

@@ -1,134 +0,0 @@
-const schema = {
-    description: 'Returns an object containing various details about a specific block on the blockchain.',
-    summary: 'Returns an object containing various details about a specific block on the blockchain.',
-    tags: ['state'],
-    body: {
-        type: ['object', 'string'],
-        properties: {
-            "block_num_or_id": {
-                description: 'Provide a block number or a block id',
-                type: 'string'
-            },
-        },
-        required: ["block_num_or_id"]
-    }
-};
-
-const route = '/get_block';
-const {blockStore} = require('../blockStore');
-const {getCacheByHash} = require("../../helpers/functions");
-
-let last_get_info = 0;
-let chain_info;
-
-async function getInfo(eosjs) {
-    if (Date.now() - last_get_info > 1000) {
-        chain_info = await eosjs.rpc.get_info();
-        last_get_info = Date.now();
-    }
-}
-
-async function processBlockData(block, eosjs) {
-    // Calculate total usage
-    let total_cpu = 0;
-    let total_net = 0;
-    block.transactions.forEach((trx) => {
-        total_cpu += trx['cpu_usage_us'];
-        total_net += trx['net_usage_words'];
-    });
-    block['total_cpu_usage_us'] = total_cpu;
-    block['total_net_usage_words'] = total_net;
-
-    // check irreversibility
-    await getInfo(eosjs);
-    block['last_irreversible_block_num'] = chain_info.last_irreversible_block_num;
-    block['irreversible'] = block['block_num'] <= block['last_irreversible_block_num'];
-    return block;
-}
-
-async function getBlock(fastify, request, reply) {
-    const {eosjs, redis} = fastify;
-    let block_num_or_id;
-    try {
-        if (typeof request.body === 'object') {
-            block_num_or_id = request.body['block_num_or_id'];
-        } else {
-            block_num_or_id = JSON.parse(request.body)['block_num_or_id'];
-        }
-    } catch (e) {
-        console.log(request.body);
-        console.log(e);
-    }
-
-    if (parseInt(block_num_or_id) === 0) {
-        await getInfo(eosjs);
-        block_num_or_id = chain_info.head_block_num;
-    }
-
-    if (block_num_or_id) {
-        let cachedResponse, hash;
-        const key = route + '_' + block_num_or_id;
-        [cachedResponse, hash] = await getCacheByHash(redis, key);
-        if (cachedResponse) {
-            cachedResponse = JSON.parse(cachedResponse);
-            if (cachedResponse['irreversible']) {
-                reply.send(cachedResponse);
-                return;
-            }
-        }
-
-        try {
-            let blockdata = await eosjs.rpc.get_block(block_num_or_id);
-            blockdata = await processBlockData(blockdata, eosjs);
-            const blockstring = JSON.stringify(blockdata);
-            if (blockdata['irreversible']) {
-                redis.set(hash, blockstring, 'EX', 86400);
-            } else {
-                redis.set(hash, blockstring, 'EX', 10);
-            }
-            reply.send(blockdata);
-        } catch (e) {
-            reply.code(400).send({
-                "code": 400,
-                "message": "Unknown Block",
-                "error": {
-                    "code": 3100002,
-                    "name": "unknown_block_exception",
-                    "what": "Unknown block",
-                    "details": [
-                        {
-                            "message": "Could not find block: " + block_num_or_id,
-                            "file": "chain_plugin.cpp",
-                            "line_number": 1852,
-                            "method": "get_block"
-                        }
-                    ]
-                }
-            });
-        }
-
-    } else {
-        reply.code(400).send({});
-    }
-
-    // TODO: fallback to state history
-    // let from_block;
-    // if (request.body['block_num_or_id']) {
-    //     from_block = parseInt(request.body['block_num_or_id']);
-    //     let response;
-    //     console.log('Local Block store size:', blockStore.localBlocks.size);
-    //     blockStore.requestRange(from_block, from_block + 1, (block) => {
-    //         response = block;
-    //     }, () => {
-    //         reply.send(response);
-    //     });
-    // }
-
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.post(route, {schema}, async (request, reply) => {
-        await getBlock(fastify, request, reply);
-    });
-    next();
-};

api/routes/v1-chain/get_block/index.ts

@@ -0,0 +1,18 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Returns an object containing various details about a specific block on the blockchain.',
+        {
+            "block_num_or_id": {
+                description: "Provide a `block number` or a `block id`",
+                type: 'string'
+            }
+        },
+        ["block_num_or_id"]
+    );
+    next();
+}

api/routes/v1-chain/get_block_header_state/index.ts

@@ -0,0 +1,18 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves the block header state',
+        {
+            "block_num_or_id": {
+                "type": "string",
+                "description": "Provide a block_number or a block_id"
+            }
+        },
+        ["block_num_or_id"]
+    );
+    next();
+}

api/routes/v1-chain/get_code/index.ts

@@ -0,0 +1,20 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves contract code',
+        {
+            "account_name": 'AccountName#',
+            "code_as_wasm": {
+                "type": "integer",
+                "default": 1,
+                "description": "This must be 1 (true)"
+            }
+        },
+        ["account_name", "code_as_wasm"]
+    );
+    next();
+}

api/routes/v1-chain/get_currency_balance/index.ts

@@ -0,0 +1,17 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves the current balance',
+        {
+            "code": 'AccountName#',
+            "account": 'AccountName#',
+            "symbol": 'Symbol#',
+        },
+        ["code", "account", "symbol"]
+    );
+    next();
+}

api/routes/v1-chain/get_currency_stats/index.ts

@@ -0,0 +1,24 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves currency stats',
+        {
+            "code": {
+                description: 'contract name',
+                type: 'string',
+                minLength: 1,
+                maxLength: 12
+            },
+            "symbol": {
+                description: 'token symbol',
+                type: 'string',
+            }
+        },
+        ["code", "symbol"]
+    );
+    next();
+}

api/routes/v1-chain/get_info/index.ts

@@ -0,0 +1,11 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Returns an object containing various details about the blockchain.'
+    );
+    next();
+}

api/routes/v1-chain/get_producers/index.ts

@@ -0,0 +1,25 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves producers list',
+        {
+            "limit": {
+                "type": "string",
+                "description": "total number of producers to retrieve"
+            },
+            "lower_bound": {
+                "type": "string",
+                "description": "In conjunction with limit can be used to paginate through the results. For example, limit=10 and lower_bound=10 would be page 2"
+            },
+            "json": {
+                "type": "boolean",
+                "description": "return result in JSON format"
+            }
+        }
+    );
+    next();
+}

api/routes/v1-chain/get_raw_abi/index.ts

@@ -0,0 +1,15 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves raw ABI for a contract based on account name',
+        {
+            "account_name": 'AccountName#'
+        },
+        ["account_name"]
+    );
+    next();
+}

api/routes/v1-chain/get_raw_code_and_abi/index.ts

@@ -0,0 +1,15 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves raw code and ABI for a contract based on account name',
+        {
+            "account_name": 'AccountName#'
+        },
+        ["account_name"]
+    );
+    next();
+}

api/routes/v1-chain/get_scheduled_transaction/index.ts

@@ -0,0 +1,27 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves the scheduled transaction',
+        {
+            "lower_bound": {
+                "type": "string",
+                "description": "Date/time string in the format YYYY-MM-DDTHH:MM:SS.sss",
+                "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}$",
+                "title": "DateTimeSeconds"
+            },
+            "limit": {
+                "description": "The maximum number of transactions to return",
+                "type": "integer"
+            },
+            "json": {
+                "description": "true/false whether the packed transaction is converted to json",
+                "type": "boolean"
+            }
+        }
+    );
+    next();
+}

api/routes/v1-chain/get_table_by_scope/index.ts

@@ -0,0 +1,38 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Retrieves table scope',
+        {
+            "code": {
+                "type": "string",
+                "description": "`name` of the contract to return table data for"
+            },
+            "table": {
+                "type": "string",
+                "description": "Filter results by table"
+            },
+            "lower_bound": {
+                "type": "string",
+                "description": "Filters results to return the first element that is not less than provided value in set"
+            },
+            "upper_bound": {
+                "type": "string",
+                "description": "Filters results to return the first element that is greater than provided value in set"
+            },
+            "limit": {
+                "type": "integer",
+                "description": "Limit number of results returned."
+            },
+            "reverse": {
+                "type": "boolean",
+                "description": "Reverse the order of returned results"
+            }
+        },
+        ["code"]
+    );
+    next();
+}

api/routes/v1-chain/get_table_rows/index.ts

@@ -0,0 +1,42 @@
+import {FastifyInstance} from "fastify";
+import {addChainApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addChainApiRoute(
+        fastify,
+        getRouteName(__filename),
+        'Returns an object containing rows from the specified table.',
+        {
+            "code": {
+                "type": "string",
+                "description": "The name of the smart contract that controls the provided table"
+            },
+            "table": {
+                "type": "string",
+                "description": "The name of the table to query"
+            },
+            "scope": {
+                "type": "string",
+                "description": "The account to which this data belongs"
+            },
+            "index_position": {
+                "type": "string",
+                "description": "Position of the index used, accepted parameters `primary`, `secondary`, `tertiary`, `fourth`, `fifth`, `sixth`, `seventh`, `eighth`, `ninth` , `tenth`"
+            },
+            "key_type": {
+                "type": "string",
+                "description": "Type of key specified by index_position (for example - `uint64_t` or `name`)"
+            },
+            "encode_type": {
+                "type": "string"
+            },
+            "upper_bound": {
+                "type": "string"
+            },
+            "lower_bound": {
+                "type": "string"
+            }
+        }
+    );
+    next();
+}

api/routes/v1-chain/push_transaction/index.ts

@@ -0,0 +1,39 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, chainApiHandler, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addApiRoute(
+        fastify,
+        'POST',
+        getRouteName(__filename),
+        chainApiHandler,
+        {
+            description: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+            summary: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+            tags: ['chain'],
+            body: {
+                type: ['object', 'string'],
+                properties: {
+                    signatures: {
+                        "type": "array",
+                        "description": "array of signatures required to authorize transaction",
+                        "items": 'Signature#'
+                    },
+                    compression: {
+                        "type": "boolean",
+                        "description": "Compression used, usually false"
+                    },
+                    packed_context_free_data: {
+                        "type": "string",
+                        "description": "json to hex"
+                    },
+                    packed_trx: {
+                        "type": "string",
+                        "description": "Transaction object json to hex"
+                    }
+                }
+            }
+        }
+    );
+    next();
+}

api/routes/v1-chain/push_transactions/index.ts

@@ -0,0 +1,46 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, chainApiHandler, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addApiRoute(
+        fastify,
+        'POST',
+        getRouteName(__filename),
+        chainApiHandler,
+        {
+            description: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+            summary: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+            tags: ['chain'],
+            body: {
+                type: ['array', 'object', 'string'],
+                items: {
+                    type: ['object', 'string'],
+                    additionalProperties: false,
+                    minProperties: 8,
+                    required: [
+                        "expiration",
+                        "ref_block_num",
+                        "ref_block_prefix",
+                        "max_net_usage_words",
+                        "max_cpu_usage_ms",
+                        "delay_sec",
+                        "context_free_actions",
+                        "actions"
+                    ],
+                    properties: {
+                        "expiration": 'Expiration#',
+                        "ref_block_num": {"type": "integer"},
+                        "ref_block_prefix": {"type": "integer"},
+                        "max_net_usage_words": 'WholeNumber#',
+                        "max_cpu_usage_ms": 'WholeNumber#',
+                        "delay_sec": {"type": "integer"},
+                        "context_free_actions": {"type": "array", "items": 'ActionItems#'},
+                        "actions": {"type": "array", "items": 'ActionItems#'},
+                        "transaction_extensions": {"type": "array", "items": 'BlockExtensions#'}
+                    },
+                }
+            }
+        }
+    );
+    next();
+}

api/routes/v1-chain/send_transaction/index.ts

@@ -0,0 +1,39 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, chainApiHandler, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    addApiRoute(
+        fastify,
+        'POST',
+        getRouteName(__filename),
+        chainApiHandler,
+        {
+            description: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+            summary: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+            tags: ['chain'],
+            body: {
+                type: ['object', 'string'],
+                properties: {
+                    signatures: {
+                        "type": "array",
+                        "description": "array of signatures required to authorize transaction",
+                        "items": 'Signature#'
+                    },
+                    compression: {
+                        "type": "boolean",
+                        "description": "Compression used, usually false"
+                    },
+                    packed_context_free_data: {
+                        "type": "string",
+                        "description": "json to hex"
+                    },
+                    packed_trx: {
+                        "type": "string",
+                        "description": "Transaction object json to hex"
+                    }
+                }
+            }
+        }
+    );
+    next();
+}

api/routes/v1-history/get_actions/get_actions.ts

@@ -1,14 +1,17 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
+import * as flatstr from 'flatstr';
 
 const terms = ["notified", "act.authorization.actor"];
 const extendedActions = new Set(["transfer", "newaccount", "updateauth"]);
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
+
     if (typeof request.body === 'string') {
         request.body = JSON.parse(request.body);
     }
+
     const reqBody = request.body;
     const should_array = [];
     for (const entry of terms) {
@@ -16,7 +19,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
         tObj.term[entry] = reqBody.account_name;
         should_array.push(tObj);
     }
-    let code, method, pos, offset, parent;
+    let code, method, pos, offset;
     let sort_direction = 'desc';
     let filterObj = [];
     if (reqBody.filter) {
@@ -172,7 +175,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             };
             mergeActionMeta(action);
             act.action_trace.act = action.act;
-            act.action_trace.act.hex_data = Buffer.from(JSON.stringify(action.act.data)).toString('hex');
+            act.action_trace.act.hex_data = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
             if (action.act.account_ram_deltas) {
                 act.action_trace.account_ram_deltas = action.account_ram_deltas
             }

api/routes/v1-history/get_actions/index.ts

@@ -8,7 +8,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
     addApiRoute(fastify, 'POST', getRouteName(__filename), getActionsHandler, {
         description: 'legacy get actions query',
         summary: 'get actions',
-        tags: ['actions', 'history'],
+        tags: ['history'],
         body: {
             type: ['object', 'string'],
             properties: {

api/routes/v1-history/get_controlled_accounts/index.ts

@@ -6,7 +6,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
     addApiRoute(fastify, 'POST', getRouteName(__filename), getControlledAccountsHandler, {
         description: 'get controlled accounts by controlling accounts',
         summary: 'get controlled accounts by controlling accounts',
-        tags: ['state'],
+        tags: ['accounts'],
         body: {
             type: ['object', 'string'],
             properties: {

api/routes/v1-history/get_key_accounts/index.ts

@@ -8,9 +8,9 @@ export default function (fastify: FastifyInstance, opts: any, next) {
     addApiRoute(fastify, 'POST', getRouteName(__filename), getKeyAccountsHandler, {
         description: 'get accounts by public key',
         summary: 'get accounts by public key',
-        tags: ['accounts', 'state'],
+        tags: ['accounts'],
         body: {
-            type: 'object',
+            type: ['object', 'string'],
             properties: {"public_key": {description: 'public key', type: 'string'}},
             required: ["public_key"]
         },

api/routes/v1-history/get_transaction/get_transaction.ts

@@ -2,191 +2,194 @@ import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
 import {createHash} from "crypto";
+import * as flatstr from 'flatstr';
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
-    if (typeof request.body === 'string') {
-        request.body = JSON.parse(request.body)
-    }
-    const pResults = await Promise.all([fastify.eosjs.rpc.get_info(), fastify.elastic['search']({
-        "index": fastify.manager.chain + '-action-*',
-        "body": {
-            "query": {
-                "bool": {
-                    must: [
-                        {term: {"trx_id": request.body.id.toLowerCase()}}
-                    ]
-                }
-            },
-            "sort": {
-                "global_sequence": "asc"
-            }
-        }
-    })]);
-    const results = pResults[1];
-    const response: any = {
-        "id": request.body.id,
-        "trx": {
-            "receipt": {
-                "status": "executed",
-                "cpu_usage_us": 0,
-                "net_usage_words": 0,
-                "trx": [1, {
-                    "signatures": "",
-                    "compression": "none",
-                    "packed_context_free_data": "",
-                    "packed_trx": ""
-                }]
-            },
-            "trx": {}
-        },
-        "block_num": 0,
-        "block_time": "",
-        "last_irreversible_block": pResults[0].last_irreversible_block_num,
-        "traces": []
-    };
+	if (typeof request.body === 'string') {
+		request.body = JSON.parse(request.body)
+	}
+	const pResults = await Promise.all([fastify.eosjs.rpc.get_info(), fastify.elastic['search']({
+		"index": fastify.manager.chain + '-action-*',
+		"body": {
+			"query": {
+				"bool": {
+					must: [
+						{term: {"trx_id": request.body.id.toLowerCase()}}
+					]
+				}
+			},
+			"sort": {
+				"global_sequence": "asc"
+			}
+		}
+	})]);
+	const results = pResults[1];
+	const response: any = {
+		"id": request.body.id,
+		"trx": {
+			"receipt": {
+				"status": "executed",
+				"cpu_usage_us": 0,
+				"net_usage_words": 0,
+				"trx": [1, {
+					"signatures": "",
+					"compression": "none",
+					"packed_context_free_data": "",
+					"packed_trx": ""
+				}]
+			},
+			"trx": {}
+		},
+		"block_num": 0,
+		"block_time": "",
+		"last_irreversible_block": pResults[0].last_irreversible_block_num,
+		"traces": []
+	};
 
-    const hits = results['body']['hits']['hits'];
+	const hits = results['body']['hits']['hits'];
 
-    if (hits.length > 0) {
-        const actions = hits;
-        response.trx.trx = {
-            "expiration": "",
-            "ref_block_num": 0,
-            "ref_block_prefix": 0,
-            "max_net_usage_words": 0,
-            "max_cpu_usage_ms": 0,
-            "delay_sec": 0,
-            "context_free_actions": [],
-            "actions": [],
-            "transaction_extensions": [],
-            "signatures": [],
-            "context_free_data": []
-        };
-        let traces = {};
-        let seqNum = 0;
-        for (let action of actions) {
-            action = action._source;
-            mergeActionMeta(action);
-            action.act['hex_data'] = Buffer.from(JSON.stringify(action.act.data)).toString('hex');
-            if (action.parent === 0) {
-                response.trx.trx.actions.push(action.act);
-            }
-            response.block_num = action.block_num;
-            response.block_time = action['@timestamp'];
-            seqNum += 1;
-            let trace = {
-                receipt: {
-                    receiver: action.act.account,
-                    global_sequence: String(action.global_sequence),
-                    auth_sequence: [
-                        action.act.authorization[0].actor,
-                        seqNum
-                    ],
-                    act_digest: '',
-                    recv_sequence: seqNum,
-                    code_sequence: 1,
-                    abi_sequence: 1
-                },
-                act: action.act,
-                account_ram_deltas: action.account_ram_deltas || [],
-                context_free: false,
-                block_num: action.block_num,
-                block_time: action['@timestamp'],
-                console: "",
-                elapsed: 0,
-                except: null,
-                inline_traces: [],
-                producer_block_id: "",
-                trx_id: request.body.id,
-                notified: action.notified
-            };
-            let hash = createHash('sha256');
-            hash.update(JSON.stringify(action.act));
-            trace.receipt.act_digest = hash.digest('hex');
-            traces[action.global_sequence] = trace;
-        }
+	if (hits.length > 0) {
+		const actions = hits;
+		response.trx.trx = {
+			"expiration": "",
+			"ref_block_num": 0,
+			"ref_block_prefix": 0,
+			"max_net_usage_words": 0,
+			"max_cpu_usage_ms": 0,
+			"delay_sec": 0,
+			"context_free_actions": [],
+			"actions": [],
+			"transaction_extensions": [],
+			"signatures": [],
+			"context_free_data": []
+		};
+		let traces = {};
+		let seqNum = 0;
+		for (let action of actions) {
+			action = action._source;
+			mergeActionMeta(action);
+			action.act['hex_data'] = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+			if (action.parent === 0) {
+				response.trx.trx.actions.push(action.act);
+			}
+			response.block_num = action.block_num;
+			response.block_time = action['@timestamp'];
+			seqNum += 1;
+			let trace = {
+				receipt: {
+					receiver: action.act.account,
+					global_sequence: String(action.global_sequence),
+					auth_sequence: [
+						action.act.authorization[0].actor,
+						seqNum
+					],
+					act_digest: '',
+					recv_sequence: seqNum,
+					code_sequence: 1,
+					abi_sequence: 1
+				},
+				act: action.act,
+				account_ram_deltas: action.account_ram_deltas || [],
+				context_free: false,
+				block_num: action.block_num,
+				block_time: action['@timestamp'],
+				console: "",
+				elapsed: 0,
+				except: null,
+				inline_traces: [],
+				producer_block_id: "",
+				trx_id: request.body.id,
+				notified: action.notified
+			};
+			let hash = createHash('sha256');
+			hash.update(JSON.stringify(action.act));
+			trace.receipt.act_digest = hash.digest('hex');
+			traces[action.global_sequence] = trace;
+		}
 
-        actions.forEach(action => {
-            action = action._source;
-            for (let i = 0; i < traces[action.global_sequence].notified.length; i++) {
-                if (traces[action.global_sequence].notified[i] === action.act.account) {
-                    traces[action.global_sequence].notified.splice(i, 1);
-                    break;
-                }
-            }
-            if (action.parent !== 0 && action.parent) {
-                if (traces[action.parent]) {
-                    for (let i = 0; i < traces[action.parent].notified.length; i++) {
-                        if (traces[action.parent].notified[i] === action.act.account) {
-                            traces[action.parent].notified.splice(i, 1);
-                            break;
-                        }
-                    }
-                    traces[action.parent].inline_traces.push(traces[action.global_sequence]);
-                }
-            }
-        });
+		actions.forEach(action => {
+			action = action._source;
+			for (let i = 0; i < traces[action.global_sequence].notified.length; i++) {
+				if (traces[action.global_sequence].notified[i] === action.act.account) {
+					traces[action.global_sequence].notified.splice(i, 1);
+					break;
+				}
+			}
+			if (action.parent !== 0 && action.parent) {
+				if (traces[action.parent]) {
+					for (let i = 0; i < traces[action.parent].notified.length; i++) {
+						if (traces[action.parent].notified[i] === action.act.account) {
+							traces[action.parent].notified.splice(i, 1);
+							break;
+						}
+					}
+					traces[action.parent].inline_traces.push(traces[action.global_sequence]);
+				}
+			}
+		});
 
-        actions.forEach(action => {
-            action = action._source;
-            response.traces.push(traces[action.global_sequence]);
-            traces[action.global_sequence].notified.forEach((note, index) => {
-                seqNum += 1;
-                let trace = {
-                    receipt: {
-                        receiver: note,
-                        global_sequence: String(action.global_sequence + index + 1),
-                        auth_sequence: [
-                            action.act.authorization[0].actor,
-                            seqNum
-                        ],
-                        act_digest: traces[action.global_sequence].receipt.act_digest,
-                        recv_sequence: seqNum,
-                        code_sequence: 1,
-                        abi_sequence: 1
-                    },
-                    account_ram_deltas: action.account_ram_deltas || [],
-                    act: action.act,
-                    block_num: action.block_num,
-                    block_time: action['@timestamp'],
-                    console: "",
-                    context_free: false,
-                    elapsed: 0,
-                    except: null,
-                    inline_traces: [],
-                    producer_block_id: "",
-                    trx_id: request.body.id,
-                };
-                traces[action.global_sequence].inline_traces.unshift(trace);
-                response.traces.push(trace);
-            });
-            delete traces[action.global_sequence].notified;
-        });
-    } else {
-        const errmsg = "Transaction " + request.body.id.toLowerCase() + " not found in history and no block hint was given";
-        return {
-            code: 500,
-            message: "Internal Service Error",
-            error: {
-                code: 3040011,
-                name: "tx_not_found",
-                what: "The transaction can not be found",
-                details: [
-                    {
-                        message: errmsg,
-                        file: "",
-                        line_number: 1,
-                        method: "get_transaction"
-                    }
-                ]
-            }
-        }
-    }
-    return response;
+		actions.forEach(action => {
+			action = action._source;
+			response.traces.push(traces[action.global_sequence]);
+			if (traces[action.global_sequence] && traces[action.global_sequence].notified) {
+				traces[action.global_sequence].notified.forEach((note, index) => {
+					seqNum += 1;
+					let trace = {
+						receipt: {
+							receiver: note,
+							global_sequence: String(action.global_sequence + index + 1),
+							auth_sequence: [
+								action.act.authorization[0].actor,
+								seqNum
+							],
+							act_digest: traces[action.global_sequence].receipt.act_digest,
+							recv_sequence: seqNum,
+							code_sequence: 1,
+							abi_sequence: 1
+						},
+						account_ram_deltas: action.account_ram_deltas || [],
+						act: action.act,
+						block_num: action.block_num,
+						block_time: action['@timestamp'],
+						console: "",
+						context_free: false,
+						elapsed: 0,
+						except: null,
+						inline_traces: [],
+						producer_block_id: "",
+						trx_id: request.body.id,
+					};
+					traces[action.global_sequence].inline_traces.unshift(trace);
+					response.traces.push(trace);
+				});
+				delete traces[action.global_sequence].notified;
+			}
+		});
+	} else {
+		const errmsg = "Transaction " + request.body.id.toLowerCase() + " not found in history and no block hint was given";
+		return {
+			code: 500,
+			message: "Internal Service Error",
+			error: {
+				code: 3040011,
+				name: "tx_not_found",
+				what: "The transaction can not be found",
+				details: [
+					{
+						message: errmsg,
+						file: "",
+						line_number: 1,
+						method: "get_transaction"
+					}
+				]
+			}
+		}
+	}
+	return response;
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getTransaction, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+		reply.send(await timedQuery(getTransaction, fastify, request, route));
+	}
 }

api/routes/v1-history/get_transaction/index.ts

@@ -6,7 +6,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
     addApiRoute(fastify, 'POST', getRouteName(__filename), getTransactionHandler, {
         description: 'get all actions belonging to the same transaction',
         summary: 'get transaction by id',
-        tags: ['transactions', 'history'],
+        tags: ['history'],
         body: {
             type: ['object', 'string'],
             properties: {id: {description: 'transaction id', type: 'string'}},

api/routes/v1-trace/get_block/get_block.ts

@@ -0,0 +1,127 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {ServerResponse} from "http";
+import {timedQuery} from "../../../helpers/functions";
+import * as _ from "lodash";
+
+interface getBlockTraceResponse {
+    id: string;
+    number: number;
+    previous_id: string;
+    status: string;
+    timestamp: string;
+    producer: string;
+    transactions: any[];
+}
+
+async function getBlockTrace(fastify: FastifyInstance, request: FastifyRequest) {
+
+    if (typeof request.body === 'string') {
+        request.body = JSON.parse(request.body);
+    }
+
+    const reqBody = request.body;
+    const targetBlock = parseInt(reqBody.block_num);
+    let searchBody;
+
+    if (reqBody.block_id) {
+        searchBody = {
+            query: {
+                bool: {
+                    must: [
+                        {term: {block_id: reqBody.block_id}}
+                    ]
+                }
+            }
+        };
+    } else {
+        if (targetBlock > 0) {
+            searchBody = {
+                query: {
+                    bool: {
+                        must: [
+                            {term: {block_num: targetBlock}}
+                        ]
+                    }
+                }
+            };
+        } else if (targetBlock < 0) {
+            searchBody = {
+                query: {match_all: {}},
+                sort: {block_num: "desc"}
+            }
+        }
+    }
+
+    const response = {transactions: []} as getBlockTraceResponse;
+
+    if (searchBody) {
+
+        const getBlockHeader = await fastify.elastic.search({
+            index: fastify.manager.chain + "-block-*",
+            size: 1,
+            body: searchBody
+        });
+
+        if (getBlockHeader.body.hits.hits.length === 1) {
+
+            const block = getBlockHeader.body.hits.hits[0]._source;
+            const info = await fastify.eosjs.rpc.get_info();
+
+            response.id = block.block_id;
+            response.number = block.block_num;
+            response.previous_id = block.prev_id;
+            response.status = info.last_irreversible_block_num > block.block_num ? "irreversible" : "pending"
+            response.timestamp = block['@timestamp'];
+            response.producer = block.producer;
+
+            // lookup all actions on block
+            const getActionsResponse = await fastify.elastic.search({
+                index: fastify.manager.chain + "-action-*",
+                size: fastify.manager.config.api.limits.get_actions || 1000,
+                body: {
+                    query: {
+                        bool: {
+                            must: [
+                                {term: {block_num: block.block_num}}
+                            ]
+                        }
+                    },
+                    sort: {global_sequence: "asc"}
+                }
+            });
+
+            const hits = getActionsResponse.body.hits.hits;
+            if (hits.length > 0) {
+                _.forEach(_.groupBy(hits, (v) => v['_source']['trx_id']), (value, key) => {
+                    const actArray = [];
+                    for (const act of value) {
+                        const action = act['_source']
+                        for (const receipt of action.receipts) {
+                            actArray.push({
+                                receiver: receipt.receiver,
+                                account: action.act.account,
+                                action: action.act.name,
+                                authorization: action.act.authorization.map(auth => {
+                                    return {account: auth.actor, permission: auth.permission};
+                                }),
+                                data: action.act.data
+                            });
+                        }
+                    }
+                    response.transactions.push({id: key, actions: actArray});
+                });
+            }
+            return response;
+        } else {
+            throw new Error('block not found!');
+        }
+    } else {
+        throw new Error("invalid block number or id");
+    }
+}
+
+export function getBlockTraceHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        reply.send(await timedQuery(getBlockTrace, fastify, request, route));
+    }
+}

api/routes/v1-trace/get_block/index.ts

@@ -0,0 +1,72 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getBlockTraceHandler} from "./get_block";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get block traces',
+        summary: 'get block traces',
+        tags: ['history'],
+        body: {
+            type: ['object', 'string'],
+            properties: {
+                "block_num": {
+                    description: 'block number',
+                    type: 'integer'
+                },
+                "block_id": {
+                    description: 'block id',
+                    type: 'string'
+                }
+            }
+        },
+        response: extendResponseSchema({
+            "id": {type: "string"},
+            "number": {type: "integer"},
+            "previous_id": {type: "string"},
+            "status": {type: "string"},
+            "timestamp": {type: "string"},
+            "producer": {type: "string"},
+            "transactions": {
+                type: "array",
+                items: {
+                    type: 'object',
+                    properties: {
+                        "id": {type: "string"},
+                        "actions": {
+                            type: "array",
+                            items: {
+                                type: 'object',
+                                properties: {
+                                    "receiver": {type: "string"},
+                                    "account": {type: "string"},
+                                    "action": {type: "string"},
+                                    "authorization": {
+                                        type: "array", items: {
+                                            type: 'object',
+                                            properties: {
+                                                "account": {type: "string"},
+                                                "permission": {type: "string"},
+                                            }
+                                        }
+                                    },
+                                    "data": {
+                                        additionalProperties: true
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        })
+    };
+    addApiRoute(
+        fastify,
+        'POST',
+        getRouteName(__filename),
+        getBlockTraceHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-history/get_abi_snapshot/get_abi_snapshot.ts

@@ -0,0 +1,50 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {ServerResponse} from "http";
+import {timedQuery} from "../../../helpers/functions";
+
+async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest) {
+
+    const response = {
+        block_num: null
+    };
+
+    const code = request.query.contract;
+    const block = request.query.block;
+    const should_fetch = request.query.fetch;
+
+    const mustArray = [];
+
+    mustArray.push({"term": {"account": code}});
+
+    if (block) {
+        mustArray.push({"range": {"block": {"lte": parseInt(block)}}});
+    }
+
+    const results = await fastify.elastic.search({
+        index: fastify.manager.chain + '-abi',
+        size: 1,
+        body: {
+            query: {bool: {must: mustArray}},
+            sort: [{block: {order: "desc"}}]
+        }
+    });
+    if (results['body']['hits']['hits'].length > 0) {
+        if (should_fetch) {
+            response['abi'] = JSON.parse(results['body']['hits']['hits'][0]['_source']['abi']);
+        } else {
+            response['present'] = true;
+        }
+        response.block_num = results['body']['hits']['hits'][0]['_source']['block'];
+    } else {
+        response['present'] = false;
+        response['error'] = 'abi not found for ' + code + ' until block ' + block;
+    }
+
+    return response;
+}
+
+export function getAbiSnapshotHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        reply.send(await timedQuery(getAbiSnapshot, fastify, request, route));
+    }
+}

api/routes/v2-history/get_abi_snapshot/index.ts

@@ -0,0 +1,40 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, getRouteName} from "../../../helpers/functions";
+import {getAbiSnapshotHandler} from "./get_abi_snapshot";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'fetch contract abi at specific block',
+        summary: 'fetch abi at specific block',
+        tags: ['history'],
+        querystring: {
+            type: 'object',
+            properties: {
+                "contract": {
+                    description: 'contract account',
+                    type: 'string',
+                    minLength: 1,
+                    maxLength: 12
+                },
+                "block": {
+                    description: 'target block',
+                    type: 'integer',
+                    minimum: 1
+                },
+                "fetch": {
+                    description: 'should fetch the ABI',
+                    type: 'boolean'
+                }
+            },
+            required: ["contract"]
+        }
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getAbiSnapshotHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-history/get_actions/definitions.ts

@@ -1,5 +1,5 @@
 export const terms = [
-    "notified.keyword",
+    "notified",
     "act.authorization.actor"
 ];
 
@@ -8,7 +8,10 @@ export const extendedActions = new Set([
     "newaccount",
     "updateauth",
     "buyram",
-    "buyrambytes"
+    "buyrambytes",
+    "delegatebw",
+    "undelegatebw",
+    "voteproducer"
 ]);
 
 export const primaryTerms = [
@@ -20,5 +23,6 @@ export const primaryTerms = [
     "creator_action_ordinal",
     "action_ordinal",
     "cpu_usage_us",
-    "net_usage_words"
+    "net_usage_words",
+    "trx_id"
 ];

api/routes/v2-history/get_actions/functions.ts

@@ -179,7 +179,7 @@ export function applyCodeActionFilters(query, queryStruct) {
     }
 }
 
-export function getSkipLimit(query) {
+export function getSkipLimit(query, max?: number) {
     let skip, limit;
     skip = parseInt(query.skip, 10);
     if (skip < 0) {
@@ -188,6 +188,8 @@ export function getSkipLimit(query) {
     limit = parseInt(query.limit, 10);
     if (limit < 1) {
         throw new Error('invalid limit parameter');
+    } else if (limit > max) {
+        throw new Error(`limit too big, maximum: ${max}`);
     }
     return {skip, limit};
 }

api/routes/v2-history/get_actions/get_actions.ts

@@ -13,6 +13,7 @@ import {
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
     const query = request.query;
+    const maxActions = fastify.manager.config.api.limits.get_actions;
     const queryStruct = {
         "bool": {
             must: [],
@@ -21,7 +22,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
         }
     };
 
-    const {skip, limit} = getSkipLimit(query);
+    const {skip, limit} = getSkipLimit(query, maxActions);
     const sort_direction = getSortDir(query);
     applyAccountFilters(query, queryStruct);
     applyGenericFilters(query, queryStruct);
@@ -40,24 +41,34 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
     addSortedBy(query, query_body, sort_direction);
 
     // Perform search
-    const maxActions = fastify.manager.config.api.limits.get_actions;
-    const pResults = await Promise.all([
-        fastify.eosjs.rpc.get_info(),
-        fastify.elastic.search({
-            "index": fastify.manager.chain + '-action-*',
-            "from": skip || 0,
-            "size": (limit > maxActions ? maxActions : limit) || 10,
-            "body": query_body
-        })
-    ]);
 
-    const results = pResults[1]['body']['hits'];
+    let indexPattern = fastify.manager.chain + '-action-*';
+    if (query.hot_only) {
+        indexPattern = fastify.manager.chain + '-action';
+    }
+
+    const esResults = await fastify.elastic.search({
+        "index": indexPattern,
+        "from": skip || 0,
+        "size": (limit > maxActions ? maxActions : limit) || 10,
+        "body": query_body
+    });
+
+    const results = esResults['body']['hits'];
     const response: any = {
         cached: false,
-        lib: pResults[0].last_irreversible_block_num,
+        lib: 0,
         total: results['total']
     };
 
+    if (query.hot_only) {
+        response.hot_only = true;
+    }
+
+    if (query.checkLib) {
+        response.lib = (await fastify.eosjs.rpc.get_info()).last_irreversible_block_num;
+    }
+
     if (query.simple) {
         response['simple_actions'] = [];
     } else {
@@ -69,10 +80,21 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
         for (let action of actions) {
             action = action._source;
             mergeActionMeta(action);
+
+            if (query.noBinary === true) {
+                for (const key in action['act']['data']) {
+                    if (action['act']['data'].hasOwnProperty(key)) {
+                        if (typeof action['act']['data'][key] === 'string' && action['act']['data'][key].length > 256) {
+                            action['act']['data'][key] = action['act']['data'][key].slice(0, 32) + "...";
+                        }
+                    }
+                }
+            }
+
             if (query.simple) {
                 response.simple_actions.push({
                     block: action['block_num'],
-                    irreversible: action['block_num'] < pResults[0].last_irreversible_block_num,
+                    irreversible: response.lib !== 0 ? action['block_num'] < response.lib : undefined,
                     timestamp: action['@timestamp'],
                     transaction_id: action['trx_id'],
                     actors: action['act']['authorization'].map(a => `${a.actor}@${a.permission}`).join(","),
@@ -81,6 +103,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
                     action: action['act']['name'],
                     data: action['act']['data']
                 });
+
             } else {
                 response.actions.push(action);
             }

api/routes/v2-history/get_actions/index.ts

@@ -2,12 +2,49 @@ import {FastifyInstance, RouteSchema} from "fastify";
 import {getActionsHandler} from "./get_actions";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
+export const getActionResponseSchema = {
+    "@timestamp": {type: "string"},
+    "timestamp": {type: "string"},
+    "block_num": {type: "number"},
+    "trx_id": {type: "string"},
+    "act": {
+        type: 'object',
+        properties: {
+            "account": {type: "string"},
+            "name": {type: "string"}
+        },
+        additionalProperties: true
+    },
+    "notified": {
+        type: "array", items: {type: "string"}
+    },
+    "cpu_usage_us": {type: "number"},
+    "net_usage_words": {type: "number"},
+    "account_ram_deltas": {
+        type: "array",
+        items: {
+            type: "object",
+            properties: {
+                "account": {type: "string"},
+                "delta": {type: "number"}
+            },
+            additionalProperties: true
+        }
+    },
+    "global_sequence": {type: "number"},
+    "receiver": {type: 'string'},
+    "producer": {type: "string"},
+    "parent": {type: "number"},
+    "action_ordinal": {type: 'number'},
+    "creator_action_ordinal": {type: 'number'}
+};
+
 export default function (fastify: FastifyInstance, opts: any, next) {
     const schema: RouteSchema = {
         description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
             ' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
         summary: 'get root actions',
-        tags: ['actions', 'history'],
+        tags: ['history'],
         querystring: extendQueryStringSchema({
             "account": {
                 description: 'notified account',
@@ -40,7 +77,19 @@ export default function (fastify: FastifyInstance, opts: any, next) {
             "simple": {
                 description: 'simplified output mode',
                 type: 'boolean'
-            }
+            },
+            "hot_only": {
+                description: 'search only the latest hot index',
+                type: 'boolean'
+            },
+            "noBinary": {
+                description: "exclude large binary data",
+                type: 'boolean'
+            },
+            "checkLib": {
+                description: "perform reversibility check",
+                type: 'boolean'
+            },
         }),
         response: extendResponseSchema({
             "simple_actions": {
@@ -66,42 +115,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                 type: "array",
                 items: {
                     type: 'object',
-                    properties: {
-                        "@timestamp": {type: "string"},
-                        "timestamp": {type: "string"},
-                        "block_num": {type: "number"},
-                        "trx_id": {type: "string"},
-                        "act": {
-                            type: 'object',
-                            properties: {
-                                "account": {type: "string"},
-                                "name": {type: "string"}
-                            },
-                            additionalProperties: true
-                        },
-                        "notified": {
-                            type: "array", items: {type: "string"}
-                        },
-                        "cpu_usage_us": {type: "number"},
-                        "net_usage_words": {type: "number"},
-                        "account_ram_deltas": {
-                            type: "array",
-                            items: {
-                                type: "object",
-                                properties: {
-                                    "account": {type: "string"},
-                                    "delta": {type: "number"}
-                                },
-                                additionalProperties: true
-                            }
-                        },
-                        "global_sequence": {type: "number"},
-                        "receiver": {type: 'string'},
-                        "producer": {type: "string"},
-                        "parent": {type: "number"},
-                        "action_ordinal": {type: 'number'},
-                        "creator_action_ordinal": {type: 'number'}
-                    }
+                    properties: getActionResponseSchema
                 }
             }
         })

api/routes/v2-history/get_created_accounts/get_created_accounts.ts

@@ -1,17 +1,17 @@
-const {getCreatedAccountsSchema} = require("../../schemas");
-const {getCacheByHash} = require("../../helpers/functions");
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {ServerResponse} from "http";
+import {timedQuery} from "../../../helpers/functions";
+import {getSkipLimit} from "../get_actions/functions";
 
-const route = '/get_created_accounts';
+async function getCreatedAccounts(fastify: FastifyInstance, request: FastifyRequest) {
 
-async function getCreatedAccounts(fastify, request) {
-    const t0 = Date.now();
-    const {redis, elastic} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, JSON.stringify(request.query));
-    if (cachedResponse) {
-        return cachedResponse;
-    }
-    const results = await elastic['search']({
-        "index": process.env.CHAIN + '-action-*',
+
+    const {skip, limit} = getSkipLimit(request.query);
+    const maxActions = fastify.manager.config.api.limits.get_created_accounts;
+    const results = await fastify.elastic.search({
+        "index": fastify.manager.chain + '-action-*',
+        "from": skip || 0,
+        "size": (limit > maxActions ? maxActions : limit) || 100,
         "body": {
             "query": {
                 "bool": {
@@ -27,9 +27,9 @@ async function getCreatedAccounts(fastify, request) {
             }
         }
     });
-    const response = {
-        "accounts": []
-    };
+
+    const response = {accounts: []};
+
     if (results['body']['hits']['hits'].length > 0) {
         const actions = results['body']['hits']['hits'];
         for (let action of actions) {
@@ -47,16 +47,12 @@ async function getCreatedAccounts(fastify, request) {
             response.accounts.push(_tmp);
         }
     }
-    response['query_time'] = Date.now() - t0;
-    redis.set(hash, JSON.stringify(response), 'EX', 30);
+
     return response;
 }
 
-module.exports = function (fastify, opts, next) {
-    fastify.get(route, {
-        schema: getCreatedAccountsSchema.GET
-    }, async (request) => {
-        return await getCreatedAccounts(fastify, request);
-    });
-    next()
-};
+export function getCreatedAccountsHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        reply.send(await timedQuery(getCreatedAccounts, fastify, request, route));
+    }
+}

api/routes/v2-history/get_created_accounts/index.ts

@@ -0,0 +1,44 @@
+import {FastifyInstance, RouteSchema} from "fastify";
+import {getCreatedAccountsHandler} from "./get_created_accounts";
+import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema: RouteSchema = {
+        description: 'get all accounts created by one creator',
+        summary: 'get created accounts',
+        tags: ['accounts'],
+        querystring: extendQueryStringSchema({
+            "account": {
+                description: 'creator account',
+                type: 'string',
+                minLength: 1,
+                maxLength: 12
+            }
+        }, ["account"]),
+        response: extendResponseSchema({
+            "query_time": {
+                type: "number"
+            },
+            "total": {
+                type: "object",
+                properties: {
+                    "value": {type: "number"},
+                    "relation": {type: "string"}
+                }
+            },
+            "accounts": {
+                type: "array",
+                items: {
+                    type: 'object',
+                    properties: {
+                        'name': {type: 'string'},
+                        'timestamp': {type: 'string'},
+                        'trx_id': {type: 'string'}
+                    }
+                }
+            }
+        })
+    };
+    addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatedAccountsHandler, schema);
+    next();
+}

api/routes/v2-history/get_creator/get_creator.ts

@@ -0,0 +1,99 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {ServerResponse} from "http";
+import {timedQuery} from "../../../helpers/functions";
+
+async function getCreator(fastify: FastifyInstance, request: FastifyRequest) {
+
+	const response = {
+		account: request.query.account,
+		creator: '',
+		timestamp: '',
+		block_num: 0,
+		trx_id: '',
+	};
+
+	if (request.query.account === fastify.manager.config.settings.eosio_alias) {
+		const genesisBlock = await fastify.eosjs.rpc.get_block(1);
+		if (genesisBlock) {
+			response.creator = '__self__';
+			response.timestamp = genesisBlock.timestamp;
+			response.block_num = 1;
+			response.trx_id = "";
+			return response;
+		} else {
+			throw new Error("genesis block not found");
+		}
+	}
+
+	const results = await fastify.elastic.search({
+		"index": fastify.manager.chain + '-action-*',
+		"body": {
+			size: 1,
+			query: {
+				bool: {
+					must: [{term: {"@newaccount.newact": request.query.account}}]
+				}
+			}
+		}
+	});
+
+	if (results['body']['hits']['hits'].length === 1) {
+		const result = results['body']['hits']['hits'][0]._source;
+		response.trx_id = result.trx_id;
+		response.block_num = result.block_num;
+		response.creator = result.act.data.creator;
+		response.timestamp = result['@timestamp'];
+		return response;
+	} else {
+		let accountInfo;
+		try {
+			accountInfo = await fastify.eosjs.rpc.get_account(request.query.account);
+		} catch (e) {
+			throw new Error("account not found");
+		}
+		if (accountInfo) {
+			try {
+				response.timestamp = accountInfo.created;
+				const blockHeader = await fastify.elastic.search({
+					"index": fastify.manager.chain + '-block-*',
+					"body": {
+						size: 1,
+						query: {
+							bool: {
+								must: [{term: {"@timestamp": response.timestamp}}]
+							}
+						}
+					}
+				});
+				const blockId = blockHeader['body']['hits']['hits'][0]._source.block_id;
+				const blockData = await fastify.eosjs.rpc.get_block(blockId);
+				response.block_num = blockData.block_num;
+				for (const transaction of blockData["transactions"]) {
+					if (typeof transaction.trx !== 'string') {
+						const actions = transaction.trx.transaction.actions;
+						for (const act of actions) {
+							if (act.name === 'newaccount') {
+								if (act.data.name === request.query.account) {
+									response.creator = act.data.creator;
+									response.trx_id = transaction.id;
+									return response;
+								}
+							}
+						}
+					}
+				}
+				return response;
+			} catch (e) {
+				throw new Error("account creation not found");
+			}
+		} else {
+			throw new Error("account creation not found");
+		}
+	}
+}
+
+export function getCreatorHandler(fastify: FastifyInstance, route: string) {
+	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+		reply.send(await timedQuery(getCreator, fastify, request, route));
+	}
+}

api/routes/v2-history/get_creator/index.ts

@@ -0,0 +1,45 @@
+import {FastifyInstance, RouteSchema} from "fastify";
+import {getCreatorHandler} from "./get_creator";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema: RouteSchema = {
+        description: 'get account creator',
+        summary: 'get account creator',
+        tags: ['accounts'],
+        querystring: {
+            type: 'object',
+            properties: {
+                "account": {
+                    description: 'created account',
+                    type: 'string',
+                    minLength: 1,
+                    maxLength: 12
+                }
+            },
+            required: ["account"]
+        },
+        response: extendResponseSchema({
+            "account": {
+                type: "string"
+            },
+            "creator": {
+                type: "string"
+            },
+            "timestamp": {
+                type: "string"
+            },
+            "block_num": {
+                type: "integer"
+            },
+            "trx_id": {
+                type: "string"
+            },
+            "indirect_creator": {
+                type: "string"
+            }
+        })
+    };
+    addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatorHandler, schema);
+    next();
+}

api/routes/v2-history/get_deltas/get_deltas.ts

@@ -1,76 +1,91 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {ServerResponse} from "http";
 import {mergeDeltaMeta, timedQuery} from "../../../helpers/functions";
+import {applyTimeFilter} from "../get_actions/functions";
 
 async function getDeltas(fastify: FastifyInstance, request: FastifyRequest) {
-    let skip, limit;
-    let sort_direction = 'desc';
-    const mustArray = [];
-    for (const param in request.query) {
-        if (Object.prototype.hasOwnProperty.call(request.query, param)) {
-            const value = request.query[param];
-            switch (param) {
-                case 'limit': {
-                    limit = parseInt(value, 10);
-                    if (limit < 1) {
-                        return 'invalid limit parameter';
-                    }
-                    break;
-                }
-                case 'skip': {
-                    skip = parseInt(value, 10);
-                    if (skip < 0) {
-                        return 'invalid skip parameter';
-                    }
-                    break;
-                }
-                case 'sort': {
-                    if (value === 'asc' || value === '1') {
-                        sort_direction = 'asc';
-                    } else if (value === 'desc' || value === '-1') {
-                        sort_direction = 'desc'
-                    } else {
-                        return 'invalid sort direction';
-                    }
-                    break;
-                }
-                default: {
-                    const values = request.query[param].split(",");
-                    const terms = {};
-                    terms[param] = values;
-                    const shouldArray = {terms: terms};
-                    const boolStruct = {bool: {should: [shouldArray]}};
-                    mustArray.push(boolStruct);
-                    break;
-                }
-            }
-        }
-    }
+	let skip, limit;
+	let sort_direction = 'desc';
+	const mustArray = [];
+	for (const param in request.query) {
+		if (Object.prototype.hasOwnProperty.call(request.query, param)) {
+			const value = request.query[param];
+			switch (param) {
+				case 'limit': {
+					limit = parseInt(value, 10);
+					if (limit < 1) {
+						return 'invalid limit parameter';
+					}
+					break;
+				}
+				case 'skip': {
+					skip = parseInt(value, 10);
+					if (skip < 0) {
+						return 'invalid skip parameter';
+					}
+					break;
+				}
+				case 'sort': {
+					if (value === 'asc' || value === '1') {
+						sort_direction = 'asc';
+					} else if (value === 'desc' || value === '-1') {
+						sort_direction = 'desc'
+					} else {
+						return 'invalid sort direction';
+					}
+					break;
+				}
+				case 'before': {
+					break;
+				}
+				case 'after': {
+					break;
+				}
+				default: {
+					const values = request.query[param].split(",");
+					if (values.length > 1) {
+						const terms = {};
+						terms[param] = values;
+						mustArray.push({terms: terms});
+					} else {
+						const term = {};
+						term[param] = values[0];
+						mustArray.push({term: term});
+					}
+					break;
+				}
+			}
+		}
+	}
 
-    const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
-    const results = await fastify.elastic.search({
-        "index": fastify.manager.chain + '-delta-*',
-        "from": skip || 0,
-        "size": (limit > maxDeltas ? maxDeltas : limit) || 10,
-        "body": {
-            query: {bool: {must: mustArray}},
-            sort: {
-                "block_num": sort_direction
-            }
-        }
-    });
-    const deltas = results['body']['hits']['hits'].map((d) => {
-        return mergeDeltaMeta(d._source);
-    });
-    return {
-        query_time: null,
-        total: results['body']['hits']['total'],
-        deltas
-    };
+	const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
+	const queryStruct = {bool: {must: mustArray}};
+
+	applyTimeFilter(request.query, queryStruct);
+
+	const results = await fastify.elastic.search({
+		"index": fastify.manager.chain + '-delta-*',
+		"from": skip || 0,
+		"size": (limit > maxDeltas ? maxDeltas : limit) || 10,
+		"body": {
+			query: queryStruct,
+			sort: {
+				"block_num": sort_direction
+			}
+		}
+	});
+	const deltas = results['body']['hits']['hits'].map((d) => {
+		return mergeDeltaMeta(d._source);
+	});
+	return {
+		query_time: null,
+		total: results['body']['hits']['total'],
+		deltas
+	};
 }
 
 export function getDeltasHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getDeltas, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+		reply.send(await timedQuery(getDeltas, fastify, request, route));
+	}
 }

api/routes/v2-history/get_deltas/index.ts

@@ -23,6 +23,14 @@ export default function (fastify: FastifyInstance, opts: any, next) {
             "payer": {
                 description: 'payer account',
                 type: 'string'
+            },
+            "after": {
+                description: 'filter after specified date (ISO8601)',
+                type: 'string'
+            },
+            "before": {
+                description: 'filter before specified date (ISO8601)',
+                type: 'string'
             }
         }),
         response: extendResponseSchema({
@@ -39,6 +47,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                         "payer": {type: 'string'},
                         "present": {type: 'boolean'},
                         "block_num": {type: 'number'},
+                        "block_id": {type: 'string'},
                         "data": {
                             type: 'object',
                             additionalProperties: true

api/routes/v2-history/get_schedule/get_schedule.ts

@@ -0,0 +1,71 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {ServerResponse} from "http";
+import {timedQuery} from "../../../helpers/functions";
+import {createHash} from "crypto";
+import {base58ToBinary, binaryToBase58} from "eosjs/dist/eosjs-numeric";
+import {Search} from "@elastic/elasticsearch/api/requestParams";
+
+function convertToLegacyKey(block_signing_key: string) {
+    if (block_signing_key.startsWith("PUB_K1_")) {
+        const buf = base58ToBinary(37, block_signing_key.substr(7));
+        const data = buf.slice(0, buf.length - 4);
+        const merged = Buffer.concat([
+            data,
+            createHash('ripemd160')
+                .update(data)
+                .digest()
+                .slice(0, 4)
+        ]);
+        return "EOS" + binaryToBase58(merged);
+    } else {
+        return block_signing_key;
+    }
+}
+
+async function getSchedule(fastify: FastifyInstance, request: FastifyRequest) {
+    const response: any = {
+        producers: []
+    };
+    const searchParams: Search<any> = {
+        track_total_hits: true,
+        index: fastify.manager.chain + "-block-*",
+        size: 1,
+        body: {
+            query: {
+                bool: {
+                    must: []
+                }
+            },
+            sort: {block_num: "desc"}
+        }
+    };
+    if (request.query.version) {
+        searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": request.query.version}}});
+    } else {
+        searchParams.body.query.bool.must.push({
+            "exists": {
+                "field": "new_producers.version"
+            }
+        });
+    }
+    const apiResponse = await fastify.elastic.search(searchParams);
+    const results = apiResponse.body.hits.hits;
+    if (results) {
+        response.timestamp = results[0]._source["@timestamp"];
+        response.block_num = results[0]._source.block_num;
+        response.version = results[0]._source.new_producers.version;
+        response.producers = results[0]._source.new_producers.producers.map(prod => {
+            return {
+                ...prod,
+                legacy_key: convertToLegacyKey(prod.block_signing_key)
+            }
+        });
+    }
+    return response;
+}
+
+export function getScheduleHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        reply.send(await timedQuery(getSchedule, fastify, request, route));
+    }
+}

api/routes/v2-history/get_schedule/index.ts

@@ -0,0 +1,61 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getScheduleHandler} from "./get_schedule";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get producer schedule by version',
+        summary: 'get producer schedule by version',
+        tags: ['history'],
+        querystring: {
+            type: 'object',
+            properties: {
+                "producer": {
+                    description: 'search by producer',
+                    type: 'string'
+                },
+                "key": {
+                    description: 'search by key',
+                    type: 'string'
+                },
+                "after": {
+                    description: 'filter after specified date (ISO8601)',
+                    type: 'string'
+                },
+                "before": {
+                    description: 'filter before specified date (ISO8601)',
+                    type: 'string'
+                },
+                "version": {
+                    description: 'schedule version',
+                    type: 'integer',
+                    minimum: 1
+                }
+            }
+        },
+        response: extendResponseSchema({
+            "timestamp": {type: "string"},
+            "block_num": {type: "number"},
+            "version": {type: "number"},
+            "producers": {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        "producer_name": {type: "string"},
+                        "block_signing_key": {type: "string"},
+                        "legacy_key": {type: "string"}
+                    }
+                }
+            },
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getScheduleHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-history/get_transaction/get_transaction.ts

@@ -3,43 +3,111 @@ import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
-    const pResults = await Promise.all([
-        fastify.eosjs.rpc.get_info(),
-        fastify.elastic.search({
-            "index": fastify.manager.chain + '-action-*',
-            "body": {
-                "query": {
-                    "bool": {
-                        must: [
-                            {term: {"trx_id": request.query.id.toLowerCase()}}
-                        ]
-                    }
-                },
-                "sort": {
-                    "global_sequence": "asc"
-                }
-            }
-        })
-    ]);
-    const results = pResults[1];
-    const response = {
-        "trx_id": request.query.id,
-        "lib": pResults[0].last_irreversible_block_num,
-        "actions": []
-    };
-    const hits = results['body']['hits']['hits'];
-    if (hits.length > 0) {
-        for (let action of hits) {
-            action = action._source;
-            mergeActionMeta(action);
-            response.actions.push(action);
-        }
-    }
-    return response;
+
+	const redis = fastify.redis;
+	const trxId = request.query.id.toLowerCase();
+	const cachedData = await redis.hgetall(trxId);
+
+	let hits;
+	let hits2;
+
+	const response = {
+		"query_time_ms": undefined,
+		"cached": undefined,
+		"cache_expires_in": undefined,
+		"executed": false,
+		"hot_only": false,
+		"trx_id": trxId,
+		"lib": undefined,
+		"actions": [],
+		"generated": undefined
+	};
+
+	if (cachedData && Object.keys(cachedData).length > 0) {
+		const gsArr = [];
+		for (let cachedDataKey in cachedData) {
+			gsArr.push(cachedData[cachedDataKey]);
+		}
+		gsArr.sort((a, b) => {
+			return a.global_sequence - b.global_sequence;
+		});
+		response.cache_expires_in = await redis.ttl(trxId);
+		hits = gsArr.map(value => {
+			return {_source: JSON.parse(value)}
+		});
+	}
+
+	if (!hits) {
+		const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
+		let indexPattern = fastify.manager.chain + '-action-*';
+		if (request.query.hot_only) {
+			indexPattern = fastify.manager.chain + '-action';
+		}
+
+		const pResults = await Promise.all([
+			fastify.eosjs.rpc.get_info(),
+			fastify.elastic.search({
+				index: indexPattern,
+				size: _size,
+				body: {
+					query: {bool: {must: [{term: {trx_id: request.query.id.toLowerCase()}}]}},
+					sort: {global_sequence: "asc"}
+				}
+			}),
+			fastify.elastic.search({
+				index: fastify.manager.chain + '-gentrx-*',
+				size: _size,
+				body: {
+					query: {bool: {must: [{term: {trx_id: request.query.id.toLowerCase()}}]}}
+				}
+			})
+		]);
+
+		const results = pResults[1];
+		const genTrxRes = pResults[2];
+
+		response.lib = pResults[0].last_irreversible_block_num;
+
+		if (request.query.hot_only) {
+			response.hot_only = true;
+		}
+
+		hits = results['body']['hits']['hits'];
+		hits2 = genTrxRes['body']['hits']['hits'];
+	}
+
+	if (hits) {
+		if (hits.length > 0) {
+			let highestBlockNum = 0;
+			for (let action of hits) {
+				if (action._source.block_num > highestBlockNum) {
+					highestBlockNum = action._source.block_num;
+				}
+			}
+			for (let action of hits) {
+				if (action._source.block_num === highestBlockNum) {
+					action = action._source;
+					mergeActionMeta(action);
+					response.actions.push(action);
+				}
+			}
+			response.executed = true;
+		}
+	}
+
+	if (hits2 && hits2.length > 0) {
+		if (hits2[0]._source['@timestamp']) {
+			hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
+			delete hits2[0]._source['@timestamp'];
+		}
+		response.generated = hits2[0]._source;
+	}
+
+	return response;
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getTransaction, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+		reply.send(await timedQuery(getTransaction, fastify, request, route));
+	}
 }

api/routes/v2-history/get_transaction/index.ts

@@ -6,7 +6,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
     const schema = {
         description: 'get all actions belonging to the same transaction',
         summary: 'get transaction by id',
-        tags: ['transactions', 'history'],
+        tags: ['history'],
         querystring: {
             type: 'object',
             properties: {

api/routes/v2-history/get_transaction_legacy/get_transaction_legacy.ts

@@ -0,0 +1,128 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {ServerResponse} from "http";
+import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
+
+async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
+    const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
+
+    let indexPattern = fastify.manager.chain + '-action-*';
+    if (request.query.hot_only) {
+        indexPattern = fastify.manager.chain + '-action';
+    }
+
+    const pResults = await Promise.all([
+        fastify.eosjs.rpc.get_info(),
+        fastify.elastic.search({
+            index: indexPattern,
+            size: _size,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {term: {trx_id: request.query.id.toLowerCase()}}
+                        ]
+                    }
+                },
+                sort: {
+                    global_sequence: "asc"
+                }
+            }
+        }),
+        fastify.elastic.search({
+            index: fastify.manager.chain + '-gentrx-*',
+            size: _size,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {term: {trx_id: request.query.id.toLowerCase()}}
+                        ]
+                    }
+                }
+            }
+        })
+    ]);
+
+    const results = pResults[1];
+    const genTrxRes = pResults[2];
+
+    const response = {
+        "executed": false,
+        "hot_only": false,
+        "trx_id": request.query.id,
+        "lib": pResults[0].last_irreversible_block_num,
+        "actions": [],
+        "generated": undefined
+    };
+
+    if (request.query.hot_only) {
+        response.hot_only = true;
+    }
+
+    const hits = results['body']['hits']['hits'];
+
+    if (hits.length > 0) {
+
+        // const producers = {};
+        // for (let hit of hits) {
+        // 	if (hit._source.producer) {
+        // 		if (producers[hit._source.producer]) {
+        // 			producers[hit._source.producer]++;
+        // 		} else {
+        // 			producers[hit._source.producer] = 1;
+        // 		}
+        // 	}
+        // }
+        // let useBlocknumber;
+        // if (Object.keys(producers).length > 1) {
+        // 	// multiple producers of the same tx id, forked actions are present, attempt to cleanup
+        // 	let trueProd = '';
+        // 	let highestActCount = 0;
+        // 	for (const prod in producers) {
+        // 		if (producers.hasOwnProperty(prod)) {
+        // 			if(producers[prod] === highestActCount) {
+        // 				useBlocknumber = true;
+        // 			} else if (producers[prod] > highestActCount) {
+        // 				highestActCount = producers[prod];
+        // 				trueProd = prod;
+        // 			}
+        // 		}
+        // 	}
+        // }
+
+        let highestBlockNum = 0;
+        for (let action of hits) {
+            if (action._source.block_num > highestBlockNum) {
+                highestBlockNum = action._source.block_num;
+            }
+        }
+
+        for (let action of hits) {
+            if (action._source.block_num === highestBlockNum) {
+                action = action._source;
+                mergeActionMeta(action);
+                response.actions.push(action);
+            }
+        }
+
+        response.executed = true;
+    }
+
+    const hits2 = genTrxRes['body']['hits']['hits'];
+
+    if (hits2 && hits2.length > 0) {
+        if (hits2[0]._source['@timestamp']) {
+            hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
+            delete hits2[0]._source['@timestamp'];
+        }
+        response.generated = hits2[0]._source;
+    }
+
+    return response;
+}
+
+export function getTransactionHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        reply.send(await timedQuery(getTransaction, fastify, request, route));
+    }
+}

api/routes/v2-history/get_transaction_legacy/index.ts

@@ -0,0 +1,29 @@
+import {FastifyInstance} from "fastify";
+import {getTransactionHandler} from "./get_transaction_legacy";
+import {addApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get all actions belonging to the same transaction',
+        summary: 'get transaction by id',
+        tags: ['history'],
+        querystring: {
+            type: 'object',
+            properties: {
+                "id": {
+                    description: 'transaction id',
+                    type: 'string'
+                }
+            },
+            required: ["id"]
+        }
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getTransactionHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-state/get_account/get_account.ts

@@ -6,32 +6,41 @@ import {timedQuery} from "../../../helpers/functions";
 async function getAccount(fastify: FastifyInstance, request: FastifyRequest) {
 
     const response = {
-        query_time: null,
-        cached: false,
         account: null,
         actions: null,
+        total_actions: 0,
         tokens: null,
         links: null
     };
 
     const account = request.query.account;
     const reqQueue = [];
-    reqQueue.push(fastify.eosjs.rpc.get_account(account));
+
+    try {
+        response.account = await fastify.eosjs.rpc.get_account(account);
+    } catch (e) {
+        throw new Error("Account not found!");
+    }
 
     const localApi = `http://${fastify.manager.config.api.server_addr}:${fastify.manager.config.api.server_port}/v2`;
     const getTokensApi = localApi + '/state/get_tokens';
     const getActionsApi = localApi + '/history/get_actions';
+    const getLinksApi = localApi + '/state/get_links';
 
     // fetch recent actions
-    reqQueue.push(got.get(`${getActionsApi}?account=${account}&limit=10`));
+    reqQueue.push(got.get(`${getActionsApi}?account=${account}&limit=20&noBinary=true&track=true`).json());
 
     // fetch account tokens
-    reqQueue.push(got.get(`${getTokensApi}?account=${account}`));
+    reqQueue.push(got.get(`${getTokensApi}?account=${account}`).json());
+
+    // fetch account permission links
+    reqQueue.push(got.get(`${getLinksApi}?account=${account}`).json());
 
     const results = await Promise.all(reqQueue);
-    response.account = results[0];
-    response.actions = JSON.parse(results[1].body).actions;
-    response.tokens = JSON.parse(results[2].body).tokens;
+    response.actions = results[0].actions;
+    response.total_actions = results[0].total.value;
+    response.tokens = results[1].tokens;
+    response.links = results[2].links;
     return response;
 }
 

api/routes/v2-state/get_account/index.ts

@@ -1,21 +1,58 @@
 import {FastifyInstance} from "fastify";
 import {getAccountHandler} from "./get_account";
-import {addApiRoute, getRouteName} from "../../../helpers/functions";
+import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getActionResponseSchema} from "../../v2-history/get_actions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
     const schema = {
         description: 'get account data',
         summary: 'get account summary',
-        tags: ['accounts', 'state'],
-        querystring: {
-            type: 'object',
-            properties: {
-                "account": {
-                    description: 'account name',
-                    type: 'string'
-                }
+        tags: ['accounts'],
+        querystring: extendQueryStringSchema({
+            "account": {
+                description: 'account name',
+                type: 'string',
+                minLength: 1,
+                maxLength: 12
             }
-        }
+        }, ["account"]),
+        response: extendResponseSchema({
+            "account": {
+                additionalProperties: true
+            },
+            "links": {
+                type: "array",
+                items: {
+                    type: 'object',
+                    properties: {
+                        "timestamp": {type: "string"},
+                        "permission": {type: "string"},
+                        "code": {type: "string"},
+                        "action": {type: "string"},
+                    }
+                }
+            },
+            "tokens": {
+                type: "array",
+                items: {
+                    type: 'object',
+                    properties: {
+                        "symbol": {type: "string"},
+                        "precision": {type: "integer"},
+                        "amount": {type: "number"},
+                        "contract": {type: "string"},
+                    }
+                }
+            },
+            "total_actions": {type: "number"},
+            "actions": {
+                type: "array",
+                items: {
+                    type: 'object',
+                    properties: getActionResponseSchema
+                }
+            },
+        })
     };
     addApiRoute(
         fastify,

api/routes/v2-state/get_key_accounts/get_key_accounts.ts

@@ -2,6 +2,7 @@ import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {Numeric} from "eosjs/dist";
+import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 function invalidKey() {
     const err: any = new Error();
@@ -13,6 +14,13 @@ function invalidKey() {
 async function getKeyAccounts(fastify: FastifyInstance, request: FastifyRequest) {
 
     let publicKey;
+    if (typeof request.body === 'string' && request.req.method === 'POST') {
+        request.body = JSON.parse(request.body);
+    }
+
+    const {skip, limit} = getSkipLimit(request.query);
+    const maxDocs = fastify.manager.config.api.limits.get_key_accounts ?? 1000;
+
     const public_Key = request.req.method === 'POST' ? request.body.public_key : request.query.public_key;
 
     if (public_Key.startsWith("PUB_")) {
@@ -21,13 +29,61 @@ async function getKeyAccounts(fastify: FastifyInstance, request: FastifyRequest)
         try {
             publicKey = Numeric.convertLegacyPublicKey(public_Key);
         } catch (e) {
-            console.log(e);
+            console.log(e.message);
             invalidKey();
         }
     } else {
         invalidKey();
     }
 
+    const response = {
+        account_names: []
+    } as any;
+
+    if (request.req.method === 'GET' && request.query.details) {
+        response.permissions = [];
+    }
+
+    try {
+
+        const permTableResults = await fastify.elastic.search({
+            index: fastify.manager.chain + '-perm-*',
+            size: (limit > maxDocs ? maxDocs : limit) || 100,
+            from: skip || 0,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {
+                                term: {
+                                    "auth.keys.key.keyword": publicKey
+                                }
+                            }
+                        ],
+                    }
+                }
+            }
+        });
+
+        if (permTableResults.body.hits.hits.length > 0) {
+            for (const perm of permTableResults.body.hits.hits) {
+                response.account_names.push(perm._source.owner);
+                if (request.req.method === 'GET' && request.query.details) {
+                    response.permissions.push(perm._source);
+                }
+            }
+        }
+
+    } catch (e) {
+        console.log(e);
+    }
+
+    if (response.account_names.length > 0) {
+        response.account_names = [...(new Set(response.account_names))];
+        return response;
+    }
+
+    // Fallback to action search
     const _body = {
         query: {
             bool: {
@@ -44,12 +100,11 @@ async function getKeyAccounts(fastify: FastifyInstance, request: FastifyRequest)
 
     const results = await fastify.elastic.search({
         index: fastify.manager.chain + '-action-*',
+        size: (limit > maxDocs ? maxDocs : limit) || 100,
+        from: skip || 0,
         body: _body
     });
 
-    const response = {
-        account_names: []
-    };
     if (results['body']['hits']['hits'].length > 0) {
         response.account_names = results['body']['hits']['hits'].map((v) => {
             if (v._source.act.name === 'newaccount') {
@@ -67,15 +122,12 @@ async function getKeyAccounts(fastify: FastifyInstance, request: FastifyRequest)
             }
         });
     }
+
     if (response.account_names.length > 0) {
         response.account_names = [...(new Set(response.account_names))];
-        return response;
-    } else {
-        const err: any = new Error();
-        err.statusCode = 404;
-        err.message = 'no accounts associated with ' + public_Key;
-        throw err;
     }
+
+    return response;
 }
 
 export function getKeyAccountsHandler(fastify: FastifyInstance, route: string) {

api/routes/v2-state/get_key_accounts/index.ts

@@ -1,6 +1,6 @@
 import {FastifyInstance} from "fastify";
 import {getKeyAccountsHandler} from "./get_key_accounts";
-import {addApiRoute, getRouteName} from "../../../helpers/functions";
+import {addApiRoute, extendQueryStringSchema, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
 
@@ -8,25 +8,38 @@ export default function (fastify: FastifyInstance, opts: any, next) {
     const getSchema = {
         description: 'get accounts by public key',
         summary: 'get accounts by public key',
-        tags: ['accounts', 'state'],
-        querystring: {
-            type: 'object',
-            properties: {
-                "public_key": {
-                    description: 'public key',
-                    type: 'string'
-                },
+        tags: ['accounts'],
+        querystring: extendQueryStringSchema({
+            "public_key": {
+                description: 'public key',
+                type: 'string'
             },
-            required: ["public_key"]
-        },
+            "details": {
+                description: 'include permission details',
+                type: 'boolean'
+            },
+        }, ["public_key"]),
         response: {
             200: {
                 type: 'object',
                 properties: {
                     "account_names": {
+                        type: "array",
+                        items: {type: "string"}
+                    },
+                    "permissions": {
                         type: "array",
                         items: {
-                            type: "string"
+                            type: "object",
+                            properties: {
+                                owner: {type: 'string'},
+                                block_num: {type: 'integer'},
+                                parent: {type: 'string'},
+                                last_updated: {type: 'string'},
+                                auth: {},
+                                name: {type: 'string'},
+                                present: {type: 'boolean'}
+                            }
                         }
                     }
                 }

api/routes/v2-state/get_links/get_links.ts

@@ -31,6 +31,9 @@ async function getLinks(fastify: FastifyInstance, request: FastifyRequest) {
         queryStruct.bool.must.push({'term': {'permissions': permissions}});
     }
 
+    // only present deltas
+    queryStruct.bool.must.push({'term': {'present': true}});
+
     // Prepare query body
     const query_body = {
         track_total_hits: getTrackTotalHits(query),
@@ -42,7 +45,7 @@ async function getLinks(fastify: FastifyInstance, request: FastifyRequest) {
     const results: ApiResponse = await fastify.elastic.search({
         index: fastify.manager.chain + '-link-*',
         from: skip || 0,
-        size: (limit > maxLinks ? maxLinks : limit) || 10,
+        size: (limit > maxLinks ? maxLinks : limit) || 50,
         body: query_body
     });
     const hits = results.body.hits.hits;
@@ -51,6 +54,7 @@ async function getLinks(fastify: FastifyInstance, request: FastifyRequest) {
         total: results.body.hits.total,
         links: []
     };
+
     for (const hit of hits) {
         const link = hit._source;
         link.timestamp = link['@timestamp'];

api/routes/v2-state/get_links/index.ts

@@ -6,7 +6,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
     const schema = {
         description: 'get permission links',
         summary: 'get permission links',
-        tags: ['accounts', 'state'],
+        tags: ['accounts'],
         querystring: {
             type: 'object',
             properties: {

api/routes/v2-state/get_proposals/get_proposals.ts

@@ -21,16 +21,19 @@ async function getProposals(fastify: FastifyInstance, request: FastifyRequest) {
         }
     };
 
-    // Filter by account
+    // Filter by accounts
     if (request.query.account) {
-        queryStruct.bool.must.push({
-            "bool": {
-                "should": [
-                    {"term": {"requested_approvals.actor": request.query.account}},
-                    {"term": {"provided_approvals.actor": request.query.account}}
-                ]
-            }
-        });
+        const accounts = request.query.account.split(',');
+        for(const acc of accounts) {
+            queryStruct.bool.must.push({
+                "bool": {
+                    "should": [
+                        {"term": {"requested_approvals.actor": acc}},
+                        {"term": {"provided_approvals.actor": acc}}
+                    ]
+                }
+            });
+        }
     }
 
     // Filter by proposer account

api/routes/v2-state/get_proposals/index.ts

@@ -6,7 +6,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
     const schema = {
         description: 'get proposals',
         summary: 'get proposals',
-        tags: ['state'],
+        tags: ['system'],
         querystring: {
             type: 'object',
             properties: {

api/routes/v2-state/get_tokens/get_tokens.ts

@@ -1,63 +1,67 @@
 import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 
 async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
 
-    const response = {
-        query_time: null,
-        cached: false,
-        'account': request.query.account,
-        'tokens': []
-    };
+    const response = {'account': request.query.account, 'tokens': []};
 
-    const results = await fastify.elastic.search({
-        "index": fastify.manager.chain + '-action-*',
+    const {skip, limit} = getSkipLimit(request.query);
+    const maxDocs = fastify.manager.config.api.limits.get_tokens ?? 100;
+
+    const stateResult = await fastify.elastic.search({
+        "index": fastify.manager.chain + '-table-accounts-*',
+        "size": (limit > maxDocs ? maxDocs : limit) || 50,
+        "from": skip || 0,
         "body": {
-            size: 0,
             query: {
                 bool: {
-                    filter: [
-                        {term: {"notified": request.query.account}},
-                        {terms: {"act.name": ["transfer", "issue"]}}
-                    ]
-                }
-            },
-            aggs: {
-                tokens: {
-                    terms: {
-                        field: "act.account",
-                        size: 1000
-                    }
+                    filter: [{term: {"scope": request.query.account}}]
                 }
             }
         }
     });
 
-    for (const bucket of results['body']['aggregations']['tokens']['buckets']) {
-        let token_data;
-        try {
-            token_data = await fastify.eosjs.rpc.get_currency_balance(bucket['key'], request.query.account);
-        } catch (e) {
-            console.log(`get_currency_balance error - contract:${bucket['key']} - account:${request.query.account}`);
+    for (const hit of stateResult.body.hits.hits) {
+        const data = hit._source;
+        if (typeof data.present !== "undefined" && data.present === false) {
             continue;
         }
-        for (const entry of token_data) {
-            let precision = 0;
-            const [amount, symbol] = entry.split(" ");
-            const amount_arr = amount.split(".");
-            if (amount_arr.length === 2) {
-                precision = amount_arr[1].length;
-            }
-            response.tokens.push({
-                symbol: symbol,
-                precision: precision,
-                amount: parseFloat(amount),
-                contract: bucket['key']
-            });
+        let precision;
+        const key = `${data.code}_${data.symbol}`;
+        if (!fastify.tokenCache) {
+            fastify.tokenCache = new Map<string, any>();
         }
+        if (fastify.tokenCache.has(key)) {
+            precision = fastify.tokenCache.get(key).precision;
+        } else {
+            let token_data;
+            try {
+                token_data = await fastify.eosjs.rpc.get_currency_balance(data.code, request.query.account, data.symbol);
+                if (token_data.length > 0) {
+                    const [amount, symbol] = token_data[0].split(" ");
+                    const amount_arr = amount.split(".");
+                    if (amount_arr.length === 2) {
+                        precision = amount_arr[1].length;
+                        fastify.tokenCache.set(key, {precision});
+                        // console.log('Caching token precision -', key, precision);
+                    }
+                }
+            } catch (e) {
+                console.log(`get_currency_balance error - contract:${data.code} - account:${request.query.account}`);
+            }
+        }
+
+        response.tokens.push({
+            symbol: data.symbol,
+            precision: precision,
+            amount: parseFloat(data.amount),
+            contract: data.code
+        });
     }
+
     return response;
 }
 

api/routes/v2-state/get_tokens/index.ts

@@ -1,21 +1,20 @@
 import {FastifyInstance} from "fastify";
-import {addApiRoute, getRouteName} from "../../../helpers/functions";
+import {addApiRoute, extendQueryStringSchema, getRouteName} from "../../../helpers/functions";
 import {getTokensHandler} from "./get_tokens";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
     const schema = {
-        description: 'get account data',
-        summary: 'get account summary',
-        tags: ['accounts', 'state'],
-        querystring: {
-            type: 'object',
-            properties: {
-                "account": {
-                    description: 'account name',
-                    type: 'string'
-                }
+        description: 'get tokens from an account',
+        summary: 'get all tokens',
+        tags: ['accounts'],
+        querystring: extendQueryStringSchema({
+            "account": {
+                description: 'account name',
+                type: 'string',
+                minLength: 1,
+                maxLength: 12
             }
-        }
+        }, ["account"])
     };
     addApiRoute(
         fastify,

api/routes/v2-state/get_voters/get_voters.ts

@@ -1,22 +1,13 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
+import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 async function getVoters(fastify: FastifyInstance, request: FastifyRequest) {
 
-    let skip, limit;
-
-    skip = parseInt(request.query.skip, 10);
-    if (skip < 0) {
-        return 'invalid skip parameter';
-    }
-    limit = parseInt(request.query.limit, 10);
-    if (limit < 1) {
-        return 'invalid limit parameter';
-    }
+    const {skip, limit} = getSkipLimit(request.query);
 
     const response = {
-        query_time: null,
         voter_count: 0,
         'voters': []
     };

api/routes/v2-state/get_voters/index.ts

@@ -1,31 +1,36 @@
 import {FastifyInstance} from "fastify";
 import {getVotersHandler} from "./get_voters";
-import {addApiRoute, getRouteName} from "../../../helpers/functions";
+import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
     const schema = {
         description: 'get voters',
         summary: 'get voters',
-        tags: ['accounts', 'state'],
-        querystring: {
-            type: 'object',
-            properties: {
-                "producer": {
-                    description: 'filter by voted producer (comma separated)',
-                    type: 'string'
-                },
-                "skip": {
-                    description: 'skip [n] actions (pagination)',
-                    type: 'integer',
-                    minimum: 0
-                },
-                "limit": {
-                    description: 'limit of [n] actions per page',
-                    type: 'integer',
-                    minimum: 1
+        tags: ['system'],
+        querystring: extendQueryStringSchema({
+            "producer": {
+                description: 'filter by voted producer (comma separated)',
+                type: 'string',
+                minLength: 1,
+                maxLength: 12
+            }
+        }),
+        response: extendResponseSchema({
+            "voters": {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        "account": {type: "string"},
+                        "weight": {type: "number"},
+                        "last_vote": {type: "number"},
+                        "data": {
+                            additionalProperties: true
+                        }
+                    }
                 }
             }
-        }
+        })
     };
     addApiRoute(
         fastify,

api/routes/v2-stats/get_action_usage/get_action_usage.ts

@@ -0,0 +1,121 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {ServerResponse} from "http";
+import {timedQuery} from "../../../helpers/functions";
+
+async function getLastSeq(fastify: FastifyInstance, date: string) {
+    const req = await fastify.elastic.search({
+        index: fastify.manager.chain + '-action-*',
+        body: {
+            "size": 1,
+            "query": {
+                "bool": {
+                    "must": [
+                        {"range": {"@timestamp": {"lt": date}}}
+                    ]
+                }
+            },
+            "_source": {
+                "includes": ["global_sequence"]
+            },
+            "sort": [{"global_sequence": "desc"}]
+        }
+    });
+    return req.body.hits.hits[0]._source.global_sequence;
+}
+
+async function getTxCount(fastify: FastifyInstance, dateFrom: string, dateTo: string) {
+    const req = await fastify.elastic.count({
+        index: fastify.manager.chain + '-action-*',
+        body: {
+            "query": {
+                "bool": {
+                    "must": [
+                        {"term": {"creator_action_ordinal": 0}},
+                        {"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}
+                    ]
+                }
+            }
+        }
+    });
+    return req.body.count;
+}
+
+async function getUniqueActors(fastify: FastifyInstance, dateFrom: string, dateTo: string) {
+    const req = await fastify.elastic.search({
+        index: fastify.manager.chain + '-action-*',
+        size: 0,
+        body: {
+            "aggs": {
+                "unique_actors": {
+                    "cardinality": {
+                        "field": "act.authorization.actor"
+                    }
+                }
+            },
+            "query": {
+                "bool": {
+                    "filter": [{"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}]
+                }
+            }
+        }
+    });
+    return req.body.aggregations.unique_actors.value;
+}
+
+async function getActionUsage(fastify: FastifyInstance, request: FastifyRequest) {
+    const query = request.query;
+    let now = new Date();
+    let expiration = 86400;
+    if (query.end_date) {
+        now = new Date(query.end_date);
+    }
+    now.setMinutes(0, 0, 0);
+    let period;
+    if (query.period === '1h') {
+        period = 3600000;
+    } else if (query.period === '24h' || query.period === '1d') {
+        period = 86400000;
+    } else {
+        return {};
+    }
+
+    if (!query.end_date) {
+        expiration = Math.floor((period - (Date.now() - now.getTime())) / 1000);
+    }
+
+    const response = {} as any;
+    const periodStart = new Date(now.getTime() - period);
+    const pStart = periodStart.toISOString();
+    const pEnd = now.toISOString();
+
+    const cacheKey = `${fastify.manager.chain}_act_stats_${pStart}_${pEnd}_${query.unique_actors}`;
+    const cachedValue = await fastify.redis.get(cacheKey);
+    if (cachedValue) {
+        const cachedResponse = JSON.parse(cachedValue);
+        cachedResponse.cached = true;
+        cachedResponse.cache_exp = expiration;
+        return cachedResponse;
+    }
+
+    const firstReq = await getLastSeq(fastify, pStart);
+    const lastReq = await getLastSeq(fastify, pEnd);
+    response.action_count = lastReq - firstReq;
+    response.tx_count = await getTxCount(fastify, pStart, pEnd);
+
+    if (query.unique_actors) {
+        response.unique_actors = await getUniqueActors(fastify, pStart, pEnd);
+    }
+
+    response.period = query.period;
+    response.from = pStart;
+    response.to = pEnd;
+
+    await fastify.redis.set(cacheKey, JSON.stringify(response), 'EX', expiration);
+    return response;
+}
+
+export function getActionUsageHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        reply.send(await timedQuery(getActionUsage, fastify, request, route));
+    }
+}

api/routes/v2-stats/get_action_usage/index.ts

@@ -0,0 +1,37 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, getRouteName} from "../../../helpers/functions";
+import {getActionUsageHandler} from "./get_action_usage";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get action and transaction stats for a given period',
+        summary: 'get action and transaction stats for a given period',
+        tags: ['stats'],
+        querystring: {
+            type: 'object',
+            required: ["period"],
+            properties: {
+                "period": {
+                    description: 'analysis period',
+                    type: 'string'
+                },
+                "end_date": {
+                    description: 'final date',
+                    type: 'string'
+                },
+                "unique_actors": {
+                    description: 'compute unique actors',
+                    type: 'boolean'
+                }
+            }
+        }
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getActionUsageHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-stats/get_missed_blocks/get_missed_blocks.ts

@@ -0,0 +1,64 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {timedQuery} from "../../../helpers/functions";
+import {ServerResponse} from "http";
+import {Search} from "@elastic/elasticsearch/api/requestParams";
+import {applyTimeFilter} from "../../v2-history/get_actions/functions";
+
+async function getMissedBlocks(fastify: FastifyInstance, request: FastifyRequest) {
+    const response = {
+        stats: {
+            by_producer: {}
+        },
+        events: []
+    };
+    const searchParams: Search<any> = {
+        track_total_hits: true,
+        index: fastify.manager.chain + "-logs-*",
+        size: 100,
+        body: {
+            query: {
+                bool: {
+                    must: [
+                        {term: {"type": "missed_blocks"}}
+                    ]
+                }
+            },
+            sort: {
+                "@timestamp": "desc"
+            }
+        }
+    };
+
+    let minBlocks = 0;
+    if (request.query.min_blocks) {
+        minBlocks = parseInt(request.query.min_blocks);
+        searchParams.body.query.bool.must.push({range: {"missed_blocks.size": {gte: minBlocks}}})
+    }
+
+    if (request.query.producer) {
+        searchParams.body.query.bool.must.push({term: {"missed_blocks.producer": request.query.producer}});
+    }
+
+    applyTimeFilter(request.query, searchParams.body.query);
+
+    const apiResponse = await fastify.elastic.search(searchParams);
+    apiResponse.body.hits.hits.forEach(v => {
+        const ev = v._source;
+        response.events.push({
+            "@timestamp": ev["@timestamp"],
+            ...ev.missed_blocks
+        });
+        if (!response.stats.by_producer[ev.missed_blocks.producer]) {
+            response.stats.by_producer[ev.missed_blocks.producer] = ev.missed_blocks.size;
+        } else {
+            response.stats.by_producer[ev.missed_blocks.producer] += ev.missed_blocks.size;
+        }
+    });
+    return response;
+}
+
+export function getMissedBlocksHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        reply.send(await timedQuery(getMissedBlocks, fastify, request, route));
+    }
+}

api/routes/v2-stats/get_missed_blocks/index.ts

@@ -0,0 +1,62 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getMissedBlocksHandler} from "./get_missed_blocks";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get missed blocks',
+        summary: 'get missed blocks',
+        tags: ['stats'],
+        querystring: {
+            type: 'object',
+            properties: {
+                "producer": {
+                    description: 'filter by producer',
+                    type: 'string'
+                },
+                "after": {
+                    description: 'filter after specified date (ISO8601)',
+                    type: 'string'
+                },
+                "before": {
+                    description: 'filter before specified date (ISO8601)',
+                    type: 'string'
+                },
+                "min_blocks": {
+                    description: 'min. blocks threshold',
+                    type: 'integer',
+                    minimum: 1
+                }
+            }
+        },
+        response: extendResponseSchema({
+            "stats": {
+                type: "object",
+                properties: {
+                    "by_producer": {}
+                }
+            },
+            "events": {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        "@timestamp": {type: "string"},
+                        "last_block": {type: "number"},
+                        "schedule_version": {type: "number"},
+                        "size": {type: "number"},
+                        "producer": {type: "string"},
+                    }
+                }
+            },
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getMissedBlocksHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-stats/get_resource_usage/get_resource_usage.ts

@@ -0,0 +1,75 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {ServerResponse} from "http";
+import {timedQuery} from "../../../helpers/functions";
+
+const percentiles = [1, 5, 25, 50, 75, 95, 99];
+
+async function getResourceUsage(fastify: FastifyInstance, request: FastifyRequest) {
+	const searchBody: any = {
+		query: {
+			bool: {
+				must: [
+					{term: {"act.account": request.query.code}},
+					{term: {"act.name": request.query.action}},
+					{term: {"action_ordinal": {"value": 1}}},
+					{
+						range: {
+							"@timestamp": {
+								gte: "now-2d/d",
+								lt: "now/d"
+							}
+						}
+					}
+				]
+			}
+		},
+		track_total_hits: true,
+		size: 0,
+		aggs: {
+			cpu_extended_stats: {extended_stats: {field: "cpu_usage_us"}},
+			net_extended_stats: {extended_stats: {field: "net_usage_words"}},
+			cpu_percentiles: {
+				percentiles: {
+					field: "cpu_usage_us",
+					percents: percentiles
+				}
+			},
+			net_percentiles: {
+				percentiles: {
+					field: "net_usage_words",
+					percents: percentiles
+				}
+			}
+		}
+	};
+
+	if (request.query['@transfer.to']) {
+		searchBody.query.bool.must.push({term: {"@transfer.to": request.query['@transfer.to']}});
+	}
+
+	const results = await fastify.elastic.search({
+		index: fastify.manager.chain + "-action-*",
+		body: searchBody
+	});
+	if (results.body) {
+		const data = results.body.aggregations;
+		return {
+			cpu: {
+				stats: data.cpu_extended_stats,
+				percentiles: data.cpu_percentiles.values
+			},
+			net: {
+				stats: data.net_extended_stats,
+				percentiles: data.net_percentiles.values
+			}
+		};
+	} else {
+		return {};
+	}
+}
+
+export function getResourceUsageHandler(fastify: FastifyInstance, route: string) {
+	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+		reply.send(await timedQuery(getResourceUsage, fastify, request, route));
+	}
+}

api/routes/v2-stats/get_resource_usage/index.ts

@@ -0,0 +1,33 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, getRouteName} from "../../../helpers/functions";
+import {getResourceUsageHandler} from "./get_resource_usage";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get resource usage stats for a specific action',
+        summary: 'get resource usage stats for a specific action',
+        tags: ['stats'],
+        querystring: {
+            type: 'object',
+            required: ["code", "action"],
+            properties: {
+                "code": {
+                    description: 'contract',
+                    type: 'string'
+                },
+                "action": {
+                    description: 'action name',
+                    type: 'string'
+                }
+            }
+        }
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getResourceUsageHandler,
+        schema
+    );
+    next();
+}

api/routes/v2/health/health.ts

@@ -2,7 +2,7 @@ import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {ServerResponse} from "http";
 import {connect} from "amqplib";
 import {timedQuery} from "../../../helpers/functions";
-import {getLastIndexedBlock} from "../../../../helpers/common_functions";
+import {getLastIndexedBlockWithTotalBlocks} from "../../../../helpers/common_functions";
 
 async function checkRabbit(fastify: FastifyInstance) {
     try {
@@ -20,7 +20,7 @@ async function checkNodeos(fastify: FastifyInstance) {
     try {
         const results = await rpc.get_info();
         if (results) {
-            const diff = (new Date().getTime()) - (new Date(results.head_block_time).getTime());
+            const diff = (new Date().getTime()) - (new Date(results.head_block_time + '+00:00').getTime());
             return createHealth('NodeosRPC', 'OK', {
                 head_block_num: results.head_block_num,
                 head_block_time: results.head_block_time,
@@ -39,9 +39,10 @@ async function checkNodeos(fastify: FastifyInstance) {
 async function checkElastic(fastify: FastifyInstance) {
     try {
         let esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
-        let lastIndexedBlock = await getLastIndexedBlock(fastify.elastic, fastify.manager.chain);
+        let indexedBlocks = await getLastIndexedBlockWithTotalBlocks(fastify.elastic, fastify.manager.chain);
         const data = {
-            last_indexed_block: lastIndexedBlock,
+            last_indexed_block: indexedBlocks[0],
+            total_indexed_blocks: indexedBlocks[1],
             active_shards: esStatus.body[0]['active_shards_percent']
         };
         let stat = 'OK';

api/server.ts

@@ -9,8 +9,11 @@ import {registerPlugins} from "./plugins";
 import {AddressInfo} from "net";
 import {registerRoutes} from "./routes";
 import {generateOpenApiConfig} from "./config/open_api";
-import {createWriteStream} from "fs";
+import {createWriteStream, existsSync, mkdirSync, writeFileSync} from "fs";
 import {SocketManager} from "./socketManager";
+import got from "got";
+import {join} from "path";
+import * as io from 'socket.io-client';
 
 class HyperionApiServer {
 
@@ -20,6 +23,8 @@ class HyperionApiServer {
     private readonly chain: string;
     socketManager: SocketManager;
 
+    private hub: SocketIOClient.Socket;
+
     constructor() {
         const cm = new ConfigurationModule();
         this.conf = cm.config;
@@ -28,9 +33,17 @@ class HyperionApiServer {
         this.manager = new ConnectionManager(cm);
         this.manager.calculateServerHash();
         this.manager.getHyperionVersion();
+
+        if (!existsSync('./logs/' + this.chain)) {
+            mkdirSync('./logs/' + this.chain, {recursive: true});
+        }
+
         const logStream = createWriteStream('./logs/' + this.chain + '/api.access.log');
         this.fastify = Fastify({
-            ignoreTrailingSlash: false, trustProxy: true, logger: this.conf.api.access_log ? {
+            ignoreTrailingSlash: false,
+            trustProxy: true,
+            pluginTimeout: 5000,
+            logger: this.conf.api.access_log ? {
                 stream: logStream,
                 redact: ['req.headers.authorization'],
                 level: 'info',
@@ -51,10 +64,24 @@ class HyperionApiServer {
             } : false
         });
         this.fastify.decorate('manager', this.manager);
+
+        if (this.conf.api.chain_api && this.conf.api.chain_api !== "") {
+            this.fastify.decorate('chain_api', this.conf.api.chain_api);
+        } else {
+            this.fastify.decorate('chain_api', this.manager.conn.chains[this.chain].http);
+        }
+
+        if (this.conf.api.push_api && this.conf.api.push_api !== "") {
+            this.fastify.decorate('push_api', this.conf.api.push_api);
+        }
+
+        console.log(`Chain API URL: ${this.fastify.chain_api}`);
+        console.log(`Push API URL: ${this.fastify.push_api}`);
+
         const ioRedisClient = new Redis(this.manager.conn.redis);
         const api_rate_limit = {
             max: 1000,
-            whitelist: [],
+            whitelist: ['127.0.0.1'],
             timeWindow: '1 minute',
             redis: ioRedisClient
         };
@@ -73,24 +100,27 @@ class HyperionApiServer {
             fastify_eosjs: this.manager,
         });
 
-        registerRoutes(this.fastify);
-
         this.addGenericTypeParsing();
-        this.fastify.ready().then(async () => {
-            await this.fastify.oas();
-            console.log(this.chain + ' api ready!');
-        }, (err) => {
-            console.log('an error happened', err)
-        });
     }
 
     activateStreaming() {
         console.log('Importing stream module');
         import('./socketManager').then((mod) => {
             const connOpts = this.manager.conn.chains[this.chain];
+
+            let _port = 57200;
+            if (connOpts.WS_ROUTER_PORT) {
+                _port = connOpts.WS_ROUTER_PORT;
+            }
+
+            let _host = "127.0.0.1";
+            if (connOpts.WS_ROUTER_HOST) {
+                _host = connOpts.WS_ROUTER_HOST;
+            }
+
             this.socketManager = new mod.SocketManager(
                 this.fastify,
-                `http://${connOpts['WS_ROUTER_HOST']}:${connOpts['WS_ROUTER_PORT']}`,
+                `http://${_host}:${_port}`,
                 this.manager.conn.redis
             );
             this.socketManager.startRelay();
@@ -114,17 +144,87 @@ class HyperionApiServer {
     }
 
     async init() {
+
+        await this.fetchChainLogo();
+
+        registerRoutes(this.fastify);
+
+        this.fastify.ready().then(async () => {
+            await this.fastify.oas();
+            console.log(this.chain + ' api ready!');
+        }, (err) => {
+            console.log('an error happened', err)
+        });
+
         try {
             await this.fastify.listen({
                 host: this.conf.api.server_addr,
                 port: this.conf.api.server_port
             });
             console.log(`server listening on ${(this.fastify.server.address() as AddressInfo).port}`);
+            this.startHyperionHub();
         } catch (err) {
-            this.fastify.log.error(err);
+            console.log(err);
             process.exit(1)
         }
     }
+
+    async fetchChainLogo() {
+        try {
+            if (this.conf.api.chain_logo_url && this.conf.api.enable_explorer) {
+                console.log(`Downloading chain logo from ${this.conf.api.chain_logo_url}...`);
+                const chainLogo = await got(this.conf.api.chain_logo_url);
+                const path = join(__dirname, '..', 'hyperion-explorer', 'dist', 'assets', this.chain + '_logo.png');
+                writeFileSync(path, chainLogo.rawBody);
+                this.conf.api.chain_logo_url = 'https://' + this.conf.api.server_name + '/v2/explore/assets/' + this.chain + '_logo.png';
+            }
+        } catch (e) {
+            console.log(e);
+        }
+    }
+
+    startHyperionHub() {
+        if (this.conf.hub) {
+            const url = this.conf.hub.inform_url;
+            hLog(`Connecting API to Hyperion Hub`);
+            this.hub = io(url, {
+                query: {
+                    key: this.conf.hub.publisher_key,
+                    client_mode: false
+                }
+            });
+            this.hub.on('connect', () => {
+                hLog(`Hyperion Hub connected!`);
+                this.emitHubApiUpdate();
+            });
+            // this.hub.on('reconnect', () => {
+            //     hLog(`Reconnecting...`);
+            //     this.emitHubApiUpdate();
+            // });
+        }
+    }
+
+    private emitHubApiUpdate() {
+        this.hub.emit('hyp_info', {
+            type: 'api',
+            production: this.conf.hub.production,
+            location: this.conf.hub.location,
+            chainId: this.manager.conn.chains[this.chain].chain_id,
+            providerName: this.conf.api.provider_name,
+            explorerEnabled: this.conf.api.enable_explorer,
+            providerUrl: this.conf.api.provider_url,
+            providerLogo: this.conf.api.provider_logo,
+            chainLogo: this.conf.api.chain_logo_url,
+            chainCodename: this.chain,
+            chainName: this.conf.api.chain_name,
+            endpoint: this.conf.api.server_name,
+            features: this.conf.features,
+            filters: {
+                blacklists: this.conf.blacklists,
+                whitelists: this.conf.whitelists
+            }
+        });
+    }
 }
 
 const server = new HyperionApiServer();

api/socketManager.ts

@@ -1,7 +1,6 @@
 import {checkFilter, hLog} from '../helpers/common_functions';
 import * as sockets from 'socket.io';
 import * as IOClient from 'socket.io-client';
-import * as socketIOredis from 'socket.io-redis';
 import {FastifyInstance} from "fastify";
 
 export interface StreamDeltasRequest {
@@ -132,9 +131,12 @@ async function streamPastDeltas(fastify: FastifyInstance, socket, data) {
             console.log(`${counter} past deltas streamed to ${socket.id}`);
             break;
         }
+
         const next_response = await fastify.elastic.scroll({
-            scroll_id: body['_scroll_id'],
-            scroll: '30s',
+            body: {
+                scroll_id: body['_scroll_id'],
+                scroll: '30s'
+            }
         });
         responseQueue.push(next_response);
     }
@@ -191,14 +193,11 @@ async function streamPastActions(fastify: FastifyInstance, socket, data) {
         size: 20,
         body: search_body,
     });
-
     responseQueue.push(init_response);
-
     while (responseQueue.length) {
         const {body} = responseQueue.shift();
         const enqueuedMessages = [];
         counter += body['hits']['hits'].length;
-
         for (const doc of body['hits']['hits']) {
             let allow = false;
             if (onDemandFilters.length > 0) {
@@ -212,29 +211,25 @@ async function streamPastActions(fastify: FastifyInstance, socket, data) {
                 enqueuedMessages.push(doc._source);
             }
         }
-
         if (socket.connected) {
-            socket.emit('message', {
-                type: 'action_trace',
-                mode: 'history',
-                messages: enqueuedMessages,
-            });
+            socket.emit('message', {type: 'action_trace', mode: 'history', messages: enqueuedMessages});
         } else {
             console.log('LOST CLIENT');
             break;
         }
-
         if (body['hits'].total.value === counter) {
             console.log(`${counter} past actions streamed to ${socket.id}`);
             break;
         }
-
-        const next_response = await fastify.elastic.scroll({
-            scroll_id: body['_scroll_id'],
-            scroll: '30s',
-        });
-
-        responseQueue.push(next_response);
+        if (init_response.body.hits.total.value < 1000) {
+            const next_response = await fastify.elastic.scroll({
+                body: {scroll_id: body['_scroll_id'], scroll: '30s'}
+            });
+            responseQueue.push(next_response);
+        } else {
+            console.log('Request too large!');
+            socket.emit('message', {type: 'action_trace', mode: 'history', messages: []});
+        }
     }
 }
 
@@ -255,8 +250,6 @@ export class SocketManager {
             transports: ['websocket', 'polling'],
         });
 
-        this.io.adapter(socketIOredis(redisOpts));
-
         this.io.on('connection', (socket) => {
 
             if (socket.handshake.headers['x-forwarded-for']) {
@@ -276,24 +269,28 @@ export class SocketManager {
             }
 
             socket.on('delta_stream_request', async (data: StreamDeltasRequest, callback) => {
-                try {
-                    if (data.start_from) {
-                        await streamPastDeltas(this.server, socket, data);
+                if (typeof callback === 'function' && data) {
+                    try {
+                        if (data.start_from) {
+                            await streamPastDeltas(this.server, socket, data);
+                        }
+                        this.emitToRelay(data, 'delta_request', socket, callback);
+                    } catch (e) {
+                        console.log(e);
                     }
-                    this.emitToRelay(data, 'delta_request', socket, callback);
-                } catch (e) {
-                    console.log(e);
                 }
             });
 
             socket.on('action_stream_request', async (data: StreamActionsRequest, callback) => {
-                try {
-                    if (data.start_from) {
-                        await streamPastActions(this.server, socket, data);
+                if (typeof callback === 'function' && data) {
+                    try {
+                        if (data.start_from) {
+                            await streamPastActions(this.server, socket, data);
+                        }
+                        this.emitToRelay(data, 'action_request', socket, callback);
+                    } catch (e) {
+                        console.log(e);
                     }
-                    this.emitToRelay(data, 'action_request', socket, callback);
-                } catch (e) {
-                    console.log(e);
                 }
             });
 
@@ -311,9 +308,9 @@ export class SocketManager {
 
     startRelay() {
         console.log(`starting relay - ${this.url}`);
-        this.relay = IOClient(this.url, {
-            path: '/router',
-        });
+
+        this.relay = IOClient(this.url, {path: '/router'});
+
         this.relay.on('connect', () => {
             console.log('Relay Connected!');
             if (this.relay_down) {
@@ -322,21 +319,36 @@ export class SocketManager {
                 this.io.emit('status', 'relay_restored');
             }
         });
+
         this.relay.on('disconnect', () => {
             console.log('Relay disconnected!');
             this.io.emit('status', 'relay_down');
             this.relay_down = true;
             this.relay_restored = false;
         });
+
         this.relay.on('delta', (traceData) => {
             this.emitToClient(traceData, 'delta_trace');
         });
+
         this.relay.on('trace', (traceData) => {
             this.emitToClient(traceData, 'action_trace');
         });
-        setTimeout(() => {
-            console.log(`Relay status: ${this.relay.connected}`);
-        }, 2000);
+
+        // Relay LIB info to clients;
+        this.relay.on('lib_update', (data) => {
+            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+                this.io.emit('lib_update', data);
+            }
+        });
+
+        // Relay LIB info to clients;
+        this.relay.on('fork_event', (data) => {
+            console.log(data);
+            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+                this.io.emit('fork_event', data);
+            }
+        });
     }
 
     emitToClient(traceData, type) {

auto_setup.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+git clone https://github.com/eosrio/hyperion-history-api.git  || exit
+cd hyperion-history-api || exit
+bash ./install_env.sh

chains/example.config.json

@@ -6,6 +6,8 @@
     "server_name": "127.0.0.1:7000",
     "provider_name": "Example Provider",
     "provider_url": "https://example.com",
+    "chain_api": "",
+    "push_api": "",
     "chain_logo_url": "",
     "enable_caching": true,
     "cache_life": 1,
@@ -13,10 +15,16 @@
       "get_actions": 1000,
       "get_voters": 100,
       "get_links": 1000,
-      "get_deltas": 1000
+      "get_deltas": 1000,
+      "get_trx_actions": 200
     },
     "access_log": false,
-    "enable_explorer": false
+    "enable_explorer": false,
+    "chain_api_error_log": false,
+    "custom_core_token": "",
+    "enable_export_action": false,
+    "disable_tx_cache": false,
+    "tx_cache_expiration_sec": 3600
   },
   "settings": {
     "preview": false,
@@ -26,10 +34,17 @@
     "auto_stop": 300,
     "index_version": "v1",
     "debug": false,
-    "rate_monitoring": true,
     "bp_logs": false,
     "bp_monitoring": false,
-    "ipc_debug_rate": 2000
+    "ipc_debug_rate": 60000,
+    "allow_custom_abi": false,
+    "rate_monitoring": true,
+    "max_ws_payload_kb": 256,
+    "ds_profiling": false,
+    "auto_mode_switch": false,
+    "hot_warm_policy": false,
+    "custom_policy": "",
+    "bypass_index_map": false
   },
   "blacklists": {
     "actions": [],
@@ -37,11 +52,11 @@
   },
   "whitelists": {
     "actions": [],
-    "deltas": []
+    "deltas": [],
+    "max_depth": 10,
+    "root_only": false
   },
   "scaling": {
-    "batch_size": 10000,
-    "queue_limit": 50000,
     "readers": 1,
     "ds_queues": 1,
     "ds_threads": 1,
@@ -49,13 +64,19 @@
     "indexing_queues": 1,
     "ad_idx_queues": 1,
     "max_autoscale": 4,
-    "auto_scale_trigger": 20000
+    "batch_size": 5000,
+    "resume_trigger": 5000,
+    "auto_scale_trigger": 20000,
+    "block_queue_limit": 10000,
+    "max_queue_limit": 100000,
+    "routing_mode": "heatmap",
+    "polling_interval": 10000
   },
   "indexer": {
     "start_on": 0,
     "stop_on": 0,
     "rewrite": false,
-    "purge_queues": true,
+    "purge_queues": false,
     "live_reader": false,
     "live_only_mode": false,
     "abi_scan_mode": true,
@@ -63,9 +84,7 @@
     "fetch_traces": true,
     "disable_reading": false,
     "disable_indexing": false,
-    "process_deltas": true,
-    "repair_mode": false,
-    "max_inline": 20
+    "process_deltas": true
   },
   "features": {
     "streaming": {
@@ -76,13 +95,15 @@
     "tables": {
       "proposals": true,
       "accounts": true,
-      "voters": true,
-      "userres": false,
-      "delband": false
+      "voters": true
     },
     "index_deltas": true,
-    "index_transfer_memo": true,
-    "index_all_deltas": true
+    "index_transfer_memo": false,
+    "index_all_deltas": true,
+    "deferred_trx": false,
+    "failed_trx": false,
+    "resource_limits": false,
+    "resource_usage": false
   },
   "prefetch": {
     "read": 50,

connections/amqp.ts

@@ -1,75 +1,91 @@
-import {hLog} from "../helpers/common_functions";
-
-const got = require('got');
+import {debugLog, hLog} from "../helpers/common_functions";
+import got, {HTTPError} from "got";
 import {connect, Connection} from 'amqplib';
 
-const {debugLog} = require("../helpers/functions");
-
 export async function createConnection(config): Promise<Connection> {
-    try {
-        const amqp_url = getAmpqUrl(config);
-        const conn: Connection = await connect(amqp_url);
-        debugLog("[AMQP] connection established");
-        return conn;
-    } catch (e) {
-        console.log("[AMQP] failed to connect!");
-        console.log(e.message);
-        await new Promise(resolve => setTimeout(resolve, 5000));
-        return await createConnection(config);
-    }
+	try {
+		const amqp_url = getAmpqUrl(config);
+		const conn: Connection = await connect(amqp_url);
+		debugLog("[AMQP] connection established");
+		return conn;
+	} catch (e) {
+		console.log("[AMQP] failed to connect!");
+		console.log(e.message);
+		await new Promise(resolve => setTimeout(resolve, 5000));
+		return await createConnection(config);
+	}
 }
 
 export function getAmpqUrl(config): string {
-    return `amqp://${config.user}:${config.pass}@${config.host}/%2F${config.vhost}`;
+	let frameMaxValue = '0x10000';
+	if (config.frameMax) {
+		frameMaxValue = config.frameMax;
+	}
+	const u = encodeURIComponent(config.user);
+	const p = encodeURIComponent(config.pass);
+	const v = encodeURIComponent(config.vhost)
+	return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
 }
 
 
 async function createChannels(connection) {
-    try {
-        const channel = await connection.createChannel();
-        const confirmChannel = await connection.createConfirmChannel();
-        return [channel, confirmChannel];
-    } catch (e) {
-        console.log("[AMQP] failed to create channels");
-        console.error(e);
-        return null;
-    }
+	try {
+		const channel = await connection.createChannel();
+		const confirmChannel = await connection.createConfirmChannel();
+		return [channel, confirmChannel];
+	} catch (e) {
+		console.log("[AMQP] failed to create channels");
+		console.error(e);
+		return null;
+	}
 }
 
 export async function amqpConnect(onReconnect, config, onClose) {
-    let connection = await createConnection(config);
-    if (connection) {
-        const channels = await createChannels(connection);
-        if (channels) {
-            connection.on('error', (err) => {
-                hLog(err.message);
-            });
-            connection.on('close', () => {
-                hLog('Connection closed!');
-                onClose();
-                setTimeout(async () => {
-                    hLog('Retrying in 5 seconds...');
-                    const _channels = await amqpConnect(onReconnect, config, onClose);
-                    onReconnect(_channels);
-                    return _channels;
-                }, 5000);
-            });
-            return channels;
-        } else {
-            return null;
-        }
-    } else {
-        return null;
-    }
+	let connection = await createConnection(config);
+	if (connection) {
+		const channels = await createChannels(connection);
+		if (channels) {
+			connection.on('error', (err) => {
+				hLog(err.message);
+			});
+			connection.on('close', () => {
+				hLog('Connection closed!');
+				onClose();
+				setTimeout(async () => {
+					hLog('Retrying in 5 seconds...');
+					const _channels = await amqpConnect(onReconnect, config, onClose);
+					onReconnect(_channels);
+					return _channels;
+				}, 5000);
+			});
+			return channels;
+		} else {
+			return null;
+		}
+	} else {
+		return null;
+	}
 }
 
 export async function checkQueueSize(q_name, config) {
-    try {
-        const apiUrl = `http://${config.user}:${config.pass}@${config.api}/api/queues/%2F${config.vhost}/${q_name}`;
-        const result = JSON.parse((await got(apiUrl)).body);
-        return result.messages;
-    } catch (e) {
-        console.log('Checking queue size failed, HTTP API is not ready!');
-        return 10000000;
-    }
+	try {
+		const v = encodeURIComponent(config.vhost);
+		const apiUrl = `http://${config.api}/api/queues/${v}/${encodeURIComponent(q_name)}`;
+		const opts = {
+			username: config.user,
+			password: config.pass
+		};
+		const data = await got.get(apiUrl, opts).json() as any;
+		return data.messages;
+	} catch (e) {
+		hLog('[WARNING] Checking queue size failed, HTTP API is not ready!');
+		if (e instanceof HTTPError) {
+			if(e.response.body) {
+				hLog(e.response.body);
+			}
+		} else {
+			hLog(JSON.stringify(e, null, 2));
+		}
+		return 0;
+	}
 }

connections/manager.class.ts

@@ -6,7 +6,7 @@ import {HyperionConnections} from "../interfaces/hyperionConnections";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
 import {amqpConnect, checkQueueSize, getAmpqUrl} from "./amqp";
 import {StateHistorySocket} from "./state-history";
-import fetch from 'cross-fetch'
+import fetch from 'cross-fetch';
 import {exec} from "child_process";
 import {hLog} from "../helpers/common_functions";
 
@@ -32,16 +32,21 @@ export class ConnectionManager {
     }
 
     get nodeosJsonRPC() {
+        // @ts-ignore
         return new JsonRpc(this.conn.chains[this.chain].http, {fetch});
     }
 
     async purgeQueues() {
         hLog(`Purging all ${this.chain} queues!`);
-        const apiUrl = `http://${this.conn.amqp.user}:${this.conn.amqp.pass}@${this.conn.amqp.api}`;
+        const apiUrl = `http://${this.conn.amqp.api}`;
         const getAllQueuesFromVHost = apiUrl + `/api/queues/%2F${this.conn.amqp.vhost}`;
+        const opts = {
+            username: this.conn.amqp.user,
+            password: this.conn.amqp.pass
+        };
         let result;
         try {
-            const data = await got(getAllQueuesFromVHost);
+            const data = await got(getAllQueuesFromVHost, opts);
             if (data) {
                 result = JSON.parse(data.body);
             }
@@ -56,7 +61,7 @@ export class ConnectionManager {
                     const msg_count = parseInt(queue.messages);
                     if (msg_count > 0) {
                         try {
-                            await got.delete(apiUrl + `/api/queues/%2F${this.conn.amqp.vhost}/${queue.name}/contents`);
+                            await got.delete(apiUrl + `/api/queues/%2F${this.conn.amqp.vhost}/${queue.name}/contents`, opts);
                             hLog(`${queue.messages} messages deleted on queue ${queue.name}`);
                         } catch (e) {
                             console.log(e.message);
@@ -72,12 +77,20 @@ export class ConnectionManager {
     prepareESClient() {
         let es_url;
         const _es = this.conn.elasticsearch;
-        if (_es.user !== '') {
-            es_url = `http://${_es.user}:${_es.pass}@${_es.host}`;
-        } else {
-            es_url = `http://${_es.host}`
+        if (!_es.protocol) {
+            _es.protocol = 'http';
         }
-        this.esIngestClient = new Client({node: es_url});
+        if (_es.user !== '') {
+            es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${_es.host}`;
+        } else {
+            es_url = `${_es.protocol}://${_es.host}`
+        }
+        this.esIngestClient = new Client({
+            node: es_url,
+            ssl: {
+                rejectUnauthorized: false
+            }
+        });
     }
 
     get elasticsearchClient() {
@@ -86,16 +99,25 @@ export class ConnectionManager {
 
     prepareIngestClients() {
         const _es = this.conn.elasticsearch;
+        if (!_es.protocol) {
+            _es.protocol = 'http';
+        }
         if (_es.ingest_nodes) {
             if (_es.ingest_nodes.length > 0) {
                 for (const node of _es.ingest_nodes) {
                     let es_url;
                     if (_es.user !== '') {
-                        es_url = `http://${_es.user}:${_es.pass}@${node}`;
+                        es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${node}`;
                     } else {
-                        es_url = `http://${node}`
+                        es_url = `${_es.protocol}://${node}`
                     }
-                    this.esIngestClients.push(new Client({node: es_url, pingTimeout: 100}));
+                    this.esIngestClients.push(new Client({
+                        node: es_url,
+                        pingTimeout: 100,
+                        ssl: {
+                            rejectUnauthorized: false
+                        }
+                    }));
                 }
             }
         }
@@ -118,7 +140,7 @@ export class ConnectionManager {
     }
 
     get shipClient(): StateHistorySocket {
-        return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship']);
+        return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_kb);
     }
 
     get ampqUrl() {

connections/state-history.ts

@@ -1,19 +1,26 @@
-import {hLog} from "../helpers/common_functions";
+import {debugLog, hLog} from "../helpers/common_functions";
 
 const WebSocket = require('ws');
 
 export class StateHistorySocket {
     private ws;
     private readonly shipUrl;
+    private readonly max_payload_kb;
 
-    constructor(ship_url) {
+    constructor(ship_url, max_payload_kb) {
         this.shipUrl = ship_url;
+        if (max_payload_kb) {
+            this.max_payload_kb = max_payload_kb;
+        } else {
+            this.max_payload_kb = 256;
+        }
     }
 
     connect(onMessage, onDisconnect, onError, onConnected) {
-        hLog(`Connecting to ${this.shipUrl}...`);
+        debugLog(`Connecting to ${this.shipUrl}...`);
         this.ws = new WebSocket(this.shipUrl, null, {
-            perMessageDeflate: false
+            perMessageDeflate: false,
+            maxPayload: this.max_payload_kb * 1024 * 1024,
         });
         this.ws.on('open', () => {
             hLog('Websocket connected!');

custom-abi/fio/fio.address-1-100000000.json

@@ -0,0 +1,385 @@
+{
+  "version": "eosio::abi/1.0",
+  "types": [],
+  "structs": [
+    {
+      "name": "fioname",
+      "base": "",
+      "fields": [
+        {
+          "name": "id",
+          "type": "uint64"
+        },
+        {
+          "name": "name",
+          "type": "string"
+        },
+        {
+          "name": "namehash",
+          "type": "uint128"
+        },
+        {
+          "name": "domain",
+          "type": "string"
+        },
+        {
+          "name": "domainhash",
+          "type": "uint128"
+        },
+        {
+          "name": "expiration",
+          "type": "uint64"
+        },
+        {
+          "name": "owner_account",
+          "type": "name"
+        },
+        {
+          "name": "addresses",
+          "type": "tokenpubaddr[]"
+        },
+        {
+          "name": "bundleeligiblecountdown",
+          "type": "uint64"
+        }
+      ]
+    },
+    {
+      "name": "domain",
+      "base": "",
+      "fields": [
+        {
+          "name": "id",
+          "type": "uint64"
+        },
+        {
+          "name": "name",
+          "type": "string"
+        },
+        {
+          "name": "domainhash",
+          "type": "uint128"
+        },
+        {
+          "name": "account",
+          "type": "name"
+        },
+        {
+          "name": "is_public",
+          "type": "uint8"
+        },
+        {
+          "name": "expiration",
+          "type": "uint32"
+        }
+      ]
+    },
+    {
+      "name": "eosio_name",
+      "base": "",
+      "fields": [
+        {
+          "name": "account",
+          "type": "name"
+        },
+        {
+          "name": "clientkey",
+          "type": "string"
+        },
+        {
+          "name": "keyhash",
+          "type": "uint128"
+        }
+      ]
+    },
+    {
+      "name": "regaddress",
+      "base": "",
+      "fields": [
+        {
+          "name": "fio_address",
+          "type": "string"
+        },
+        {
+          "name": "owner_fio_public_key",
+          "type": "string"
+        },
+        {
+          "name": "max_fee",
+          "type": "int64"
+        },
+        {
+          "name": "actor",
+          "type": "name"
+        },
+        {
+          "name": "tpid",
+          "type": "string"
+        }
+      ]
+    },
+    {
+      "name": "tokenpubaddr",
+      "base": "",
+      "fields": [
+        {
+          "name": "token_code",
+          "type": "string"
+        },
+        {
+          "name": "chain_code",
+          "type": "string"
+        },
+        {
+          "name": "public_address",
+          "type": "string"
+        }
+      ]
+    },
+    {
+      "name": "addaddress",
+      "base": "",
+      "fields": [
+        {
+          "name": "fio_address",
+          "type": "string"
+        },
+        {
+          "name": "public_addresses",
+          "type": "tokenpubaddr[]"
+        },
+        {
+          "name": "max_fee",
+          "type": "int64"
+        },
+        {
+          "name": "actor",
+          "type": "name"
+        },
+        {
+          "name": "tpid",
+          "type": "string"
+        }
+      ]
+    },
+    {
+      "name": "regdomain",
+      "base": "",
+      "fields": [
+        {
+          "name": "fio_domain",
+          "type": "string"
+        },
+        {
+          "name": "owner_fio_public_key",
+          "type": "string"
+        },
+        {
+          "name": "max_fee",
+          "type": "int64"
+        },
+        {
+          "name": "actor",
+          "type": "name"
+        },
+        {
+          "name": "tpid",
+          "type": "string"
+        }
+      ]
+    },
+    {
+      "name": "renewdomain",
+      "base": "",
+      "fields": [
+        {
+          "name": "fio_domain",
+          "type": "string"
+        },
+        {
+          "name": "max_fee",
+          "type": "int64"
+        },
+        {
+          "name": "tpid",
+          "type": "string"
+        },
+        {
+          "name": "actor",
+          "type": "name"
+        }
+      ]
+    },
+    {
+      "name": "renewaddress",
+      "base": "",
+      "fields": [
+        {
+          "name": "fio_address",
+          "type": "string"
+        },
+        {
+          "name": "max_fee",
+          "type": "int64"
+        },
+        {
+          "name": "tpid",
+          "type": "string"
+        },
+        {
+          "name": "actor",
+          "type": "name"
+        }
+      ]
+    },
+    {
+      "name": "setdomainpub",
+      "base": "",
+      "fields": [
+        {
+          "name": "fio_domain",
+          "type": "string"
+        },
+        {
+          "name": "is_public",
+          "type": "int8"
+        },
+        {
+          "name": "max_fee",
+          "type": "int64"
+        },
+        {
+          "name": "actor",
+          "type": "name"
+        },
+        {
+          "name": "tpid",
+          "type": "string"
+        }
+      ]
+    },
+    {
+      "name": "burnexpired",
+      "base": "",
+      "fields": []
+    },
+    {
+      "name": "decrcounter",
+      "base": "",
+      "fields": [
+        {
+          "name": "fio_address",
+          "type": "string"
+        },
+        {
+          "name": "step",
+          "type": "int32"
+        }
+      ]
+    },
+    {
+      "name": "bind2eosio",
+      "base": "",
+      "fields": [
+        {
+          "name": "account",
+          "type": "name"
+        },
+        {
+          "name": "client_key",
+          "type": "string"
+        },
+        {
+          "name": "existing",
+          "type": "bool"
+        }
+      ]
+    }
+  ],
+  "actions": [
+    {
+      "name": "decrcounter",
+      "type": "decrcounter",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "regaddress",
+      "type": "regaddress",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "addaddress",
+      "type": "addaddress",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "regdomain",
+      "type": "regdomain",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "renewdomain",
+      "type": "renewdomain",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "renewaddress",
+      "type": "renewaddress",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "burnexpired",
+      "type": "burnexpired",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "setdomainpub",
+      "type": "setdomainpub",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "bind2eosio",
+      "type": "bind2eosio",
+      "ricardian_contract": ""
+    }
+  ],
+  "tables": [
+    {
+      "name": "fionames",
+      "index_type": "i64",
+      "key_names": [
+        "id"
+      ],
+      "key_types": [
+        "string"
+      ],
+      "type": "fioname"
+    },
+    {
+      "name": "domains",
+      "index_type": "i64",
+      "key_names": [
+        "id"
+      ],
+      "key_types": [
+        "string"
+      ],
+      "type": "domain"
+    },
+    {
+      "name": "accountmap",
+      "index_type": "i64",
+      "key_names": [
+        "account"
+      ],
+      "key_types": [
+        "uint64"
+      ],
+      "type": "eosio_name"
+    }
+  ],
+  "ricardian_clauses": [],
+  "error_messages": [],
+  "abi_extensions": [],
+  "variants": []
+}

custom-abi/fio/fio.treasury-1-100000000.json

@@ -0,0 +1,232 @@
+{
+  "version": "eosio::abi/1.1",
+  "types": [],
+  "structs": [
+    {
+      "name": "tpidclaim",
+      "base": "",
+      "fields": [
+        {
+          "name": "actor",
+          "type": "name"
+        }
+      ]
+    },
+    {
+      "name": "startclock",
+      "base": "",
+      "fields": []
+    },
+    {
+      "name": "bpclaim",
+      "base": "",
+      "fields": [
+        {
+          "name": "fio_address",
+          "type": "string"
+        },
+        {
+          "name": "actor",
+          "type": "name"
+        }
+      ]
+    },
+    {
+      "name": "treasurystate",
+      "base": "",
+      "fields": [
+        {
+          "name": "lasttpidpayout",
+          "type": "uint64"
+        },
+        {
+          "name": "payschedtimer",
+          "type": "uint64"
+        },
+        {
+          "name": "rewardspaid",
+          "type": "uint64"
+        },
+        {
+          "name": "reservetokensminted",
+          "type": "uint64"
+        }
+      ]
+    },
+    {
+      "name": "bpreward",
+      "base": "",
+      "fields": [
+        {
+          "name": "rewards",
+          "type": "uint64"
+        }
+      ]
+    },
+    {
+      "name": "fdtnreward",
+      "base": "",
+      "fields": [
+        {
+          "name": "rewards",
+          "type": "uint64"
+        }
+      ]
+    },
+    {
+      "name": "fdtnrwdupdat",
+      "base": "",
+      "fields": [
+        {
+          "name": "amount",
+          "type": "uint64"
+        }
+      ]
+    },
+    {
+      "name": "bprewdupdate",
+      "base": "",
+      "fields": [
+        {
+          "name": "amount",
+          "type": "uint64"
+        }
+      ]
+    },
+    {
+      "name": "bppoolupdate",
+      "base": "",
+      "fields": [
+        {
+          "name": "amount",
+          "type": "uint64"
+        }
+      ]
+    },
+    {
+      "name": "bucketpool",
+      "base": "",
+      "fields": [
+        {
+          "name": "rewards",
+          "type": "uint64"
+        }
+      ]
+    },
+    {
+      "name": "bppaysched",
+      "base": "",
+      "fields": [
+        {
+          "name": "owner",
+          "type": "name"
+        },
+        {
+          "name": "abpayshare",
+          "type": "uint64"
+        },
+        {
+          "name": "sbpayshare",
+          "type": "uint64"
+        },
+        {
+          "name": "votes",
+          "type": "float64"
+        }
+      ]
+    }
+  ],
+  "actions": [
+    {
+      "name": "tpidclaim",
+      "type": "tpidclaim",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "bpclaim",
+      "type": "bpclaim",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "startclock",
+      "type": "startclock",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "fdtnrwdupdat",
+      "type": "fdtnrwdupdat",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "bprewdupdate",
+      "type": "bprewdupdate",
+      "ricardian_contract": ""
+    },
+    {
+      "name": "bppoolupdate",
+      "type": "bppoolupdate",
+      "ricardian_contract": ""
+    }
+  ],
+  "tables": [
+    {
+      "name": "clockstate",
+      "index_type": "i64",
+      "key_names": [
+        "lastrun"
+      ],
+      "key_types": [
+        "uint64"
+      ],
+      "type": "treasurystate"
+    },
+    {
+      "name": "bprewards",
+      "index_type": "i64",
+      "key_names": [
+        "rewards"
+      ],
+      "key_types": [
+        "uint64"
+      ],
+      "type": "bpreward"
+    },
+    {
+      "name": "fdtnrewards",
+      "index_type": "i64",
+      "key_names": [
+        "rewards"
+      ],
+      "key_types": [
+        "uint64"
+      ],
+      "type": "fdtnreward"
+    },
+    {
+      "name": "bpbucketpool",
+      "index_type": "i64",
+      "key_names": [
+        "rewards"
+      ],
+      "key_types": [
+        "uint64"
+      ],
+      "type": "bucketpool"
+    },
+    {
+      "name": "voteshares",
+      "index_type": "i64",
+      "key_names": [
+        "owner"
+      ],
+      "key_types": [
+        "uint64"
+      ],
+      "type": "bppaysched"
+    }
+  ],
+  "ricardian_clauses": [],
+  "error_messages": [],
+  "abi_extensions": [],
+  "variants": []
+}

definitions/ecosystem_settings.js

@@ -1,36 +1,38 @@
 function addIndexer(chainName) {
-    return {
-        script: "./launcher.js",
-        name: chainName + "-indexer",
-        namespace: chainName,
-        interpreter: 'node',
-        interpreter_args: ["--max-old-space-size=4096", "--trace-deprecation"],
-        autorestart: false,
-        kill_timeout: 3600,
-        time: true, // include timestamps in pm2 logs
-        env: {
-            CONFIG_JSON: 'chains/' + chainName + '.config.json',
-            TRACE_LOGS: 'false'
-        }
-    };
+  return {
+    script: './launcher.js',
+    name: chainName + '-indexer',
+    namespace: chainName,
+    interpreter: 'node',
+    interpreter_args: ['--max-old-space-size=4096', '--trace-deprecation'],
+    autorestart: false,
+    kill_timeout: 3600,
+    watch: false,
+    time: true,
+    env: {
+      CONFIG_JSON: 'chains/' + chainName + '.config.json',
+      TRACE_LOGS: 'false',
+    },
+  };
 }
 
 function addApiServer(chainName, threads) {
-    return {
-        script: "./api/server.js",
-        name: chainName + "-api",
-        namespace: chainName,
-        node_args: ["--trace-deprecation"],
-        exec_mode: 'cluster',
-        merge_logs: true,
-        instances: threads,
-        autorestart: true,
-        exp_backoff_restart_delay: 100,
-        watch: ["api"],
-        env: {
-            CONFIG_JSON: 'chains/' + chainName + '.config.json'
-        }
-    }
+  return {
+    script: './api/server.js',
+    name: chainName + '-api',
+    namespace: chainName,
+    node_args: ['--trace-deprecation'],
+    exec_mode: 'cluster',
+    merge_logs: true,
+    instances: threads,
+    autorestart: true,
+    exp_backoff_restart_delay: 100,
+    watch: false,
+    time: true,
+    env: {
+      CONFIG_JSON: 'chains/' + chainName + '.config.json',
+    },
+  };
 }
 
 module.exports = {addIndexer, addApiServer};

definitions/index-lifecycle-policies.ts

@@ -1,18 +1,97 @@
-export const ILPs = [
+import {IlmPutLifecycle} from "@elastic/elasticsearch/api/requestParams";
+
+export const ILPs: IlmPutLifecycle[] = [
     {
-        "policy": "50G30D",
-        "body": {
+        policy: "hyperion-rollover",
+        body: {
             "policy": {
                 "phases": {
                     "hot": {
                         "min_age": "0ms",
                         "actions": {
                             "rollover": {
-                                "max_age": "30d",
-                                "max_size": "50gb"
+                                "max_size": "200gb",
+                                "max_age": "60d",
+                                "max_docs": 100000000
                             },
                             "set_priority": {
-                                "priority": 100
+                                "priority": 50
+                            }
+                        }
+                    },
+                    "warm": {
+                        "min_age": "2d",
+                        "actions": {
+                            "allocate": {
+                                "exclude": {
+                                    "data": "hot"
+                                }
+                            },
+                            "set_priority": {
+                                "priority": 25
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        policy: "50G30D",
+        body: {
+            policy: {
+                phases: {
+                    hot: {
+                        min_age: "0ms",
+                        actions: {
+                            rollover: {
+                                max_age: "30d",
+                                max_size: "50gb"
+                            },
+                            set_priority: {
+                                priority: 100
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        policy: "10G30D",
+        body: {
+            policy: {
+                phases: {
+                    hot: {
+                        min_age: "0ms",
+                        actions: {
+                            rollover: {
+                                max_age: "30d",
+                                max_size: "10gb",
+                                max_docs: 100000000
+                            },
+                            set_priority: {
+                                priority: 100
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    },
+    {
+        policy: "200G",
+        body: {
+            policy: {
+                phases: {
+                    hot: {
+                        min_age: "0ms",
+                        actions: {
+                            rollover: {
+                                max_size: "200gb"
+                            },
+                            set_priority: {
+                                priority: 100
                             }
                         }
                     }

definitions/index-templates.ts

@@ -3,7 +3,7 @@ import {ConfigurationModule} from "../modules/config";
 const shards = 2;
 const replicas = 0;
 const refresh = "1s";
-const defaultLifecyclePolicy = "50G30D";
+let defaultLifecyclePolicy = "200G";
 
 export * from './index-lifecycle-policies';
 
@@ -15,30 +15,54 @@ const compression = "best_compression";
 const cm = new ConfigurationModule();
 const chain = cm.config.settings.chain;
 
+if (cm.config.settings.hot_warm_policy) {
+    defaultLifecyclePolicy = "hyperion-rollover";
+}
+
+if(cm.config.settings.custom_policy) {
+    defaultLifecyclePolicy = cm.config.settings.custom_policy;
+}
+
+const defaultIndexSettings = {
+    "index": {
+        "number_of_shards": shards,
+        "refresh_interval": refresh,
+        "number_of_replicas": replicas,
+        "codec": compression
+    }
+};
+
+const actionSettings = {
+    index: {
+        lifecycle: {
+            "name": defaultLifecyclePolicy,
+            "rollover_alias": chain + "-action"
+        },
+        codec: compression,
+        refresh_interval: refresh,
+        number_of_shards: shards * 2,
+        number_of_replicas: replicas,
+        sort: {
+            field: "global_sequence",
+            order: "desc"
+        }
+    }
+};
+
+if (cm.config.settings.hot_warm_policy) {
+    actionSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
+}
+
 export const action = {
     order: 0,
     index_patterns: [
         chain + "-action-*"
     ],
-    settings: {
-        index: {
-            lifecycle: {
-                "name": defaultLifecyclePolicy,
-                "rollover_alias": chain + "-action"
-            },
-            codec: compression,
-            refresh_interval: refresh,
-            number_of_shards: shards * 2,
-            number_of_replicas: replicas,
-            sort: {
-                field: "global_sequence",
-                order: "desc"
-            }
-        }
-    },
+    settings: actionSettings,
     mappings: {
         properties: {
             "@timestamp": {"type": "date"},
+            "ds_error": {"type": "boolean"},
             "global_sequence": {"type": "long"},
             "account_ram_deltas.delta": {"type": "integer"},
             "account_ram_deltas.account": {"type": "keyword"},
@@ -57,6 +81,7 @@ export const action = {
             "trx_id": {"type": "keyword"},
             "producer": {"type": "keyword"},
             "notified": {"type": "keyword"},
+            "signatures": {"enabled": false},
             "inline_count": {"type": "short"},
             "max_inline": {"type": "short"},
             "inline_filtered": {"type": "boolean"},
@@ -73,6 +98,8 @@ export const action = {
                     }
                 }
             },
+
+            // eosio::newaccount
             "@newaccount": {
                 "properties": {
                     "active": {"type": "object"},
@@ -80,6 +107,8 @@ export const action = {
                     "newact": {"type": "keyword"}
                 }
             },
+
+            // eosio::updateauth
             "@updateauth": {
                 "properties": {
                     "permission": {"type": "keyword"},
@@ -87,6 +116,8 @@ export const action = {
                     "auth": {"type": "object"}
                 }
             },
+
+            // *::transfer
             "@transfer": {
                 "properties": {
                     "from": {"type": "keyword"},
@@ -96,6 +127,8 @@ export const action = {
                     "memo": {"type": "text"}
                 }
             },
+
+            // eosio::unstaketorex
             "@unstaketorex": {
                 "properties": {
                     "owner": {"type": "keyword"},
@@ -103,12 +136,16 @@ export const action = {
                     "amount": {"type": "float"}
                 }
             },
+
+            // eosio::buyrex
             "@buyrex": {
                 "properties": {
                     "from": {"type": "keyword"},
                     "amount": {"type": "float"}
                 }
             },
+
+            // eosio::buyram
             "@buyram": {
                 "properties": {
                     "payer": {"type": "keyword"},
@@ -116,6 +153,8 @@ export const action = {
                     "quant": {"type": "float"}
                 }
             },
+
+            // eosio::buyrambytes
             "@buyrambytes": {
                 "properties": {
                     "payer": {"type": "keyword"},
@@ -123,6 +162,8 @@ export const action = {
                     "bytes": {"type": "long"}
                 }
             },
+
+            // eosio::delegatebw
             "@delegatebw": {
                 "properties": {
                     "from": {"type": "keyword"},
@@ -133,6 +174,8 @@ export const action = {
                     "amount": {"type": "float"}
                 }
             },
+
+            // eosio::undelegatebw
             "@undelegatebw": {
                 "properties": {
                     "from": {"type": "keyword"},
@@ -146,16 +189,95 @@ export const action = {
     }
 };
 
+const deltaSettings = {
+    "index": {
+        "lifecycle": {
+            "name": defaultLifecyclePolicy,
+            "rollover_alias": chain + "-delta"
+        },
+        "codec": compression,
+        "number_of_shards": shards * 2,
+        "refresh_interval": refresh,
+        "number_of_replicas": replicas,
+        "sort.field": ["block_num", "scope", "primary_key"],
+        "sort.order": ["desc", "asc", "asc"]
+    }
+};
+
+if (cm.config.settings.hot_warm_policy) {
+    deltaSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
+}
+
+export const delta = {
+    "index_patterns": [chain + "-delta-*"],
+    "settings": deltaSettings,
+    "mappings": {
+        "properties": {
+
+            // base fields
+            "@timestamp": {"type": "date"},
+            "ds_error": {"type": "boolean"},
+            "block_id": {"type": "keyword"},
+            "block_num": {"type": "long"},
+            "deleted_at": {"type": "long"},
+            "code": {"type": "keyword"},
+            "scope": {"type": "keyword"},
+            "table": {"type": "keyword"},
+            "payer": {"type": "keyword"},
+            "primary_key": {"type": "keyword"},
+            "data": {"enabled": false},
+            "value": {"enabled": false},
+
+            // eosio.msig::approvals
+            "@approvals.proposal_name": {"type": "keyword"},
+            "@approvals.provided_approvals": {"type": "object"},
+            "@approvals.requested_approvals": {"type": "object"},
+
+            // eosio.msig::proposal
+            "@proposal.proposal_name": {"type": "keyword"},
+            "@proposal.transaction": {"enabled": false},
+
+            // *::accounts
+            "@accounts.amount": {"type": "float"},
+            "@accounts.symbol": {"type": "keyword"},
+
+            // eosio::voters
+            "@voters.is_proxy": {"type": "boolean"},
+            "@voters.producers": {"type": "keyword"},
+            "@voters.last_vote_weight": {"type": "double"},
+            "@voters.proxied_vote_weight": {"type": "double"},
+            "@voters.staked": {"type": "float"},
+            "@voters.proxy": {"type": "keyword"},
+
+            // eosio::producers
+            "@producers.total_votes": {"type": "double"},
+            "@producers.is_active": {"type": "boolean"},
+            "@producers.unpaid_blocks": {"type": "long"},
+
+            // eosio::global
+            "@global": {
+                "properties": {
+                    "last_name_close": {"type": "date"},
+                    "last_pervote_bucket_fill": {"type": "date"},
+                    "last_producer_schedule_update": {"type": "date"},
+                    "perblock_bucket": {"type": "double"},
+                    "pervote_bucket": {"type": "double"},
+                    "total_activated_stake": {"type": "double"},
+                    "total_voteshare_change_rate": {"type": "double"},
+                    "total_unpaid_voteshare": {"type": "double"},
+                    "total_producer_vote_weight": {"type": "double"},
+                    "total_ram_bytes_reserved": {"type": "long"},
+                    "total_ram_stake": {"type": "long"},
+                    "total_unpaid_blocks": {"type": "long"},
+                }
+            }
+        }
+    }
+};
+
 export const abi = {
     "index_patterns": [chain + "-abi-*"],
-    "settings": {
-        "index": {
-            "number_of_shards": shards,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas,
-            "codec": compression
-        }
-    },
+    "settings": defaultIndexSettings,
     "mappings": {
         "properties": {
             "@timestamp": {"type": "date"},
@@ -171,19 +293,12 @@ export const abi = {
 
 export const permissionLink = {
     "index_patterns": [chain + "-link-*"],
-    "settings": {
-        "index": {
-            "number_of_shards": shards,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas,
-            "codec": compression
-        }
-    },
+    "settings": defaultIndexSettings,
     "mappings": {
         "properties": {
+            "block_num": {"type": "long"},
             "@timestamp": {"type": "date"},
             "present": {"type": "boolean"},
-            "block_num": {"type": "long"},
             "account": {"type": "keyword"},
             "code": {"type": "keyword"},
             "action": {"type": "keyword"},
@@ -192,18 +307,92 @@ export const permissionLink = {
     }
 };
 
-export const logs = {
-    "index_patterns": [chain + "-logs-*"],
-    "settings": {
-        "index": {
-            "number_of_shards": shards,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas,
-            "codec": compression
+export const permission = {
+    "index_patterns": [chain + "-perm-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "present": {"type": "boolean"},
+            "owner": {"type": "keyword"},
+            "name": {"type": "keyword"},
+            "parent": {"type": "keyword"},
+            "last_updated": {"type": "date"},
+            "auth": {"type": "object"}
         }
     }
 };
 
+export const resourceLimits = {
+    "index_patterns": [chain + "-reslimits-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "owner": {"type": "keyword"},
+            "total_weight": {"type": "long"},
+            "net_weight": {"type": "long"},
+            "cpu_weight": {"type": "long"},
+            "ram_bytes": {"type": "long"}
+        }
+    }
+};
+
+export const generatedTransaction = {
+    "index_patterns": [chain + "-gentrx-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "sender": {"type": "keyword"},
+            "sender_id": {"type": "keyword"},
+            "payer": {"type": "keyword"},
+            "trx_id": {"type": "keyword"},
+            "actions": {"enabled": false},
+            "packed_trx": {"enabled": false}
+        }
+    }
+};
+
+export const failedTransaction = {
+    "index_patterns": [chain + "-trxerr-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "status": {"type": "short"}
+        }
+    }
+};
+
+export const resourceUsage = {
+    "index_patterns": [chain + "-userres-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "owner": {"type": "keyword"},
+            "net_used": {"type": "long"},
+            "net_total": {"type": "long"},
+            "net_pct": {"type": "float"},
+            "cpu_used": {"type": "long"},
+            "cpu_total": {"type": "long"},
+            "cpu_pct": {"type": "float"},
+            "ram": {"type": "long"}
+        }
+    }
+};
+
+
+export const logs = {
+    "index_patterns": [chain + "-logs-*"],
+    "settings": defaultIndexSettings
+};
+
 export const block = {
     "index_patterns": [chain + "-block-*"],
     "settings": {
@@ -220,6 +409,8 @@ export const block = {
         "properties": {
             "@timestamp": {"type": "date"},
             "block_num": {"type": "long"},
+            "block_id": {"type": "keyword"},
+            "prev_id": {"type": "keyword"},
             "producer": {"type": "keyword"},
             "new_producers.producers.block_signing_key": {"enabled": false},
             "new_producers.producers.producer_name": {"type": "keyword"},
@@ -231,64 +422,6 @@ export const block = {
     }
 };
 
-export const delta = {
-    "index_patterns": [chain + "-delta-*"],
-    "settings": {
-        "index": {
-            "lifecycle": {
-                "name": defaultLifecyclePolicy,
-                "rollover_alias": chain + "-delta"
-            },
-            "codec": compression,
-            "number_of_shards": shards * 2,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas
-        }
-    },
-    "mappings": {
-        "properties": {
-            // "global_sequence": {"type": "long"},
-            "@timestamp": {"type": "date"},
-            "block_num": {"type": "long"},
-            "data": {"enabled": false},
-            "code": {"type": "keyword"},
-            "present": {"type": "boolean"},
-            "scope": {"type": "keyword"},
-            "table": {"type": "keyword"},
-            "payer": {"type": "keyword"},
-            "primary_key": {"type": "keyword"},
-            "@approvals.proposal_name": {"type": "keyword"},
-            "@approvals.provided_approvals": {"type": "object"},
-            "@approvals.requested_approvals": {"type": "object"},
-            "@accounts.amount": {"type": "float"},
-            "@accounts.symbol": {"type": "keyword"},
-            "@voters.is_proxy": {"type": "boolean"},
-            "@voters.producers": {"type": "keyword"},
-            "@voters.last_vote_weight": {"type": "double"},
-            "@voters.proxied_vote_weight": {"type": "double"},
-            "@voters.staked": {"type": "float"},
-            "@voters.proxy": {"type": "keyword"},
-            "@producers.total_votes": {"type": "double"},
-            "@producers.is_active": {"type": "boolean"},
-            "@producers.unpaid_blocks": {"type": "long"},
-            "@global.data": {
-                "properties": {
-                    "last_name_close": {"type": "date"},
-                    "last_pervote_bucket_fill": {"type": "date"},
-                    "last_producer_schedule_update": {"type": "date"},
-                    "perblock_bucket": {"type": "double"},
-                    "pervote_bucket": {"type": "double"},
-                    "total_activated_stake": {"type": "double"},
-                    "total_producer_vote_weight": {"type": "double"},
-                    "total_ram_kb_reserved": {"type": "float"},
-                    "total_ram_stake": {"type": "float"},
-                    "total_unpaid_blocks": {"type": "long"},
-                }
-            }
-        }
-    }
-};
-
 export const tableProposals = {
     "index_patterns": [chain + "-table-proposals-*"],
     "settings": {
@@ -303,11 +436,12 @@ export const tableProposals = {
     },
     "mappings": {
         "properties": {
+            "block_num": {"type": "long"},
+            "present": {"type": "boolean"},
             "proposal_name": {"type": "keyword"},
             "requested_approvals": {"type": "object"},
             "provided_approvals": {"type": "object"},
-            "executed": {"type": "boolean"},
-            "block_num": {"type": "long"}
+            "executed": {"type": "boolean"}
         }
     }
 };
@@ -326,37 +460,12 @@ export const tableAccounts = {
     },
     "mappings": {
         "properties": {
+            "block_num": {"type": "long"},
+            "present": {"type": "boolean"},
             "code": {"type": "keyword"},
             "scope": {"type": "keyword"},
             "amount": {"type": "float"},
-            "symbol": {"type": "keyword"},
-            "primary_key": {"type": "keyword"},
-            "block_num": {"type": "long"}
-        }
-    }
-};
-
-export const tableUserRes = {
-    "index_patterns": [chain + "-table-userres-*"],
-    "settings": {
-        "index": {
-            "codec": compression,
-            "number_of_shards": shards,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas,
-            "sort.field": "total_weight",
-            "sort.order": "desc"
-        }
-    },
-    "mappings": {
-        "properties": {
-            "owner": {"type": "keyword"},
-            "total_weight": {"type": "float"},
-            "net_weight": {"type": "float"},
-            "cpu_weight": {"type": "float"},
-            "ram_bytes": {"type": "long"},
-            "primary_key": {"type": "keyword"},
-            "block_num": {"type": "long"}
+            "symbol": {"type": "keyword"}
         }
     }
 };
@@ -375,13 +484,12 @@ export const tableDelBand = {
     },
     "mappings": {
         "properties": {
+            "block_num": {"type": "long"},
             "from": {"type": "keyword"},
             "to": {"type": "keyword"},
             "total_weight": {"type": "float"},
             "net_weight": {"type": "float"},
-            "cpu_weight": {"type": "float"},
-            "primary_key": {"type": "keyword"},
-            "block_num": {"type": "long"}
+            "cpu_weight": {"type": "float"}
         }
     }
 };
@@ -400,15 +508,14 @@ export const tableVoters = {
     },
     "mappings": {
         "properties": {
+            "block_num": {"type": "long"},
             "voter": {"type": "keyword"},
             "producers": {"type": "keyword"},
             "last_vote_weight": {"type": "double"},
             "is_proxy": {"type": "boolean"},
             "proxied_vote_weight": {"type": "double"},
             "staked": {"type": "double"},
-            "proxy": {"type": "keyword"},
-            "primary_key": {"type": "keyword"},
-            "block_num": {"type": "long"}
+            "proxy": {"type": "keyword"}
         }
     }
 };

definitions/lifecycle_policies.js

@@ -1,25 +0,0 @@
-module.exports = {
-    "ILPs": [
-        {
-            "policy": "50G30D",
-            "body": {
-                "policy": {
-                    "phases": {
-                        "hot": {
-                            "min_age": "0ms",
-                            "actions": {
-                                "rollover": {
-                                    "max_age": "30d",
-                                    "max_size": "50gb"
-                                },
-                                "set_priority": {
-                                    "priority": 100
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-};

docker/.gitignore

@@ -0,0 +1 @@
+docker-compose.yml

docker/elasticsearch/data/.gitignore

@@ -0,0 +1,5 @@
+# Ignore everything in this directory
+*
+
+# Except this file
+!.gitignore