v3.3.9

- fix readers stopping before having finished

.gitignore

@@ -11,33 +11,8 @@ pids
 *.seed
 *.pid.lock
 
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
 # Dependency directories
 node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
 
 # Optional npm cache directory
 .npm
@@ -51,18 +26,11 @@ typings/
 # Output of 'npm pack'
 *.tgz
 
-# Yarn Integrity file
-.yarn-integrity
-
 # dotenv environment variables file
 .env
 
-# next.js build output
-.next
-
 .idea
 *.pem
-ecosystem.config.js
 package-lock.json
 connections.json
 connections.json2
@@ -106,4 +74,20 @@ modules/**/*.js.map
 addons/**/*.js
 addons/**/*.js.map
 
+scripts/hpm.js
+scripts/hpm.js.map
+
+scripts/hyp-config.js
+scripts/hyp-config.js.map
+
+plugins/.state.json
 plugins/repos
+plugins/*.js
+.gitmodules
+
+docker/redis/data
+docker/elasticsearch/data
+docker/rabbitmq/data
+docker/eosio/data
+
+configuration_backups

README.md

@@ -1,54 +1,99 @@
-# Hyperion History API
+<!--suppress HtmlUnknownTarget, HtmlDeprecatedAttribute -->
+<br></br>
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://eosrio.io/hyperion-white.png">
+    <img alt="Hyperion Logo"
+         src="https://eosrio.io/hyperion.png">
+  </picture>
+</p>
 
-<img height="64" src="https://eosrio.io/hyperion.png">
-<br/>
-Scalable Full History API Solution for EOSIO based blockchains
+<h4 align="center">
+    Scalable Full History API Solution for 
+    <a href="https://antelope.io">
+        Antelope
+    </a>
+    (former EOSIO) based blockchains
+</h4>
 
-Made with ♥ by [EOS Rio](https://eosrio.io/)
+<br>
 
-Official documentation: https://hyperion.docs.eosrio.io/
+Made with ♥ by [Rio Blocks / EOS Rio](https://rioblocks.io/?lang=en)
+
+### [Official documentation](https://hyperion.docs.eosrio.io)
+
+### How to use
+
+ - [For Providers](https://hyperion.docs.eosrio.io/manual_installation/)
+
+ - [For Developers](https://hyperion.docs.eosrio.io/howtouse/)
+
+### Official plugins:
+
+- [Hyperion Lightweight Explorer](https://github.com/eosrio/hyperion-explorer-plugin)
 
 ### 1. Overview
 
-Hyperion is a full history solution for indexing, storing and retrieving EOSIO blockchain`s historical data. EOSIO protocol is highly scalable reaching up to tens of thousands of transactions per second demanding high performance indexing and optimized storage and querying solutions. Hyperion is developed to tackle those challenges providing open source software to be operated by block producers, infrastructure providers and dApp developers.
-
-Focused on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers, Hyperion implements an improved data structure
-actions are stored in a flattened format
-a parent field is added to the inline actions to point to the parent global sequence
-if the inline action data is identical to the parent it is considered a notification and thus removed from the database
-no blocks or transaction data is stored, all information can be reconstructed from actions
+Hyperion is a full history solution for indexing, storing and retrieving Antelope blockchain's historical data.
+Antelope protocol is highly scalable reaching up to tens of thousands of transactions per second demanding high
+performance indexing and optimized storage and querying solutions. Hyperion is developed to tackle those challenges
+providing open source software to be operated by block producers, infrastructure providers and dApp developers.
 
+Focused on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers,
+Hyperion implements an improved data structure. Actions are stored in a flattened format, transaction ids are added to 
+all inline actions, allowing to group by transaction without storing a full transaction index. Besides that if the inline 
+action data is identical to the parent, it is considered a notification and thus removed from the database.
+No full block or transaction data is stored, all information can be reconstructed from actions and deltas, only a block 
+header index is stored.
 
 ### 2. Architecture
-The following components are required in order to have a fully functional Hyperion API deployment, 
-for small use cases its fine to run all components on a single machine. But for larger chains and 
-production environments we recommend setting them up into different servers under a high-speed local network.
 
-#### 2.1 - Elasticsearch Cluster
+The following components are required in order to have a fully functional Hyperion API deployment.
+* For small use cases, it is absolutely fine to run all components on a single machine.
+* For larger chains and production environments, we recommend setting them up into different servers under a high-speed local network.
+
+#### 2.1 Elasticsearch Cluster
+
 The ES cluster is responsible for storing all indexed data.
-Direct access to the Hyperion API and Indexer must be provided. We recommend nodes in the 
-cluster to have at least 32GB of RAM and 8 cpu cores. SSD/NVME drives are recommended for 
-maximum indexing throughput. For production environments a multi-node cluster is highly recommended.
+Direct access to the Hyperion API and Indexer must be provided. We recommend nodes in the
+cluster to have at least 32 GB of RAM and 8 cpu cores. SSD/NVME drives are recommended for
+maximum indexing throughput, although HDDs can be used for cold storage nodes.
+For production environments, a multi-node cluster is highly recommended.
+
+#### 2.2 Hyperion Indexer
 
-#### 2.2 - Hyperion Indexer
 The Indexer is a Node.js based app that process data from the state history plugin and allows it to be indexed.
-The PM2 process manager is used to launch and operate the indexer. The configuration flexibility is very extensive, 
-so system recommendations will depend on the use case and data load. It will require access to at least one ES node, 
+The [PM2 process manager](https://pm2.keymetrics.io) is used to launch and operate the indexer. The configuration
+flexibility is very extensive,
+so system recommendations will depend on the use case and data load. It will require access to at least one ES node,
 RabbitMQ and the state history node.
 
-#### 2.3 - Hyperion API
+#### 2.3 Hyperion API
+
 Parallelizable API server that provides the V2 and V1 (legacy history plugin) endpoints.
-It is launched by PM2 and can also operate in cluster mode. It requires direct access to 
+It is launched by PM2 and can also operate in cluster mode. It requires direct access to
 at least one ES node for the queries and all other services for full healthcheck
 
-#### 2.4 - RabbitMQ
-Use as messaging queue and data transport between the indexer stages
+#### 2.4 RabbitMQ
 
-#### 2.5 - EOSIO State History
-Nodeos plugin used to collect action traces and state deltas. Provides data via websocket to the indexer
+Used as messaging queue and data transport between the indexer stages and for real-time data streaming
 
-### 3. How to use
+#### 2.5 Redis
 
-#### 3.1 [For Providers](https://eosrio.github.io/hyperion-docs/quickstart/)
+Used for transient data storage across processes and for the preemptive transaction caching used on
+the `v2/history/get_transaction` and `v2/history/check_transaction` endpoints
 
-#### 3.2 [For Developers](https://eosrio.github.io/hyperion-docs/howtouse/)
+#### 2.6 Leap State History
+
+[Leap / Nodeos](https://github.com/AntelopeIO/leap/tree/main/plugins/state_history_plugin) plugin used
+to collect action traces and state deltas. Provides data via websocket to the indexer
+
+#### 2.7 Hyperion Stream Client (optional)
+
+Web and Node.js client for real-time streaming on enabled hyperion
+providers. [Documentation](https://hyperion.docs.eosrio.io/stream_client/)
+
+#### 2.8 Hyperion Plugins (optional)
+
+Hyperion includes a flexible plugin architecture to allow further customization.
+Plugins are managed by the `hpm` (hyperion plugin manager) command line tool.

api/config/open_api.ts

@@ -4,6 +4,7 @@ export function generateOpenApiConfig(config: HyperionConfig) {
     const packageData = require('../../package');
     const health_link = `https://${config.api.server_name}/v2/health`;
     const explorer_link = `https://${config.api.server_name}/v2/explore`;
+
     let description = `
 <img height="64" src="https://eosrio.io/hyperion.png">
 ### Scalable Full History API Solution for EOSIO based blockchains
@@ -13,9 +14,15 @@ export function generateOpenApiConfig(config: HyperionConfig) {
 #### Provided by [${config.api.provider_name}](${config.api.provider_url})
 #### Health API: <a target="_blank" href="${health_link}">${health_link}</a>
 `;
-    if(config.api.enable_explorer) {
-        description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
+
+    if (config.plugins) {
+        if (config.plugins.explorer) {
+            if (config.plugins.explorer.enabled) {
+                description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
+            }
+        }
     }
+
     return {
         routePrefix: '/v2/docs',
         exposeRoute: true,

api/helpers/cacheManager.ts

@@ -0,0 +1,86 @@
+import {HyperionConfig} from "../../interfaces/hyperionConfig";
+import {createHash} from "crypto";
+import {FastifyRequest} from "fastify";
+
+export interface CacheConfig {
+    ttl: number;
+}
+
+export interface CachedEntry {
+    data: string;
+    exp: number;
+}
+
+export class CacheManager {
+
+    v1CacheConfigs: Map<string, CacheConfig> = new Map<string, CacheConfig>();
+    v1Caches: Map<string, Map<string, CachedEntry>> = new Map<string, Map<string, CachedEntry>>();
+
+    constructor(conf: HyperionConfig) {
+        if (conf.api.v1_chain_cache) {
+            conf.api.v1_chain_cache.forEach(value => {
+                this.v1CacheConfigs.set(value.path, {
+                    ttl: value.ttl
+                });
+                this.v1Caches.set(value.path, new Map<string, CachedEntry>());
+            });
+        }
+        setInterval(() => {
+            try {
+                // remove expired entries
+                let removeCount = 0;
+                const now = Date.now();
+                this.v1Caches.forEach((pathCacheMap: Map<string, CachedEntry>, pathKey: string) => {
+                    pathCacheMap.forEach((cache: CachedEntry, entryKey: string, map: Map<string, CachedEntry>) => {
+                        if (cache.exp < now) {
+                            map.delete(entryKey);
+                            removeCount++;
+                        }
+                    });
+                });
+                if (removeCount > 0) {
+                    console.log(`${removeCount} expired cache entries removed`);
+                }
+            } catch (e) {
+                console.log(e);
+            }
+        }, 5000);
+    }
+
+    hashPayload(input: string): string {
+        return createHash('sha256').update(input).digest().toString('hex');
+    }
+
+    setCachedData(hash: string, path: string, payload: any): void {
+        if (this.v1CacheConfigs.has(path) && this.v1Caches.has(path)) {
+            this.v1Caches.get(path).set(hash, {
+                data: payload,
+                exp: this.v1CacheConfigs.get(path).ttl + Date.now()
+            });
+        }
+    }
+
+    getCachedData(request: FastifyRequest): [string | null, string, string] {
+        const urlParts = request.url.split("?");
+        const pathComponents = urlParts[0].split('/');
+        const path = pathComponents.at(-1);
+        let payload = '';
+        if (request.method === 'POST') {
+            payload = JSON.stringify(request.body);
+        } else if (request.method === 'GET') {
+            payload = request.url;
+        }
+        const hashedString = this.hashPayload(payload);
+        if (this.v1Caches.has(path)) {
+            const entry = this.v1Caches.get(path).get(hashedString);
+            if (entry && entry.exp > Date.now()) {
+                return [entry.data, hashedString, path];
+            } else {
+                return [null, hashedString, path];
+            }
+        } else {
+            return [null, hashedString, path];
+        }
+    }
+
+}

api/plugins.ts

@@ -2,22 +2,32 @@ import * as Fastify from "fastify";
 import {IncomingMessage, Server, ServerResponse} from "http";
 
 // fastify plugins
-import * as fastify_elasticsearch from 'fastify-elasticsearch';
-import * as fastify_oas from 'fastify-oas';
-import * as fastify_cors from 'fastify-cors';
-import * as fastify_formbody from 'fastify-formbody';
-import * as fastify_redis from 'fastify-redis';
-import * as fastify_rate_limit from 'fastify-rate-limit';
+import * as fastifyElasticsearch from 'fastify-elasticsearch';
+import fastifySwagger from '@fastify/swagger';
+import fastifyCors from '@fastify/cors';
+import formBodyPlugin from '@fastify/formbody';
+import fastifyRedis from '@fastify/redis';
+import fastifyRateLimit from '@fastify/rate-limit';
 
 // custom plugins
 import fastify_eosjs from "./plugins/fastify-eosjs";
 
 export function registerPlugins(server: Fastify.FastifyInstance<Server, IncomingMessage, ServerResponse>, params: any) {
-    server.register(fastify_elasticsearch, params.fastify_elasticsearch);
-    server.register(fastify_oas, params.fastify_oas);
-    server.register(fastify_cors);
-    server.register(fastify_formbody);
-    server.register(fastify_redis, params.fastify_redis);
+    server.register(fastifyElasticsearch, params.fastify_elasticsearch);
+
+    if (params.fastify_swagger) {
+        server.register(fastifySwagger, params.fastify_swagger);
+    }
+
+    server.register(fastifyCors);
+
+    server.register(formBodyPlugin);
+
+    server.register(fastifyRedis, params.fastify_redis);
+
+    if (params.fastify_rate_limit) {
+        server.register(fastifyRateLimit, params.fastify_rate_limit);
+    }
+
     server.register(fastify_eosjs, params.fastify_eosjs);
-    server.register(fastify_rate_limit, params.fastify_rate_limit);
 }

api/plugins/eosjs.js

@@ -1,21 +0,0 @@
-const fp = require('fastify-plugin');
-const {Api} = require("eosjs");
-const {ConnectionManager} = require('../../connections/manager');
-const manager = new ConnectionManager();
-
-module.exports = fp(async (fastify, options, next) => {
-    const rpc = manager.nodeosJsonRPC;
-    const chain_data = await rpc.get_info();
-    const api = new Api({
-        rpc,
-        signatureProvider: null,
-        chainId: chain_data.chain_id,
-        textDecoder: new TextDecoder(),
-        textEncoder: new TextEncoder(),
-    });
-    fastify.decorate('eosjs', {api, rpc});
-    next();
-}, {
-    fastify: '>=2.0.0',
-    name: 'fastify-eosjs'
-});

api/plugins/fastify-eosjs.ts

@@ -1,20 +1,19 @@
-import * as fp from 'fastify-plugin';
-import {FastifyInstance} from "fastify";
+import {FastifyInstance, FastifyPluginOptions} from "fastify";
 import {Api} from "eosjs/dist";
+import fp from "fastify-plugin";
 
-export default fp(async (fastify: FastifyInstance, options, next) => {
-    const rpc = fastify.manager.nodeosJsonRPC;
-    const chain_data = await rpc.get_info();
-    const api = new Api({
-        rpc,
-        signatureProvider: null,
-        chainId: chain_data.chain_id,
-        textDecoder: new TextDecoder(),
-        textEncoder: new TextEncoder(),
-    });
-    fastify.decorate('eosjs', {api, rpc});
-    next();
+export default fp(async (fastify: FastifyInstance, options: FastifyPluginOptions): Promise<void> => {
+	const rpc = fastify.manager.nodeosJsonRPC;
+	const chain_data = await rpc.get_info();
+	const api = new Api({
+		rpc,
+		signatureProvider: null,
+		chainId: chain_data.chain_id,
+		textDecoder: new TextDecoder(),
+		textEncoder: new TextEncoder(),
+	});
+	fastify.decorate('eosjs', {api, rpc});
 }, {
-    fastify: '>=2.0.0',
-    name: 'fastify-eosjs'
+	fastify: '>=2.0.0',
+	name: 'fastify-eosjs'
 });

api/plugins/websocket.js

@@ -1,30 +0,0 @@
-const fp = require('fastify-plugin');
-const WebSocket = require('ws');
-
-module.exports = fp((fastify, opts, next) => {
-
-    const wss = new WebSocket.Server({
-        server: fastify.server
-    });
-
-    const ws = new WebSocket('ws://127.0.0.1:7001', [], {
-        perMessageDeflate: false
-    });
-
-    ws.on('open', function open() {
-        console.log('router connected');
-    });
-
-    fastify.decorate('wss', wss);
-    fastify.decorate('ws', ws);
-
-    fastify.addHook('onClose', (fastify, done) => {
-        fastify.ws.close(done);
-        fastify.wss.close(done);
-    });
-
-    next();
-}, {
-    fastify: '2.x',
-    name: 'fastify-websocket'
-});

api/routes.ts

@@ -1,32 +1,52 @@
-import * as fastify_static from "fastify-static";
 import {join} from "path";
-import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
-import {createReadStream, existsSync, readFileSync, unlinkSync} from "fs";
-import * as AutoLoad from "fastify-autoload";
+import {FastifyError, FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {createReadStream} from "fs";
 import {addSharedSchemas, handleChainApiRedirect} from "./helpers/functions";
+import autoLoad from '@fastify/autoload';
+import got from "got";
 
 function addRedirect(server: FastifyInstance, url: string, redirectTo: string) {
     server.route({
         url,
         method: 'GET',
-        schema: {hide: true},
-        handler: async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        schema: {
+            hide: true
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
             reply.redirect(redirectTo);
         }
     });
 }
 
 function addRoute(server: FastifyInstance, handlersPath: string, prefix: string) {
-    server.register(AutoLoad, {
+    server.register(autoLoad, {
         dir: join(__dirname, 'routes', handlersPath),
         ignorePattern: /.*(handler|schema).js/,
+        dirNameRoutePrefix: false,
         options: {prefix}
     });
 }
 
 export function registerRoutes(server: FastifyInstance) {
 
+    // build internal map of routes
+    const routeSet = new Set<string>();
+    server.decorate('routeSet', routeSet);
+    const ignoreList = [
+        '/v2',
+        '/v2/history',
+        '/v2/state',
+        '/v1/chain/*',
+        '/v1/chain'
+    ];
+    server.addHook('onRoute', opts => {
+        if (!ignoreList.includes(opts.url)) {
+            if (opts.url.startsWith('/v')) {
+                routeSet.add(opts.url);
+            }
+        }
+    });
+
     // Register fastify api routes
     addRoute(server, 'v2', '/v2');
     addRoute(server, 'v2-history', '/v2/history');
@@ -41,6 +61,8 @@ export function registerRoutes(server: FastifyInstance) {
 
     // chain api redirects
     addRoute(server, 'v1-chain', '/v1/chain');
+
+	// other v1 requests
     server.route({
         url: '/v1/chain/*',
         method: ["GET", "POST"],
@@ -48,93 +70,45 @@ export function registerRoutes(server: FastifyInstance) {
             summary: "Wildcard chain api handler",
             tags: ["chain"]
         },
-        handler: async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
             await handleChainApiRedirect(request, reply, server);
         }
     });
 
-    server.addHook('onError', (request, reply, error, done) => {
-        console.log(`[${request.req.headers['x-real-ip']}] ${request.req.method} ${request.req.url} failed with error: ${error.message}`);
+    // /v1/node/get_supported_apis
+    server.route({
+        url: '/v1/node/get_supported_apis',
+        method: ["GET"],
+        schema: {
+            summary: "Get list of supported APIs",
+            tags: ["node"]
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
+            const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
+            if (data.apis && data.apis.length > 0) {
+                const apiSet = new Set(server.routeSet);
+                data.apis.forEach((a) => apiSet.add(a));
+                reply.send({apis: [...apiSet]});
+            } else {
+                reply.send({apis: [...server.routeSet], error: 'nodeos did not send any data'});
+            }
+        }
+    });
+
+    server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
+        console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
         done();
     });
 
-    // server.addHook('onResponse', (request, reply, done) => {
-    //     if (reply.res.statusCode !== 200) {
-    //         console.log(`${request.req.url} - code: ${reply.res.statusCode}`);
-    //     }
-    //     done();
-    // });
-
-    // Serve integrated explorer
-    if (server.manager.config.api.enable_explorer) {
-
-        server.register(require('fastify-compress'), {global: false});
-
-        try {
-            const _data = readFileSync(join(__dirname, '..', 'hyperion-explorer', 'src', 'manifest.webmanifest'));
-            const tempPath = join(__dirname, '..', 'hyperion-explorer', 'dist', 'manifest.webmanifest');
-            if (existsSync(tempPath)) {
-                unlinkSync(tempPath);
-            }
-            const baseManifest = JSON.parse(_data.toString());
-            baseManifest.name = `Hyperion Explorer - ${server.manager.config.api.chain_name}`;
-            baseManifest.short_name = baseManifest.name;
-            server.get('/v2/explore/manifest.webmanifest', (request, reply) => {
-                reply.send(baseManifest);
-            });
-        } catch (e) {
-            console.log(e);
-        }
-
-        server.register(fastify_static, {
-            root: join(__dirname, '..', 'hyperion-explorer', 'dist'),
-            redirect: true,
-            wildcard: false,
-            prefix: '/v2/explore',
-            setHeaders: (res: ServerResponse, path) => {
-                if (path.endsWith('/ngsw-worker.js')) {
-                    res.setHeader('Service-Worker-Allowed', '/');
-                }
-            }
-        });
-
-        server.get(
-            '/v2/explore/**/*',
-            {
-                schema: {
-                    tags: ['internal']
-                }
-            },
-            (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-                reply.sendFile('index.html', join(__dirname, '..', 'hyperion-explorer', 'dist'));
-            }
-        );
-
-        server.get(
-            '/v2/explorer_metadata',
-            {
-                schema: {
-                    tags: ['internal']
-                }
-            },
-            (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-                reply.send({
-                    logo: server.manager.config.api.chain_logo_url,
-                    provider: server.manager.config.api.provider_name,
-                    provider_url: server.manager.config.api.provider_url,
-                    chain_name: server.manager.config.api.chain_name,
-                    chain_id: server.manager.conn.chains[server.manager.chain].chain_id,
-                    custom_core_token: server.manager.config.api.custom_core_token
-                });
-            });
-    }
-
     if (server.manager.config.features.streaming) {
         // steam client lib
-        server.get('/stream-client.js', {schema: {tags: ['internal']}}, (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-            const stream = createReadStream('./client_bundle.js');
-            reply.type('application/javascript').send(stream);
-        });
+        server.get(
+            '/stream-client.js',
+            {schema: {tags: ['internal']}},
+            (request: FastifyRequest, reply: FastifyReply) => {
+                const stream = createReadStream('./hyperion-stream-client.js');
+                reply.type('application/javascript').send(stream);
+            });
     }
 
     // Redirect routes to documentation

api/routes/history/getTransactedAccounts.js

@@ -1,200 +0,0 @@
-const {getCacheByHash} = require("../../helpers/functions");
-
-const {getTransactedAccountsSchema} = require("../../schemas");
-
-async function getTransactedAccounts(fastify, request) {
-    const t0 = Date.now();
-    const {redis, elastic} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, JSON.stringify(request.query));
-    if (cachedResponse) {
-        return cachedResponse;
-    }
-    const {account, min, max, direction, limit, contract, symbol} = request.query;
-    let _limit = 100;
-    if (limit) {
-        _limit = parseInt(limit, 10);
-        if (_limit > 500) {
-            _limit = 500;
-        }
-    }
-    let _min;
-    if (min) {
-        _min = parseInt(min, 10);
-        if (_min < 0) {
-            _min = 0;
-        }
-    }
-    let _max;
-    if (max) {
-        _max = parseInt(max, 10);
-        if (_max < 0) {
-            _max = null;
-        }
-    }
-
-    const response = {
-        query_time: null,
-        account: account
-    };
-
-    const must_array = [
-        {term: {"notified": account}},
-        {term: {"act.name": "transfer"}}
-    ];
-
-    if (min || max) {
-        const _range = {
-            "@transfer.amount": {}
-        };
-        if (min) {
-            _range["@transfer.amount"]["gte"] = _min;
-        }
-        if (max) {
-            _range["@transfer.amount"]["lt"] = _max;
-        }
-        must_array.push({range: _range});
-    }
-
-    if (contract) {
-        must_array.push({term: {"act.account": contract}});
-        response['contract'] = contract;
-    }
-
-    if (symbol) {
-        must_array.push({term: {"@transfer.symbol": symbol}});
-        response['symbol'] = symbol;
-    }
-
-    const sum_agg = {
-        "total_amount": {
-            sum: {
-                field: "@transfer.amount"
-            }
-        }
-    };
-
-    let filter_array = [];
-    if (request.query['after'] || request.query['before']) {
-        let _lte = "now";
-        let _gte = 0;
-        if (request.query['before']) {
-            _lte = request.query['before'];
-        }
-        if (request.query['after']) {
-            _gte = request.query['after'];
-        }
-        filter_array.push({
-            range: {
-                "@timestamp": {
-                    "gte": _gte,
-                    "lte": _lte
-                }
-            }
-        });
-    } else {
-        filter_array.push({
-            range: {
-                "block_num": {
-                    "gte": 0
-                }
-            }
-        });
-    }
-
-    if (direction === 'out' || direction === 'both') {
-        const outResults = await elastic.search({
-            index: process.env.CHAIN + '-action-*',
-            body: {
-                size: 0,
-                aggs: {
-                    "receivers": {
-                        terms: {
-                            field: "@transfer.to",
-                            size: _limit,
-                            order: {
-                                "total_amount": "desc"
-                            }
-                        },
-                        aggs: sum_agg
-                    }
-                },
-                query: {
-                    bool: {
-                        must: must_array,
-                        filter: filter_array,
-                        must_not: [
-                            {term: {"@transfer.to": account}}
-                        ]
-                    }
-                }
-            }
-        });
-        response['total_out'] = 0;
-        response['outputs'] = outResults['body']['aggregations']['receivers']['buckets'].map((bucket) => {
-            const _sum = parseFloat(bucket.total_amount.value.toFixed(4));
-            response['total_out'] += _sum;
-            return {
-                account: bucket.key,
-                sum: _sum,
-                transfers: bucket['doc_count'],
-                average: parseFloat((bucket.total_amount.value / bucket['doc_count']).toFixed(4))
-            };
-        });
-        response['total_out'] = parseFloat(response['total_out'].toFixed(4));
-    }
-
-    if (direction === 'in' || direction === 'both') {
-        const inResults = await elastic.search({
-            index: process.env.CHAIN + '-action-*',
-            body: {
-                size: 0,
-                aggs: {
-                    "senders": {
-                        terms: {
-                            field: "@transfer.from",
-                            size: _limit,
-                            order: {
-                                "total_amount": "desc"
-                            }
-                        },
-                        aggs: sum_agg
-                    }
-                },
-                query: {
-                    bool: {
-                        must: must_array,
-                        filter: filter_array,
-                        must_not: [
-                            {term: {"@transfer.from": account}}
-                        ]
-                    }
-                }
-            }
-        });
-        response['total_in'] = 0;
-        response['inputs'] = inResults['body']['aggregations']['senders']['buckets'].map((bucket) => {
-            const _sum = parseFloat(bucket.total_amount.value.toFixed(4));
-            response['total_in'] += _sum;
-            return {
-                account: bucket.key,
-                sum: _sum,
-                transfers: bucket['doc_count'],
-                average: parseFloat((bucket.total_amount.value / bucket['doc_count']).toFixed(4))
-            };
-        });
-        response['total_in'] = parseFloat(response['total_in'].toFixed(4));
-    }
-
-    response['query_time'] = Date.now() - t0;
-    redis.set(hash, JSON.stringify(response), 'EX', 600);
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get('/get_transacted_accounts', {
-        schema: getTransactedAccountsSchema.GET
-    }, async (request, reply) => {
-        reply.send(await getTransactedAccounts(fastify, request));
-    });
-    next()
-};

api/routes/history/getTransfers.js

@@ -1,98 +0,0 @@
-const {getTransfersSchema} = require("../../schemas");
-const _ = require('lodash');
-const {getCacheByHash} = require("../../helpers/functions");
-const route = '/get_transfers';
-
-const maxActions = 1000;
-
-function processActions(results) {
-    const action_traces = results['body']['hits']['hits'];
-    const actions = [];
-    let sum = 0;
-    for (const aTrace of action_traces) {
-        const action = aTrace['_source'];
-        const name = action.act.name;
-        if (action['@' + name]) {
-            if (action['@transfer']) {
-                sum += parseFloat(action['@transfer']['amount']);
-            }
-            action['act']['data'] = _.merge(action['@' + name], action['act']['data']);
-            delete action['@' + name];
-        }
-        actions.push(action);
-    }
-    return [sum, actions];
-}
-
-async function getTransfers(fastify, request) {
-    const {redis, elastic} = fastify;
-    const [cachedResponse, hash] = await getCacheByHash(redis, route + JSON.stringify(request.query));
-    if (cachedResponse) {
-        return cachedResponse;
-    }
-    const must_array = [];
-    must_array.push({"term": {"act.name": {"value": "transfer"}}});
-    if (request.query['from']) {
-        must_array.push({"term": {"@transfer.from": {"value": request.query['from'].toLowerCase()}}});
-    }
-    if (request.query['to']) {
-        must_array.push({"term": {"@transfer.to": {"value": request.query['to'].toLowerCase()}}});
-    }
-    if (request.query['symbol']) {
-        must_array.push({"term": {"@transfer.symbol": {"value": request.query['symbol'].toUpperCase()}}});
-    }
-    if (request.query['contract']) {
-        must_array.push({"term": {"act.account": {"value": request.query['contract'].toLowerCase()}}});
-    }
-    if (request.query['after'] || request.query['before']) {
-        let _lte = "now";
-        let _gte = 0;
-        if (request.query['before']) {
-            _lte = request.query['before'];
-        }
-        if (request.query['after']) {
-            _gte = request.query['after'];
-        }
-        must_array.push({
-            range: {
-                "@timestamp": {
-                    "gte": _gte,
-                    "lte": _lte
-                }
-            }
-        });
-    }
-    let limit, skip;
-    limit = parseInt(request.query.limit, 10);
-    if (limit < 1) {
-        return 'invalid limit parameter';
-    }
-    skip = parseInt(request.query.skip, 10);
-    if (skip < 0) {
-        return 'invalid skip parameter';
-    }
-    const body = {"query": {"bool": {"must": must_array}}};
-    const results = await elastic.search({
-        "index": process.env.CHAIN + '-action-*',
-        "from": skip || 0,
-        "size": (limit > maxActions ? maxActions : limit) || 10,
-        "body": body
-    });
-    const [sum, _actions] = processActions(results);
-    const response = {
-        "action_count": results['body']['hits']['total']['value'],
-        "total_amount": sum,
-        "actions": _actions
-    };
-    redis.set(hash, JSON.stringify(response), 'EX', 30);
-    return response;
-}
-
-module.exports = function (fastify, opts, next) {
-    fastify.get(route, {
-        schema: getTransfersSchema.GET
-    }, async (request, reply) => {
-        reply.send(await getTransfers(fastify, request));
-    });
-    next()
-};

api/routes/v1-chain/abi_bin_to_json/index.ts

@@ -7,8 +7,8 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Returns an object containing rows from the specified table.',
         {
-            "code": 'AccountName#',
-            "action": 'AccountName#',
+            "code": {$ref: 'AccountName#'},
+            "action": {$ref: 'AccountName#'},
             "binargs": {
                 "type": "string",
                 "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",

api/routes/v1-chain/get_abi/index.ts

@@ -6,7 +6,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         fastify,
         getRouteName(__filename),
         'Retrieves the ABI for a contract based on its account name',
-        {"account_name": 'AccountName#'},
+        {"account_name": {$ref: 'AccountName#'}},
         ["account_name"]
     );
     next();

api/routes/v1-chain/get_account/index.ts

@@ -6,7 +6,9 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         fastify,
         getRouteName(__filename),
         'Returns an object containing various details about a specific account on the blockchain.',
-        {"account_name": 'AccountName#'},
+        {
+            "account_name": {$ref: 'AccountName#'}
+        },
         ["account_name"]
     );
     next();

api/routes/v1-chain/get_code/index.ts

@@ -7,7 +7,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves contract code',
         {
-            "account_name": 'AccountName#',
+            "account_name": {$ref: 'AccountName#'},
             "code_as_wasm": {
                 "type": "integer",
                 "default": 1,

api/routes/v1-chain/get_currency_balance/index.ts

@@ -7,9 +7,9 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves the current balance',
         {
-            "code": 'AccountName#',
-            "account": 'AccountName#',
-            "symbol": 'Symbol#',
+            "code": {$ref: 'AccountName#'},
+            "account": {$ref: 'AccountName#'},
+            "symbol": {$ref: 'Symbol#'}
         },
         ["code", "account", "symbol"]
     );

api/routes/v1-chain/get_currency_stats/index.ts

@@ -7,12 +7,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves currency stats',
         {
-            "code": {
-                description: 'contract name',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
+            "code": {$ref: 'AccountName#'},
             "symbol": {
                 description: 'token symbol',
                 type: 'string',

api/routes/v1-chain/get_raw_abi/index.ts

@@ -7,7 +7,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves raw ABI for a contract based on account name',
         {
-            "account_name": 'AccountName#'
+            "account_name": {$ref: 'AccountName#'}
         },
         ["account_name"]
     );

api/routes/v1-chain/get_raw_code_and_abi/index.ts

@@ -7,7 +7,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Retrieves raw code and ABI for a contract based on account name',
         {
-            "account_name": 'AccountName#'
+            "account_name": {$ref: 'AccountName#'}
         },
         ["account_name"]
     );

api/routes/v1-chain/get_table_rows/index.ts

@@ -36,7 +36,8 @@ export default function (fastify: FastifyInstance, opts: any, next) {
             "lower_bound": {
                 "type": "string"
             }
-        }
+        },
+        ["code", "table"]
     );
     next();
 }

api/routes/v1-chain/push_transaction/index.ts

@@ -17,7 +17,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     signatures: {
                         "type": "array",
                         "description": "array of signatures required to authorize transaction",
-                        "items": 'Signature#'
+                        "items": {$ref: 'Signature#'}
                     },
                     compression: {
                         "type": "boolean",

api/routes/v1-chain/push_transactions/index.ts

@@ -2,45 +2,54 @@ import {FastifyInstance} from "fastify";
 import {addApiRoute, chainApiHandler, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    addApiRoute(
-        fastify,
-        'POST',
-        getRouteName(__filename),
-        chainApiHandler,
-        {
-            description: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
-            summary: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
-            tags: ['chain'],
-            body: {
-                type: ['array', 'object', 'string'],
-                items: {
-                    type: ['object', 'string'],
-                    additionalProperties: false,
-                    minProperties: 8,
-                    required: [
-                        "expiration",
-                        "ref_block_num",
-                        "ref_block_prefix",
-                        "max_net_usage_words",
-                        "max_cpu_usage_ms",
-                        "delay_sec",
-                        "context_free_actions",
-                        "actions"
-                    ],
-                    properties: {
-                        "expiration": 'Expiration#',
-                        "ref_block_num": {"type": "integer"},
-                        "ref_block_prefix": {"type": "integer"},
-                        "max_net_usage_words": 'WholeNumber#',
-                        "max_cpu_usage_ms": 'WholeNumber#',
-                        "delay_sec": {"type": "integer"},
-                        "context_free_actions": {"type": "array", "items": 'ActionItems#'},
-                        "actions": {"type": "array", "items": 'ActionItems#'},
-                        "transaction_extensions": {"type": "array", "items": 'BlockExtensions#'}
-                    },
-                }
-            }
-        }
-    );
-    next();
+	addApiRoute(
+		fastify,
+		'POST',
+		getRouteName(__filename),
+		chainApiHandler,
+		{
+			description: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+			summary: "This method expects a transaction in JSON format and will attempt to apply it to the blockchain.",
+			tags: ['chain'],
+			body: {
+				type: ['array', 'object', 'string'],
+				items: {
+					type: ['object', 'string'],
+					additionalProperties: false,
+					minProperties: 8,
+					required: [
+						"expiration",
+						"ref_block_num",
+						"ref_block_prefix",
+						"max_net_usage_words",
+						"max_cpu_usage_ms",
+						"delay_sec",
+						"context_free_actions",
+						"actions"
+					],
+					properties: {
+						"expiration": {$ref: 'Expiration#'},
+						"ref_block_num": {"type": "integer"},
+						"ref_block_prefix": {"type": "integer"},
+						"max_net_usage_words": {$ref: 'WholeNumber#'},
+						"max_cpu_usage_ms": {$ref: 'WholeNumber#'},
+						"delay_sec": {"type": "integer"},
+						"context_free_actions": {
+							"type": "array",
+							"items": {$ref: 'ActionItems#'}
+						},
+						"actions": {
+							"type": "array",
+							"items": {$ref: 'ActionItems#'}
+						},
+						"transaction_extensions": {
+							"type": "array",
+							"items": {$ref: 'BlockExtensions#'}
+						}
+					},
+				}
+			}
+		}
+	);
+	next();
 }

api/routes/v1-chain/send_transaction/index.ts

@@ -17,7 +17,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     signatures: {
                         "type": "array",
                         "description": "array of signatures required to authorize transaction",
-                        "items": 'Signature#'
+                        "items": {$ref: 'Signature#'}
                     },
                     compression: {
                         "type": "boolean",

api/routes/v1-history/get_actions/get_actions.ts

@@ -1,7 +1,148 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
-import * as flatstr from 'flatstr';
+import {Serialize} from "eosjs";
+import {hLog} from "../../../../helpers/common_functions";
+import * as AbiEOS from "@eosrio/node-abieos";
+import {ApiResponse} from "@elastic/elasticsearch";
+import {TextDecoder, TextEncoder} from "util";
+import {JsonRpc} from "eosjs/dist";
+
+const abi_remapping = {
+    "_Bool": "bool"
+};
+
+const txEnc = new TextEncoder();
+const txDec = new TextDecoder();
+
+async function fetchAbiHexAtBlockElastic(client, chain, contract_name, last_block, get_json) {
+    try {
+        const _includes = ["actions", "tables", "block"];
+        if (get_json) {
+            _includes.push("abi");
+        } else {
+            _includes.push("abi_hex");
+        }
+        const query = {
+            bool: {
+                must: [
+                    {term: {account: contract_name}},
+                    {range: {block: {lte: last_block}}}
+                ]
+            }
+        };
+        const queryResult: ApiResponse = await client.search({
+            index: `${chain}-abi-*`,
+            body: {
+                size: 1, query,
+                sort: [{block: {order: "desc"}}],
+                _source: {includes: _includes}
+            }
+        });
+        const results = queryResult.body.hits.hits;
+        if (results.length > 0) {
+            const nextRefResponse: ApiResponse = await client.search({
+                index: `${chain}-abi-*`,
+                body: {
+                    size: 1,
+                    query: {
+                        bool: {
+                            must: [
+                                {term: {account: contract_name}},
+                                {range: {block: {gte: last_block}}}
+                            ]
+                        }
+                    },
+                    sort: [{block: {order: "asc"}}],
+                    _source: {includes: ["block"]}
+                }
+            });
+            const nextRef = nextRefResponse.body.hits.hits;
+            if (nextRef.length > 0) {
+                return {
+                    valid_until: nextRef[0]._source.block,
+                    ...results[0]._source
+                };
+            }
+            return results[0]._source;
+        } else {
+            return null;
+        }
+    } catch (e) {
+        hLog(e);
+        return null;
+    }
+}
+
+async function getAbiFromHeadBlock(rpc: JsonRpc, code) {
+    let _abi;
+    try {
+        _abi = (await rpc.get_abi(code)).abi;
+    } catch (e) {
+        hLog(e);
+    }
+    return {abi: _abi, valid_until: null, valid_from: null};
+}
+
+async function getContractAtBlock(esClient, rpc, chain, accountName: string, block_num: number, check_action?: string) {
+    let savedAbi, abi;
+    savedAbi = await fetchAbiHexAtBlockElastic(esClient, chain, accountName, block_num, true);
+    if (savedAbi === null || (savedAbi.actions && !savedAbi.actions.includes(check_action))) {
+        savedAbi = await getAbiFromHeadBlock(rpc, accountName);
+        if (!savedAbi) return [null, null];
+        abi = savedAbi.abi;
+    } else {
+        try {
+            abi = JSON.parse(savedAbi.abi);
+        } catch (e) {
+            hLog(e);
+            return [null, null];
+        }
+    }
+    if (!abi) return [null, null];
+    const initialTypes = Serialize.createInitialTypes();
+    let types;
+    try {
+        types = Serialize.getTypesFromAbi(initialTypes, abi);
+    } catch (e) {
+        let remapped = false;
+        for (const struct of abi.structs) {
+            for (const field of struct.fields) {
+                if (abi_remapping[field.type]) {
+                    field.type = abi_remapping[field.type];
+                    remapped = true;
+                }
+            }
+        }
+        if (remapped) {
+            try {
+                types = Serialize.getTypesFromAbi(initialTypes, abi);
+            } catch (e) {
+                hLog('failed after remapping abi');
+                hLog(accountName, block_num, check_action);
+                hLog(e);
+            }
+        } else {
+            hLog(accountName, block_num);
+            hLog(e);
+        }
+    }
+    const actions = new Map();
+    for (const {name, type} of abi.actions) {
+        actions.set(name, Serialize.getType(types, type));
+    }
+    const result = {types, actions, tables: abi.tables};
+    if (check_action) {
+        if (actions.has(check_action)) {
+            try {
+                AbiEOS['load_abi'](accountName, JSON.stringify(abi));
+            } catch (e) {
+                hLog(e);
+            }
+        }
+    }
+    return [result, abi];
+}
+
 
 const terms = ["notified", "act.authorization.actor"];
 const extendedActions = new Set(["transfer", "newaccount", "updateauth"]);
@@ -12,7 +153,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
         request.body = JSON.parse(request.body);
     }
 
-    const reqBody = request.body;
+    const reqBody = request.body as any;
     const should_array = [];
     for (const entry of terms) {
         const tObj = {term: {}};
@@ -155,11 +296,13 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
             actions = actions.slice(index);
         }
-        actions.forEach((action, index) => {
+
+        for (let i = 0; i < actions.length; i++) {
+            let action = actions[i];
             action = action._source;
             let act: any = {
                 "global_action_seq": action.global_sequence,
-                "account_action_seq": index,
+                "account_action_seq": i,
                 "block_num": action.block_num,
                 "block_time": action['@timestamp'],
                 "action_trace": {
@@ -175,7 +318,31 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             };
             mergeActionMeta(action);
             act.action_trace.act = action.act;
-            act.action_trace.act.hex_data = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+
+            if (reqBody.hex_data) {
+                try {
+                    const [contract, _] = await getContractAtBlock(
+                        fastify.elastic,
+                        fastify.eosjs.rpc,
+                        fastify.manager.chain,
+                        action.act.account,
+                        action.block_num
+                    );
+                    action.act.hex_data = Serialize.serializeActionData(
+                        contract,
+                        action.act.account,
+                        action.act.name,
+                        action.act.data,
+                        txEnc,
+                        txDec
+                    );
+                } catch (e: any) {
+                    console.log(e);
+                }
+            }
+
+            // TODO: Optionally re-encode using the original ABI, will increase query time
+            // act.action_trace.act.hex_data = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
             if (action.act.account_ram_deltas) {
                 act.action_trace.account_ram_deltas = action.account_ram_deltas
             }
@@ -185,13 +352,13 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             act.action_trace.receiver = receipt.receiver;
             act.account_action_seq = receipt['recv_sequence'];
             response.actions.push(act);
-        });
+        }
     }
     return response;
 }
 
 export function getActionsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getActions, fastify, request, route));
     }
 }

api/routes/v1-history/get_controlled_accounts/get_controlled_accounts.ts

@@ -1,72 +1,72 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {ApiResponse} from "@elastic/elasticsearch";
 
 async function getControlledAccounts(fastify: FastifyInstance, request: FastifyRequest) {
-    if (typeof request.body === 'string') {
-        request.body = JSON.parse(request.body)
-    }
-    let controlling_account = request.body.controlling_account;
-    const results: ApiResponse = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        size: 100,
-        body: {
-            query: {
-                bool: {
-                    should: [
-                        {
-                            term: {"@updateauth.auth.accounts.permission.actor": controlling_account}},
-                        {
-                            bool: {
-                                must: [
-                                    {term: {"act.account": "eosio"}},
-                                    {term: {"act.name": "newaccount"}},
-                                    {term: {"act.authorization.actor": controlling_account}}
-                                ]
-                            }
-                        }
-                    ],
-                    minimum_should_match: 1
-                }
-            },
-            sort: [
-                {"global_sequence": {"order": "desc"}}
-            ]
-        }
-    });
+	if (typeof request.body === 'string') {
+		request.body = JSON.parse(request.body)
+	}
+	let controlling_account = request.body["controlling_account"];
+	const results: ApiResponse = await fastify.elastic.search({
+		index: fastify.manager.chain + '-action-*',
+		size: 100,
+		body: {
+			query: {
+				bool: {
+					should: [
+						{
+							term: {"@updateauth.auth.accounts.permission.actor": controlling_account}
+						},
+						{
+							bool: {
+								must: [
+									{term: {"act.account": "eosio"}},
+									{term: {"act.name": "newaccount"}},
+									{term: {"act.authorization.actor": controlling_account}}
+								]
+							}
+						}
+					],
+					minimum_should_match: 1
+				}
+			},
+			sort: [
+				{"global_sequence": {"order": "desc"}}
+			]
+		}
+	});
 
-    const response = {
-        controlled_accounts: []
-    };
+	const response = {
+		controlled_accounts: []
+	};
 
-    const hits = results.body.hits.hits;
+	const hits = results.body.hits.hits;
 
-    if (hits.length > 0) {
-        response.controlled_accounts = hits.map((v) => {
-            if (v._source.act.name === 'newaccount') {
-                if (v._source['@newaccount'].newact) {
-                    return v._source['@newaccount'].newact;
-                } else if (v._source.act.data.newact) {
-                    return v._source.act.data.newact;
-                } else {
-                    return null;
-                }
-            } else if (v._source.act.name === 'updateauth') {
-                return v._source.act.data.account;
-            } else {
-                return null;
-            }
-        });
-    }
-    if (response.controlled_accounts.length > 0) {
-        response.controlled_accounts = [...(new Set(response.controlled_accounts))];
-    }
-    return response;
+	if (hits.length > 0) {
+		response.controlled_accounts = hits.map((v) => {
+			if (v._source.act.name === 'newaccount') {
+				if (v._source['@newaccount'].newact) {
+					return v._source['@newaccount'].newact;
+				} else if (v._source.act.data.newact) {
+					return v._source.act.data.newact;
+				} else {
+					return null;
+				}
+			} else if (v._source.act.name === 'updateauth') {
+				return v._source.act.data.account;
+			} else {
+				return null;
+			}
+		});
+	}
+	if (response.controlled_accounts.length > 0) {
+		response.controlled_accounts = [...(new Set(response.controlled_accounts))];
+	}
+	return response;
 }
 
 export function getControlledAccountsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getControlledAccounts, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getControlledAccounts, fastify, request, route));
+	}
 }

api/routes/v1-history/get_transaction/get_transaction.ts

@@ -1,31 +1,21 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
 import {createHash} from "crypto";
-import * as flatstr from 'flatstr';
+import flatstr from 'flatstr';
+import {GetInfoResult} from "eosjs/dist/eosjs-rpc-interfaces";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
 	if (typeof request.body === 'string') {
 		request.body = JSON.parse(request.body)
 	}
-	const pResults = await Promise.all([fastify.eosjs.rpc.get_info(), fastify.elastic['search']({
-		"index": fastify.manager.chain + '-action-*',
-		"body": {
-			"query": {
-				"bool": {
-					must: [
-						{term: {"trx_id": request.body.id.toLowerCase()}}
-					]
-				}
-			},
-			"sort": {
-				"global_sequence": "asc"
-			}
-		}
-	})]);
-	const results = pResults[1];
+	const body: any = request.body;
+	const redis = fastify.redis;
+	const trxId = body.id.toLowerCase();
+	const conf = fastify.manager.config;
+	const cachedData = await redis.hgetall('trx_' + trxId);
+
 	const response: any = {
-		"id": request.body.id,
+		"id": body.id,
 		"trx": {
 			"receipt": {
 				"status": "executed",
@@ -42,11 +32,89 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
 		},
 		"block_num": 0,
 		"block_time": "",
-		"last_irreversible_block": pResults[0].last_irreversible_block_num,
+		"last_irreversible_block": undefined,
 		"traces": []
 	};
 
-	const hits = results['body']['hits']['hits'];
+	let hits;
+
+	// build get_info request with caching
+	const $getInfo = new Promise<GetInfoResult>(resolve => {
+		const key = `${fastify.manager.chain}_get_info`;
+		fastify.redis.get(key).then(value => {
+			if (value) {
+				resolve(JSON.parse(value));
+			} else {
+				fastify.eosjs.rpc.get_info().then(value1 => {
+					fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
+					resolve(value1);
+				}).catch((reason) => {
+					console.log(reason);
+					response.error = 'failed to get last_irreversible_block_num'
+					resolve(null);
+				});
+			}
+		});
+	});
+
+	// reconstruct hits from cached data
+	if (cachedData && Object.keys(cachedData).length > 0) {
+		const gsArr = [];
+		for (let cachedDataKey in cachedData) {
+			gsArr.push(cachedData[cachedDataKey]);
+		}
+		gsArr.sort((a, b) => {
+			return a.global_sequence - b.global_sequence;
+		});
+		hits = gsArr.map(value => {
+			return {
+				_source: JSON.parse(value)
+			};
+		});
+		const promiseResults = await Promise.all([
+			redis.ttl('trx_' + trxId),
+			$getInfo
+		]);
+		response.cache_expires_in = promiseResults[0];
+		response.last_irreversible_block = promiseResults[1].last_irreversible_block_num;
+	}
+
+	// search on ES if cache is not present
+	if (!hits) {
+		const _size = conf.api.limits.get_trx_actions || 100;
+		const blockHint = parseInt(body.block_num_hint, 10);
+		let indexPattern = '';
+		if (blockHint) {
+			const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
+			indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
+		} else {
+			indexPattern = fastify.manager.chain + '-action-*';
+		}
+		let pResults;
+		try {
+
+			// build search request
+			const $search = fastify.elastic.search({
+				index: indexPattern,
+				size: _size,
+				body: {
+					query: {bool: {must: [{term: {trx_id: trxId}}]}},
+					sort: {global_sequence: "asc"}
+				}
+			});
+
+			// execute in parallel
+			pResults = await Promise.all([$getInfo, $search]);
+		} catch (e) {
+			console.log(e.message);
+			if (e.meta.statusCode === 404) {
+				return response;
+			}
+		}
+		hits = pResults[1]['body']['hits']['hits'];
+		response.last_irreversible_block = pResults[0].last_irreversible_block_num;
+	}
+
 
 	if (hits.length > 0) {
 		const actions = hits;
@@ -68,7 +136,9 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
 		for (let action of actions) {
 			action = action._source;
 			mergeActionMeta(action);
-			action.act['hex_data'] = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+
+			// action.act['hex_data'] = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+
 			if (action.parent === 0) {
 				response.trx.trx.actions.push(action.act);
 			}
@@ -98,7 +168,7 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
 				except: null,
 				inline_traces: [],
 				producer_block_id: "",
-				trx_id: request.body.id,
+				trx_id: body.id,
 				notified: action.notified
 			};
 			let hash = createHash('sha256');
@@ -157,7 +227,7 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
 						except: null,
 						inline_traces: [],
 						producer_block_id: "",
-						trx_id: request.body.id,
+						trx_id: body.id,
 					};
 					traces[action.global_sequence].inline_traces.unshift(trace);
 					response.traces.push(trace);
@@ -166,7 +236,7 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
 			}
 		});
 	} else {
-		const errmsg = "Transaction " + request.body.id.toLowerCase() + " not found in history and no block hint was given";
+		const errmsg = "Transaction " + body.id.toLowerCase() + " not found in history and no block hint was given";
 		return {
 			code: 500,
 			message: "Internal Service Error",
@@ -189,7 +259,7 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
 		reply.send(await timedQuery(getTransaction, fastify, request, route));
 	}
 }

api/routes/v1-history/get_transaction/index.ts

@@ -9,7 +9,10 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         tags: ['history'],
         body: {
             type: ['object', 'string'],
-            properties: {id: {description: 'transaction id', type: 'string'}},
+            properties: {
+                id: {description: 'transaction id', type: 'string'},
+                block_num_hint: {description: 'block number hint', type: 'integer'},
+            },
             required: ["id"]
         }
     });

api/routes/v1-trace/get_block/get_block.ts

@@ -1,5 +1,4 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import * as _ from "lodash";
 
@@ -19,7 +18,7 @@ async function getBlockTrace(fastify: FastifyInstance, request: FastifyRequest)
         request.body = JSON.parse(request.body);
     }
 
-    const reqBody = request.body;
+    const reqBody: any = request.body;
     const targetBlock = parseInt(reqBody.block_num);
     let searchBody;
 
@@ -121,7 +120,7 @@ async function getBlockTrace(fastify: FastifyInstance, request: FastifyRequest)
 }
 
 export function getBlockTraceHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getBlockTrace, fastify, request, route));
     }
 }

api/routes/v2-history/check_transaction/check_transaction.ts

@@ -0,0 +1,33 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {timedQuery} from "../../../helpers/functions";
+
+async function checkTransaction(fastify: FastifyInstance, request: FastifyRequest) {
+	const query: any = request.query;
+	const txId = query.id.toLowerCase();
+	const data = await fastify.redis.get(txId) as any;
+	if (data) {
+		const jsonObj = JSON.parse(data);
+		const response = {
+			id: txId,
+			status: jsonObj.status,
+			block_num: jsonObj.b,
+			root_action: jsonObj.a,
+			signatures: [],
+		};
+		if (jsonObj.s && jsonObj.s.length > 0) {
+			response.signatures = jsonObj.s;
+		}
+		return response;
+	} else {
+		return {
+			id: txId,
+			status: 'unknown'
+		};
+	}
+}
+
+export function checkTransactionHandler(fastify: FastifyInstance, route: string) {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(checkTransaction, fastify, request, route));
+	}
+}

api/routes/v2-history/check_transaction/index.ts

@@ -0,0 +1,55 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {checkTransactionHandler} from "./check_transaction";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+	const schema = {
+		description: 'check if a transaction was included in a block',
+		summary: 'check if a transaction was included in a block',
+		tags: ['history'],
+		querystring: {
+			type: 'object',
+			properties: {
+				"id": {
+					description: 'transaction id',
+					type: 'string',
+					minLength: 64,
+					maxLength: 64
+				}
+			},
+			required: ["id"]
+		},
+		response: extendResponseSchema({
+			"id": {type: "string"},
+			"status": {type: "string"},
+			"block_num": {type: "number"},
+			"root_action": {
+				type: "object",
+				properties: {
+					"account": {type: "string"},
+					"name": {type: "string"},
+					"authorization": {
+						type: "array",
+						items: {
+							type: 'object',
+							properties: {
+								"actor": {type: "string"},
+								"permission": {type: "string"}
+							}
+						}
+					},
+					"data": {type: "string"}
+				}
+			},
+			"signatures": {type: "array", items: {type: 'string'}}
+		})
+	};
+	addApiRoute(
+		fastify,
+		'GET',
+		getRouteName(__filename),
+		checkTransactionHandler,
+		schema
+	);
+	next();
+}

api/routes/v2-history/export_actions/export_actions.ts

@@ -0,0 +1,12 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {timedQuery} from "../../../helpers/functions";
+
+async function exportActions(fastify: FastifyInstance, request: FastifyRequest) {
+
+}
+
+export function exportActionsHandler(fastify: FastifyInstance, route: string) {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(exportActions, fastify, request, route));
+	}
+}

api/routes/v2-history/export_actions/index.ts

@@ -0,0 +1,19 @@
+import {FastifyInstance, FastifySchema} from "fastify";
+import {addApiRoute, getRouteName} from "../../../helpers/functions";
+import {getCreatorHandler} from "../get_creator/get_creator";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+	const schema: FastifySchema = {
+		description: 'request large action data export',
+		summary: 'request large action data export',
+		tags: ['history']
+	};
+	addApiRoute(
+		fastify,
+		'GET',
+		getRouteName(__filename),
+		getCreatorHandler,
+		schema
+	);
+	next();
+}

api/routes/v2-history/get_abi_snapshot/get_abi_snapshot.ts

@@ -1,5 +1,4 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 
 async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest) {
@@ -8,9 +7,11 @@ async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest)
         block_num: null
     };
 
-    const code = request.query.contract;
-    const block = request.query.block;
-    const should_fetch = request.query.fetch;
+    const query: any = request.query;
+
+    const code = query.contract;
+    const block = query.block;
+    const should_fetch = query.fetch;
 
     const mustArray = [];
 
@@ -21,7 +22,7 @@ async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest)
     }
 
     const results = await fastify.elastic.search({
-        index: fastify.manager.chain + '-abi',
+        index: fastify.manager.chain + '-abi-*',
         size: 1,
         body: {
             query: {bool: {must: mustArray}},
@@ -44,7 +45,7 @@ async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest)
 }
 
 export function getAbiSnapshotHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getAbiSnapshot, fastify, request, route));
     }
 }

api/routes/v2-history/get_actions/definitions.ts

@@ -1,5 +1,6 @@
 export const terms = [
     "notified",
+    "receipts.receiver",
     "act.authorization.actor"
 ];
 
@@ -15,8 +16,8 @@ export const extendedActions = new Set([
 ]);
 
 export const primaryTerms = [
-    "notified",
     "block_num",
+    "block_id",
     "global_sequence",
     "producer",
     "@timestamp",
@@ -24,5 +25,10 @@ export const primaryTerms = [
     "action_ordinal",
     "cpu_usage_us",
     "net_usage_words",
-    "trx_id"
+    "trx_id",
+    "receipts.receiver",
+    "receipts.global_sequence",
+    "receipts.recv_sequence",
+    "receipts.auth_sequence.account",
+    "receipts.auth_sequence.sequence"
 ];

api/routes/v2-history/get_actions/functions.ts

@@ -1,4 +1,4 @@
-import {extendedActions, primaryTerms, terms} from "./definitions";
+import {primaryTerms, terms} from "./definitions";
 
 export function addSortedBy(query, queryBody, sort_direction) {
     if (query['sortedBy']) {
@@ -70,68 +70,110 @@ function addRangeQuery(queryStruct, prop, pkey, query) {
 
 export function applyTimeFilter(query, queryStruct) {
     if (query['after'] || query['before']) {
-        let _lte = "now";
-        let _gte = "0";
-        if (query['before']) {
-            _lte = query['before'];
-            if (!_lte.endsWith("Z")) {
-                _lte += "Z";
-            }
-        }
-        if (query['after']) {
-            _gte = query['after'];
-            if (!_gte.endsWith("Z")) {
-                _gte += "Z";
-            }
-        }
-        if (!queryStruct.bool['filter']) {
-            queryStruct.bool['filter'] = [];
-        }
-        queryStruct.bool['filter'].push({
-            range: {
-                "@timestamp": {
-                    "gte": _gte,
-                    "lte": _lte
+        if (query['after']?.includes('T') || query['before']?.includes('T')) {
+            let _lte = "now";
+            let _gte = "0";
+            if (query['before']) {
+                try {
+                    _lte = new Date(query['before']).toISOString();
+                } catch (e) {
+                    throw new Error(e.message + ' [before]');
                 }
             }
-        });
+            if (query['after']) {
+                try {
+                    _gte = new Date(query['after']).toISOString();
+                } catch (e) {
+                    throw new Error(e.message + ' [after]');
+                }
+            }
+            if (!queryStruct.bool['filter']) {
+                queryStruct.bool['filter'] = [];
+            }
+            queryStruct.bool['filter'].push({
+                range: {
+                    "@timestamp": {
+                        "gte": _gte,
+                        "lte": _lte
+                    }
+                }
+            });
+        } else {
+            // search by block number
+            const rangeObj = {
+                range: {
+                    block_num: {}
+                }
+            };
+            if (parseInt(query['after']) > 0) {
+                rangeObj.range.block_num['gte'] = query['after'];
+            }
+            if (parseInt(query['before']) > 0) {
+                rangeObj.range.block_num['lte'] = query['before'];
+            }
+            if (Object.keys(rangeObj.range.block_num).length > 0) {
+                if (!queryStruct.bool['filter']) {
+                    queryStruct.bool['filter'] = [];
+                }
+                queryStruct.bool['filter'].push(rangeObj);
+            }
+        }
     }
 }
 
-export function applyGenericFilters(query, queryStruct) {
+export function applyGenericFilters(query, queryStruct, allowedExtraParams: Set<string>) {
     for (const prop in query) {
         if (Object.prototype.hasOwnProperty.call(query, prop)) {
             const pair = prop.split(".");
             if (pair.length > 1 || primaryTerms.includes(pair[0])) {
                 let pkey;
-                if (pair.length > 1) {
-                    pkey = extendedActions.has(pair[0]) ? "@" + prop : prop;
+                if (pair.length > 1 && allowedExtraParams) {
+                    pkey = allowedExtraParams.has(pair[0]) ? "@" + prop : prop;
                 } else {
                     pkey = prop;
                 }
                 if (query[prop].indexOf("-") !== -1) {
                     addRangeQuery(queryStruct, prop, pkey, query);
                 } else {
-                    const _termQuery = {};
+                    const _qObj = {};
                     const parts = query[prop].split(",");
                     if (parts.length > 1) {
                         processMultiVars(queryStruct, parts, prop);
                     } else if (parts.length === 1) {
-                        const andParts = parts[0].split(" ");
-                        if (andParts.length > 1) {
-                            andParts.forEach(value => {
-                                const _q = {};
-                                console.log(value);
-                                _q[pkey] = value;
-                                queryStruct.bool.must.push({term: _q});
+
+                        // @transfer.memo special case
+                        if (pkey === '@transfer.memo') {
+                            _qObj[pkey] = {
+                                query: parts[0]
+                            };
+
+                            if (query.match_fuzziness) {
+                                _qObj[pkey].fuzziness = query.match_fuzziness;
+                            }
+
+                            if (query.match_operator) {
+                                _qObj[pkey].operator = query.match_operator;
+                            }
+
+                            queryStruct.bool.must.push({
+                                match: _qObj
                             });
                         } else {
-                            if (parts[0].startsWith("!")) {
-                                _termQuery[pkey] = parts[0].replace("!", "");
-                                queryStruct.bool.must_not.push({term: _termQuery});
+                            const andParts = parts[0].split(" ");
+                            if (andParts.length > 1) {
+                                andParts.forEach(value => {
+                                    const _q = {};
+                                    _q[pkey] = value;
+                                    queryStruct.bool.must.push({term: _q});
+                                });
                             } else {
-                                _termQuery[pkey] = parts[0];
-                                queryStruct.bool.must.push({term: _termQuery});
+                                if (parts[0].startsWith("!")) {
+                                    _qObj[pkey] = parts[0].replace("!", "");
+                                    queryStruct.bool.must_not.push({term: _qObj});
+                                } else {
+                                    _qObj[pkey] = parts[0];
+                                    queryStruct.bool.must.push({term: _qObj});
+                                }
                             }
                         }
                     }

api/routes/v2-history/get_actions/get_actions.ts

@@ -1,5 +1,4 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {getTrackTotalHits, mergeActionMeta, timedQuery} from "../../../helpers/functions";
 import {
     addSortedBy,
@@ -12,7 +11,7 @@ import {
 } from "./functions";
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
-    const query = request.query;
+    const query: any = request.query;
     const maxActions = fastify.manager.config.api.limits.get_actions;
     const queryStruct = {
         "bool": {
@@ -23,11 +22,17 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
     };
 
     const {skip, limit} = getSkipLimit(query, maxActions);
+
     const sort_direction = getSortDir(query);
+
     applyAccountFilters(query, queryStruct);
-    applyGenericFilters(query, queryStruct);
+
+    applyGenericFilters(query, queryStruct, fastify.allowedActionQueryParamSet);
+
     applyTimeFilter(query, queryStruct);
+
     applyCodeActionFilters(query, queryStruct);
+
     // allow precise counting of total hits
     const trackTotalHits = getTrackTotalHits(query);
 
@@ -77,8 +82,18 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
 
     if (results['hits'].length > 0) {
         const actions = results['hits'];
-        for (let action of actions) {
-            action = action._source;
+        for (let action of actions.map(a => a._source)) {
+
+            try {
+                if (action.act.data) {
+                    if (action.act.data.account && action.act.data.name && action.act.data.authorization) {
+                        action.act.data = action.act.data.data;
+                    }
+                }
+            } catch (e: any) {
+                console.log(e);
+            }
+
             mergeActionMeta(action);
 
             if (query.noBinary === true) {
@@ -113,7 +128,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
 }
 
 export function getActionsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getActions, fastify, request, route));
     }
 }

api/routes/v2-history/get_actions/index.ts

@@ -1,22 +1,50 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {getActionsHandler} from "./get_actions";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export const getActionResponseSchema = {
-    "@timestamp": {type: "string"},
     "timestamp": {type: "string"},
     "block_num": {type: "number"},
+    "block_id": {type: "string"},
     "trx_id": {type: "string"},
     "act": {
         type: 'object',
         properties: {
             "account": {type: "string"},
-            "name": {type: "string"}
+            "name": {type: "string"},
+            "authorization": {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        "actor": {type: "string"},
+                        "permission": {type: "string"},
+                    }
+                }
+            }
         },
         additionalProperties: true
     },
-    "notified": {
-        type: "array", items: {type: "string"}
+    "receipts": {
+        type: "array",
+        items: {
+            type: "object",
+            properties: {
+                "receiver": {type: "string"},
+                "global_sequence": {type: "number"},
+                "recv_sequence": {type: "number"},
+                "auth_sequence": {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            "account": {type: "string"},
+                            "sequence": {type: "number"},
+                        }
+                    }
+                }
+            }
+        }
     },
     "cpu_usage_us": {type: "number"},
     "net_usage_words": {type: "number"},
@@ -32,15 +60,15 @@ export const getActionResponseSchema = {
         }
     },
     "global_sequence": {type: "number"},
-    "receiver": {type: 'string'},
     "producer": {type: "string"},
     "parent": {type: "number"},
     "action_ordinal": {type: 'number'},
-    "creator_action_ordinal": {type: 'number'}
+    "creator_action_ordinal": {type: 'number'},
+    "signatures": {type: "array", items: {type: 'string'}}
 };
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
+    const schema: FastifySchema = {
         description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
             ' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
         summary: 'get root actions',

api/routes/v2-history/get_created_accounts/get_created_accounts.ts

@@ -1,58 +1,58 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {getSkipLimit} from "../get_actions/functions";
 
 async function getCreatedAccounts(fastify: FastifyInstance, request: FastifyRequest) {
 
 
-    const {skip, limit} = getSkipLimit(request.query);
-    const maxActions = fastify.manager.config.api.limits.get_created_accounts;
-    const results = await fastify.elastic.search({
-        "index": fastify.manager.chain + '-action-*',
-        "from": skip || 0,
-        "size": (limit > maxActions ? maxActions : limit) || 100,
-        "body": {
-            "query": {
-                "bool": {
-                    must: [
-                        {term: {"act.authorization.actor": request.query.account.toLowerCase()}},
-                        {term: {"act.name": "newaccount"}},
-                        {term: {"act.account": "eosio"}}
-                    ]
-                }
-            },
-            sort: {
-                "global_sequence": "desc"
-            }
-        }
-    });
+	const query: any = request.query;
+	const {skip, limit} = getSkipLimit(query);
+	const maxActions = fastify.manager.config.api.limits.get_created_accounts;
+	const results = await fastify.elastic.search({
+		"index": fastify.manager.chain + '-action-*',
+		"from": skip || 0,
+		"size": (limit > maxActions ? maxActions : limit) || 100,
+		"body": {
+			"query": {
+				"bool": {
+					must: [
+						{term: {"act.authorization.actor": query.account.toLowerCase()}},
+						{term: {"act.name": "newaccount"}},
+						{term: {"act.account": "eosio"}}
+					]
+				}
+			},
+			sort: {
+				"global_sequence": "desc"
+			}
+		}
+	});
 
-    const response = {accounts: []};
+	const response = {accounts: []};
 
-    if (results['body']['hits']['hits'].length > 0) {
-        const actions = results['body']['hits']['hits'];
-        for (let action of actions) {
-            action = action._source;
-            const _tmp = {};
-            if (action['act']['data']['newact']) {
-                _tmp['name'] = action['act']['data']['newact'];
-            } else if (action['@newaccount']['newact']) {
-                _tmp['name'] = action['@newaccount']['newact'];
-            } else {
-                console.log(action);
-            }
-            _tmp['trx_id'] = action['trx_id'];
-            _tmp['timestamp'] = action['@timestamp'];
-            response.accounts.push(_tmp);
-        }
-    }
+	if (results['body']['hits']['hits'].length > 0) {
+		const actions = results['body']['hits']['hits'];
+		for (let action of actions) {
+			action = action._source;
+			const _tmp = {};
+			if (action['act']['data']['newact']) {
+				_tmp['name'] = action['act']['data']['newact'];
+			} else if (action['@newaccount']['newact']) {
+				_tmp['name'] = action['@newaccount']['newact'];
+			} else {
+				console.log(action);
+			}
+			_tmp['trx_id'] = action['trx_id'];
+			_tmp['timestamp'] = action['@timestamp'];
+			response.accounts.push(_tmp);
+		}
+	}
 
-    return response;
+	return response;
 }
 
 export function getCreatedAccountsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getCreatedAccounts, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getCreatedAccounts, fastify, request, route));
+	}
 }

api/routes/v2-history/get_created_accounts/index.ts

@@ -1,44 +1,44 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {getCreatedAccountsHandler} from "./get_created_accounts";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
-        description: 'get all accounts created by one creator',
-        summary: 'get created accounts',
-        tags: ['accounts'],
-        querystring: extendQueryStringSchema({
-            "account": {
-                description: 'creator account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            }
-        }, ["account"]),
-        response: extendResponseSchema({
-            "query_time": {
-                type: "number"
-            },
-            "total": {
-                type: "object",
-                properties: {
-                    "value": {type: "number"},
-                    "relation": {type: "string"}
-                }
-            },
-            "accounts": {
-                type: "array",
-                items: {
-                    type: 'object',
-                    properties: {
-                        'name': {type: 'string'},
-                        'timestamp': {type: 'string'},
-                        'trx_id': {type: 'string'}
-                    }
-                }
-            }
-        })
-    };
-    addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatedAccountsHandler, schema);
-    next();
+	const schema: FastifySchema = {
+		description: 'get all accounts created by one creator',
+		summary: 'get created accounts',
+		tags: ['accounts'],
+		querystring: extendQueryStringSchema({
+			"account": {
+				description: 'creator account',
+				type: 'string',
+				minLength: 1,
+				maxLength: 12
+			}
+		}, ["account"]),
+		response: extendResponseSchema({
+			"query_time": {
+				type: "number"
+			},
+			"total": {
+				type: "object",
+				properties: {
+					"value": {type: "number"},
+					"relation": {type: "string"}
+				}
+			},
+			"accounts": {
+				type: "array",
+				items: {
+					type: 'object',
+					properties: {
+						'name': {type: 'string'},
+						'timestamp': {type: 'string'},
+						'trx_id': {type: 'string'}
+					}
+				}
+			}
+		})
+	};
+	addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatedAccountsHandler, schema);
+	next();
 }

api/routes/v2-history/get_creator/get_creator.ts

@@ -1,99 +1,106 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 
 async function getCreator(fastify: FastifyInstance, request: FastifyRequest) {
 
-	const response = {
-		account: request.query.account,
-		creator: '',
-		timestamp: '',
-		block_num: 0,
-		trx_id: '',
-	};
+    const query: any = request.query;
 
-	if (request.query.account === fastify.manager.config.settings.eosio_alias) {
-		const genesisBlock = await fastify.eosjs.rpc.get_block(1);
-		if (genesisBlock) {
-			response.creator = '__self__';
-			response.timestamp = genesisBlock.timestamp;
-			response.block_num = 1;
-			response.trx_id = "";
-			return response;
-		} else {
-			throw new Error("genesis block not found");
-		}
-	}
+    const response = {
+        account: query.account,
+        creator: '',
+        timestamp: '',
+        block_num: 0,
+        trx_id: '',
+    };
 
-	const results = await fastify.elastic.search({
-		"index": fastify.manager.chain + '-action-*',
-		"body": {
-			size: 1,
-			query: {
-				bool: {
-					must: [{term: {"@newaccount.newact": request.query.account}}]
-				}
-			}
-		}
-	});
+    if (query.account === fastify.manager.config.settings.eosio_alias) {
+        try {
+            const genesisBlock = await fastify.eosjs.rpc.get_block(1);
+            if (genesisBlock) {
+                response.timestamp = genesisBlock.timestamp;
+            }
+        } catch (e) {
+            console.log(e.message);
+        }
+        response.creator = '__self__';
+        response.block_num = 1;
+        response.trx_id = "";
+        return response;
+    }
 
-	if (results['body']['hits']['hits'].length === 1) {
-		const result = results['body']['hits']['hits'][0]._source;
-		response.trx_id = result.trx_id;
-		response.block_num = result.block_num;
-		response.creator = result.act.data.creator;
-		response.timestamp = result['@timestamp'];
-		return response;
-	} else {
-		let accountInfo;
-		try {
-			accountInfo = await fastify.eosjs.rpc.get_account(request.query.account);
-		} catch (e) {
-			throw new Error("account not found");
-		}
-		if (accountInfo) {
-			try {
-				response.timestamp = accountInfo.created;
-				const blockHeader = await fastify.elastic.search({
-					"index": fastify.manager.chain + '-block-*',
-					"body": {
-						size: 1,
-						query: {
-							bool: {
-								must: [{term: {"@timestamp": response.timestamp}}]
-							}
-						}
-					}
-				});
-				const blockId = blockHeader['body']['hits']['hits'][0]._source.block_id;
-				const blockData = await fastify.eosjs.rpc.get_block(blockId);
-				response.block_num = blockData.block_num;
-				for (const transaction of blockData["transactions"]) {
-					if (typeof transaction.trx !== 'string') {
-						const actions = transaction.trx.transaction.actions;
-						for (const act of actions) {
-							if (act.name === 'newaccount') {
-								if (act.data.name === request.query.account) {
-									response.creator = act.data.creator;
-									response.trx_id = transaction.id;
-									return response;
-								}
-							}
-						}
-					}
-				}
-				return response;
-			} catch (e) {
-				throw new Error("account creation not found");
-			}
-		} else {
-			throw new Error("account creation not found");
-		}
-	}
+    const results = await fastify.elastic.search({
+        "index": fastify.manager.chain + '-action-*',
+        "body": {
+            size: 1,
+            query: {
+                bool: {
+                    must: [{term: {"@newaccount.newact": query.account}}]
+                }
+            }
+        }
+    });
+
+    if (results['body']['hits']['hits'].length === 1) {
+        const result = results['body']['hits']['hits'][0]._source;
+        response.trx_id = result.trx_id;
+        response.block_num = result.block_num;
+        response.creator = result.act.data.creator;
+        response.timestamp = result['@timestamp'];
+        return response;
+    } else {
+        let accountInfo;
+        try {
+            accountInfo = await fastify.eosjs.rpc.get_account(query.account);
+        } catch (e) {
+            throw new Error("account not found");
+        }
+        if (accountInfo) {
+            try {
+                response.timestamp = accountInfo.created;
+                const blockHeader = await fastify.elastic.search({
+                    "index": fastify.manager.chain + '-block-*',
+                    "body": {
+                        size: 1,
+                        query: {
+                            bool: {
+                                must: [{term: {"@timestamp": response.timestamp}}]
+                            }
+                        }
+                    }
+                });
+                const hits = blockHeader['body']['hits']['hits'];
+                if (hits.length > 0 && hits[0]._source) {
+                    const blockId = blockHeader['body']['hits']['hits'][0]._source.block_id;
+                    const blockData = await fastify.eosjs.rpc.get_block(blockId);
+                    response.block_num = blockData.block_num;
+                    for (const transaction of blockData["transactions"]) {
+                        if (typeof transaction.trx !== 'string') {
+                            const actions = transaction.trx.transaction.actions;
+                            for (const act of actions) {
+                                if (act.name === 'newaccount') {
+                                    if (act.data.name === query.account) {
+                                        response.creator = act.data.creator;
+                                        response.trx_id = transaction.id;
+                                        return response;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                return response;
+            } catch (e) {
+                console.log(e.message);
+                throw new Error("account creation not found");
+            }
+        } else {
+            throw new Error("account not found");
+        }
+    }
 }
 
 export function getCreatorHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-		reply.send(await timedQuery(getCreator, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getCreator, fastify, request, route));
+    }
 }

api/routes/v2-history/get_creator/index.ts

@@ -1,45 +1,45 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {getCreatorHandler} from "./get_creator";
 import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
-        description: 'get account creator',
-        summary: 'get account creator',
-        tags: ['accounts'],
-        querystring: {
-            type: 'object',
-            properties: {
-                "account": {
-                    description: 'created account',
-                    type: 'string',
-                    minLength: 1,
-                    maxLength: 12
-                }
-            },
-            required: ["account"]
-        },
-        response: extendResponseSchema({
-            "account": {
-                type: "string"
-            },
-            "creator": {
-                type: "string"
-            },
-            "timestamp": {
-                type: "string"
-            },
-            "block_num": {
-                type: "integer"
-            },
-            "trx_id": {
-                type: "string"
-            },
-            "indirect_creator": {
-                type: "string"
-            }
-        })
-    };
-    addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatorHandler, schema);
-    next();
+	const schema: FastifySchema = {
+		description: 'get account creator',
+		summary: 'get account creator',
+		tags: ['accounts'],
+		querystring: {
+			type: 'object',
+			properties: {
+				"account": {
+					description: 'created account',
+					type: 'string',
+					minLength: 1,
+					maxLength: 12
+				}
+			},
+			required: ["account"]
+		},
+		response: extendResponseSchema({
+			"account": {
+				type: "string"
+			},
+			"creator": {
+				type: "string"
+			},
+			"timestamp": {
+				type: "string"
+			},
+			"block_num": {
+				type: "integer"
+			},
+			"trx_id": {
+				type: "string"
+			},
+			"indirect_creator": {
+				type: "string"
+			}
+		})
+	};
+	addApiRoute(fastify, 'GET', getRouteName(__filename), getCreatorHandler, schema);
+	next();
 }

api/routes/v2-history/get_deltas/get_deltas.ts

@@ -1,91 +1,99 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {mergeDeltaMeta, timedQuery} from "../../../helpers/functions";
 import {applyTimeFilter} from "../get_actions/functions";
 
 async function getDeltas(fastify: FastifyInstance, request: FastifyRequest) {
-	let skip, limit;
-	let sort_direction = 'desc';
-	const mustArray = [];
-	for (const param in request.query) {
-		if (Object.prototype.hasOwnProperty.call(request.query, param)) {
-			const value = request.query[param];
-			switch (param) {
-				case 'limit': {
-					limit = parseInt(value, 10);
-					if (limit < 1) {
-						return 'invalid limit parameter';
-					}
-					break;
-				}
-				case 'skip': {
-					skip = parseInt(value, 10);
-					if (skip < 0) {
-						return 'invalid skip parameter';
-					}
-					break;
-				}
-				case 'sort': {
-					if (value === 'asc' || value === '1') {
-						sort_direction = 'asc';
-					} else if (value === 'desc' || value === '-1') {
-						sort_direction = 'desc'
-					} else {
-						return 'invalid sort direction';
-					}
-					break;
-				}
-				case 'before': {
-					break;
-				}
-				case 'after': {
-					break;
-				}
-				default: {
-					const values = request.query[param].split(",");
-					if (values.length > 1) {
-						const terms = {};
-						terms[param] = values;
-						mustArray.push({terms: terms});
-					} else {
-						const term = {};
-						term[param] = values[0];
-						mustArray.push({term: term});
-					}
-					break;
-				}
-			}
-		}
-	}
+    let skip, limit;
+    let sort_direction = 'desc';
+    const mustArray = [];
+    const query: any = request.query;
+    for (const param in query) {
+        if (Object.prototype.hasOwnProperty.call(query, param)) {
+            const value = query[param];
+            switch (param) {
+                case 'limit': {
+                    limit = parseInt(value, 10);
+                    if (limit < 1) {
+                        return 'invalid limit parameter';
+                    }
+                    break;
+                }
+                case 'skip': {
+                    skip = parseInt(value, 10);
+                    if (skip < 0) {
+                        return 'invalid skip parameter';
+                    }
+                    break;
+                }
+                case 'sort': {
+                    if (value === 'asc' || value === '1') {
+                        sort_direction = 'asc';
+                    } else if (value === 'desc' || value === '-1') {
+                        sort_direction = 'desc'
+                    } else {
+                        return 'invalid sort direction';
+                    }
+                    break;
+                }
+                case 'before': {
+                    break;
+                }
+                case 'after': {
+                    break;
+                }
+                default: {
+                    if (typeof value === 'string') {
+                        const values = query[param].split(",");
+                        if (values.length > 1) {
+                            const terms = {};
+                            terms[param] = values;
+                            mustArray.push({terms: terms});
+                        } else {
+                            const term = {};
+                            term[param] = values[0];
+                            mustArray.push({term: term});
+                        }
+                    } else {
+                        if (typeof value === 'number') {
+                            const term = {};
+                            term[param] = value;
+                            mustArray.push({term: term});
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+    }
 
-	const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
-	const queryStruct = {bool: {must: mustArray}};
+    const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
+    const queryStruct = {bool: {must: mustArray}};
 
-	applyTimeFilter(request.query, queryStruct);
+    applyTimeFilter(query, queryStruct);
 
-	const results = await fastify.elastic.search({
-		"index": fastify.manager.chain + '-delta-*',
-		"from": skip || 0,
-		"size": (limit > maxDeltas ? maxDeltas : limit) || 10,
-		"body": {
-			query: queryStruct,
-			sort: {
-				"block_num": sort_direction
-			}
-		}
-	});
-	const deltas = results['body']['hits']['hits'].map((d) => {
-		return mergeDeltaMeta(d._source);
-	});
-	return {
-		query_time: null,
-		total: results['body']['hits']['total'],
-		deltas
-	};
+    const results = await fastify.elastic.search({
+        "index": fastify.manager.chain + '-delta-*',
+        "from": skip || 0,
+        "size": (limit > maxDeltas ? maxDeltas : limit) || 10,
+        "body": {
+            query: queryStruct,
+            sort: {
+                "block_num": sort_direction
+            }
+        }
+    });
+    const deltas = results['body']['hits']['hits'].map((d) => {
+        return mergeDeltaMeta(d._source);
+    });
+    return {
+        query_time: null,
+        total: results['body']['hits']['total'],
+        deltas
+    };
 }
 
 export function getDeltasHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-		reply.send(await timedQuery(getDeltas, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getDeltas, fastify, request, route));
+    }
 }

api/routes/v2-history/get_deltas/index.ts

@@ -1,9 +1,9 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {getDeltasHandler} from "./get_deltas";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
+    const schema: FastifySchema = {
         description: 'get state deltas',
         summary: 'get state deltas',
         tags: ['history'],
@@ -31,7 +31,11 @@ export default function (fastify: FastifyInstance, opts: any, next) {
             "before": {
                 description: 'filter before specified date (ISO8601)',
                 type: 'string'
-            }
+            },
+            "present": {
+                description: 'delta present flag',
+                type: 'number'
+            },
         }),
         response: extendResponseSchema({
             "deltas": {
@@ -40,12 +44,12 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: 'object',
                     properties: {
                         "timestamp": {type: 'string'},
+                        "present": {type: 'number'},
                         "code": {type: 'string'},
                         "scope": {type: 'string'},
                         "table": {type: 'string'},
                         "primary_key": {type: 'string'},
                         "payer": {type: 'string'},
-                        "present": {type: 'boolean'},
                         "block_num": {type: 'number'},
                         "block_id": {type: 'string'},
                         "data": {

api/routes/v2-history/get_schedule/get_schedule.ts

@@ -1,71 +1,73 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {createHash} from "crypto";
 import {base58ToBinary, binaryToBase58} from "eosjs/dist/eosjs-numeric";
 import {Search} from "@elastic/elasticsearch/api/requestParams";
 
 function convertToLegacyKey(block_signing_key: string) {
-    if (block_signing_key.startsWith("PUB_K1_")) {
-        const buf = base58ToBinary(37, block_signing_key.substr(7));
-        const data = buf.slice(0, buf.length - 4);
-        const merged = Buffer.concat([
-            data,
-            createHash('ripemd160')
-                .update(data)
-                .digest()
-                .slice(0, 4)
-        ]);
-        return "EOS" + binaryToBase58(merged);
-    } else {
-        return block_signing_key;
-    }
+	if (block_signing_key.startsWith("PUB_K1_")) {
+		const buf = base58ToBinary(37, block_signing_key.substr(7));
+		const data = buf.slice(0, buf.length - 4);
+		const merged = Buffer.concat([
+			data,
+			createHash('ripemd160')
+				.update(data)
+				.digest()
+				.slice(0, 4)
+		]);
+		return "EOS" + binaryToBase58(merged);
+	} else {
+		return block_signing_key;
+	}
 }
 
 async function getSchedule(fastify: FastifyInstance, request: FastifyRequest) {
-    const response: any = {
-        producers: []
-    };
-    const searchParams: Search<any> = {
-        track_total_hits: true,
-        index: fastify.manager.chain + "-block-*",
-        size: 1,
-        body: {
-            query: {
-                bool: {
-                    must: []
-                }
-            },
-            sort: {block_num: "desc"}
-        }
-    };
-    if (request.query.version) {
-        searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": request.query.version}}});
-    } else {
-        searchParams.body.query.bool.must.push({
-            "exists": {
-                "field": "new_producers.version"
-            }
-        });
-    }
-    const apiResponse = await fastify.elastic.search(searchParams);
-    const results = apiResponse.body.hits.hits;
-    if (results) {
-        response.timestamp = results[0]._source["@timestamp"];
-        response.block_num = results[0]._source.block_num;
-        response.version = results[0]._source.new_producers.version;
-        response.producers = results[0]._source.new_producers.producers.map(prod => {
-            return {
-                ...prod,
-                legacy_key: convertToLegacyKey(prod.block_signing_key)
-            }
-        });
-    }
-    return response;
+
+	const query: any = request.query;
+
+	const response: any = {
+		producers: []
+	};
+	const searchParams: Search<any> = {
+		track_total_hits: true,
+		index: fastify.manager.chain + "-block-*",
+		size: 1,
+		body: {
+			query: {
+				bool: {
+					must: []
+				}
+			},
+			sort: {block_num: "desc"}
+		}
+	};
+	if (query.version) {
+		searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": query.version}}});
+	} else {
+		searchParams.body.query.bool.must.push({
+			"exists": {
+				"field": "new_producers.version"
+			}
+		});
+	}
+	const apiResponse = await fastify.elastic.search(searchParams);
+	const results = apiResponse.body.hits.hits;
+	if (results) {
+		response.timestamp = results[0]._source["@timestamp"];
+		response.block_num = results[0]._source.block_num;
+		response.version = results[0]._source.new_producers.version;
+		response.producers = results[0]._source.new_producers.producers.map(prod => {
+			return {
+				...prod,
+				legacy_key: convertToLegacyKey(prod.block_signing_key)
+			}
+		});
+	}
+	return response;
 }
 
 export function getScheduleHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getSchedule, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getSchedule, fastify, request, route));
+	}
 }

api/routes/v2-history/get_table_state/get_table_state.ts

@@ -0,0 +1,88 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {mergeDeltaMeta, timedQuery} from "../../../helpers/functions";
+
+async function getTableState(fastify: FastifyInstance, request: FastifyRequest) {
+	const query: any = request.query;
+	const response = {
+		...query,
+		results: [],
+		next_key: null
+	};
+	const mustArray: any[] = [
+		{"term": {"code": query.code}},
+		{"term": {"table": query.table}}
+	];
+	if (query.block_num) {
+		mustArray.push({"range": {"block_num": {"lte": query.block_num}}});
+	}
+	let after_key = "";
+	if (query.after_key) {
+		after_key = query.after_key;
+	}
+	const results = await fastify.elastic.search({
+		index: fastify.manager.chain + '-delta-*',
+		body: {
+			"query": {
+				"bool": {
+					"must": mustArray
+				}
+			},
+			"size": 0,
+			"sort": [
+				{"block_num": "desc"},
+				{"primary_key": "desc"}
+			],
+			"aggs": {
+				"scope_buckets": {
+					"composite": {
+						"size": 25,
+						"after": {
+							"scope_pk": after_key
+						},
+						"sources": [
+							{
+								"scope_pk": {
+									"terms": {
+										"script": {
+											"source": "return doc['scope'].value + '-' + doc['primary_key'].value",
+											"lang": "painless"
+										}
+									}
+								}
+							}
+						]
+					},
+					"aggs": {
+						"last_doc": {
+							"top_hits": {
+								"size": 1,
+								"sort": [{"block_num": "desc"}],
+								"_source": {
+									"excludes": ["code"]
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	});
+	const scope_buckets = results.body.aggregations.scope_buckets;
+	if (scope_buckets && scope_buckets.buckets.length > 0) {
+		if (scope_buckets.after_key) {
+			response.next_key = scope_buckets.after_key.scope_pk;
+		}
+		response.results = scope_buckets.buckets.map(bucket => {
+			const row = mergeDeltaMeta(bucket.last_doc.hits.hits[0]._source);
+			delete row.table;
+			return row;
+		});
+	}
+	return response;
+}
+
+export function getTableStateHandler(fastify: FastifyInstance, route: string) {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getTableState, fastify, request, route));
+	}
+}

api/routes/v2-history/get_table_state/index.ts

@@ -0,0 +1,69 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getTableStateHandler} from "./get_table_state";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get table state at a specific block height',
+        summary: 'get table state at a specific block height',
+        tags: ['history'],
+        querystring: {
+            type: 'object',
+            required: ["code","table"],
+            properties: {
+                "code": {
+                    description: 'search by contract',
+                    type: 'string'
+                },
+                "table": {
+                    description: 'search by key',
+                    type: 'string'
+                },
+                "block_num": {
+                    description: 'target block',
+                    type: 'integer',
+                    minimum: 1
+                },
+                "after_key": {
+                    description: 'last key for pagination',
+                    type: 'string'
+                }
+            }
+        },
+        response: extendResponseSchema({
+            "code": {type: 'string'},
+            "table": {type: 'string'},
+            "block_num": {type: 'number'},
+            "after_key": {type: 'string'},
+            "next_key": {type: 'string'},
+            "results": {
+                type: "array",
+                items: {
+                    type: 'object',
+                    properties: {
+                        "scope": {type: 'string'},
+                        "primary_key": {type: 'string'},
+                        "payer": {type: 'string'},
+                        "timestamp": {type: 'string'},
+                        "block_num": {type: 'number'},
+                        "block_id": {type: 'string'},
+                        "present": {type: 'number'},
+                        "data": {
+                            type: 'object',
+                            additionalProperties: true
+                        }
+                    },
+                    additionalProperties: true
+                }
+            }
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getTableStateHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-history/get_transaction/get_transaction.ts

@@ -1,90 +1,128 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
+import {GetInfoResult} from "eosjs/dist/eosjs-rpc-interfaces";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
-    const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
+	const redis = fastify.redis;
+	const query: any = request.query;
+	const trxId = query.id.toLowerCase();
+	const conf = fastify.manager.config;
+	const cachedData = await redis.hgetall('trx_' + trxId);
+	const response = {
+		query_time_ms: undefined,
+		executed: false,
+		cached: undefined,
+		cache_expires_in: undefined,
+		trx_id: query.id,
+		lib: undefined,
+		cached_lib: false,
+		actions: undefined,
+		generated: undefined,
+		error: undefined
+	};
+	let hits;
 
-    let indexPattern = fastify.manager.chain + '-action-*';
-    if (request.query.hot_only) {
-        indexPattern = fastify.manager.chain + '-action';
-    }
+	// build get_info request with caching
+	const $getInfo = new Promise<GetInfoResult>(resolve => {
+		const key = `${fastify.manager.chain}_get_info`;
+		fastify.redis.get(key).then(value => {
+			if (value) {
+				response.cached_lib = true;
+				resolve(JSON.parse(value));
+			} else {
+				fastify.eosjs.rpc.get_info().then(value1 => {
+					fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
+					response.cached_lib = false;
+					resolve(value1);
+				}).catch((reason) => {
+					console.log(reason);
+					response.error = 'failed to get last_irreversible_block_num'
+					resolve(null);
+				});
+			}
+		});
+	});
 
-    const pResults = await Promise.all([
-        fastify.eosjs.rpc.get_info(),
-        fastify.elastic.search({
-            index: indexPattern,
-            size: _size,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {term: {trx_id: request.query.id.toLowerCase()}}
-                        ]
-                    }
-                },
-                sort: {
-                    global_sequence: "asc"
-                }
-            }
-        }),
-        fastify.elastic.search({
-            index: fastify.manager.chain + '-gentrx-*',
-            size: _size,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {term: {trx_id: request.query.id.toLowerCase()}}
-                        ]
-                    }
-                }
-            }
-        })
-    ]);
+	// reconstruct hits from cached data
+	if (cachedData && Object.keys(cachedData).length > 0) {
+		const gsArr = [];
+		for (let cachedDataKey in cachedData) {
+			gsArr.push(cachedData[cachedDataKey]);
+		}
+		gsArr.sort((a, b) => {
+			return a.global_sequence - b.global_sequence;
+		});
+		hits = gsArr.map(value => {
+			return {
+				_source: JSON.parse(value)
+			};
+		});
+		const promiseResults = await Promise.all([
+			redis.ttl('trx_' + trxId),
+			$getInfo
+		]);
+		response.cache_expires_in = promiseResults[0];
+		response.lib = promiseResults[1].last_irreversible_block_num;
+	}
 
-    const results = pResults[1];
-    const genTrxRes = pResults[2];
+	// search on ES if cache is not present
+	if (!hits) {
+		const _size = conf.api.limits.get_trx_actions || 100;
+		const blockHint = parseInt(query.block_hint, 10);
+		let indexPattern = '';
+		if (blockHint) {
+			const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
+			indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
+		} else {
+			indexPattern = fastify.manager.chain + '-action-*';
+		}
+		let pResults;
+		try {
 
-    const response = {
-        "executed": false,
-        "hot_only": false,
-        "trx_id": request.query.id,
-        "lib": pResults[0].last_irreversible_block_num,
-        "actions": [],
-        "generated": undefined
-    };
+			// build search request
+			const $search = fastify.elastic.search({
+				index: indexPattern,
+				size: _size,
+				body: {
+					query: {bool: {must: [{term: {trx_id: trxId}}]}},
+					sort: {global_sequence: "asc"}
+				}
+			});
 
-    if (request.query.hot_only) {
-        response.hot_only = true;
-    }
+			// execute in parallel
+			pResults = await Promise.all([$getInfo, $search]);
+		} catch (e) {
+			console.log(e.message);
+			if (e.meta.statusCode === 404) {
+				response.error = 'no data near block_hint'
+				return response;
+			}
+		}
+		hits = pResults[1]['body']['hits']['hits'];
+		response.lib = pResults[0].last_irreversible_block_num;
+	}
 
-    const hits = results['body']['hits']['hits'];
-
-    if (hits.length > 0) {
-        for (let action of hits) {
-            action = action._source;
-            mergeActionMeta(action);
-            response.actions.push(action);
-        }
-        response.executed = true;
-    }
-
-    const hits2 = genTrxRes['body']['hits']['hits'];
-
-    if (hits2 && hits2.length > 0) {
-        if (hits2[0]._source['@timestamp']) {
-            hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
-            delete hits2[0]._source['@timestamp'];
-        }
-        response.generated = hits2[0]._source;
-    }
-
-    return response;
+	if (hits.length > 0) {
+		let highestBlockNum = 0;
+		for (let action of hits) {
+			if (action._source.block_num > highestBlockNum) {
+				highestBlockNum = action._source.block_num;
+			}
+		}
+		response.actions = [];
+		for (let action of hits) {
+			if (action._source.block_num === highestBlockNum) {
+				mergeActionMeta(action._source);
+				response.actions.push(action._source);
+			}
+		}
+		response.executed = true;
+	}
+	return response;
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getTransaction, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getTransaction, fastify, request, route));
+	}
 }

api/routes/v2-history/get_transaction/index.ts

@@ -10,9 +10,13 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         querystring: {
             type: 'object',
             properties: {
-                "id": {
+                id: {
                     description: 'transaction id',
                     type: 'string'
+                },
+                block_hint: {
+                    description: 'block hint to speed up tx recovery',
+                    type: 'integer'
                 }
             },
             required: ["id"]

api/routes/v2-history/get_transaction_legacy/get_transaction_legacy.ts

@@ -0,0 +1,128 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
+
+async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
+    const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
+    const query: any = request.query;
+
+    let indexPattern = fastify.manager.chain + '-action-*';
+    if (query.hot_only) {
+        indexPattern = fastify.manager.chain + '-action';
+    }
+
+    const pResults = await Promise.all([
+        fastify.eosjs.rpc.get_info(),
+        fastify.elastic.search({
+            index: indexPattern,
+            size: _size,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {term: {trx_id: query.id.toLowerCase()}}
+                        ]
+                    }
+                },
+                sort: {
+                    global_sequence: "asc"
+                }
+            }
+        }),
+        fastify.elastic.search({
+            index: fastify.manager.chain + '-gentrx-*',
+            size: _size,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {term: {trx_id: query.id.toLowerCase()}}
+                        ]
+                    }
+                }
+            }
+        })
+    ]);
+
+    const results = pResults[1];
+    const genTrxRes = pResults[2];
+
+    const response = {
+        "executed": false,
+        "hot_only": false,
+        "trx_id": query.id,
+        "lib": pResults[0].last_irreversible_block_num,
+        "actions": [],
+        "generated": undefined
+    };
+
+    if (query.hot_only) {
+        response.hot_only = true;
+    }
+
+    const hits = results['body']['hits']['hits'];
+
+    if (hits.length > 0) {
+
+        // const producers = {};
+        // for (let hit of hits) {
+        // 	if (hit._source.producer) {
+        // 		if (producers[hit._source.producer]) {
+        // 			producers[hit._source.producer]++;
+        // 		} else {
+        // 			producers[hit._source.producer] = 1;
+        // 		}
+        // 	}
+        // }
+        // let useBlockNumber;
+        // if (Object.keys(producers).length > 1) {
+        // 	// multiple producers of the same tx id, forked actions are present, attempt to clean-up
+        // 	let trueProd = '';
+        // 	let highestActCount = 0;
+        // 	for (const prod in producers) {
+        // 		if (producers.hasOwnProperty(prod)) {
+        // 			if(producers[prod] === highestActCount) {
+        // 				useBlockNumber = true;
+        // 			} else if (producers[prod] > highestActCount) {
+        // 				highestActCount = producers[prod];
+        // 				trueProd = prod;
+        // 			}
+        // 		}
+        // 	}
+        // }
+
+        let highestBlockNum = 0;
+        for (let action of hits) {
+            if (action._source.block_num > highestBlockNum) {
+                highestBlockNum = action._source.block_num;
+            }
+        }
+
+        for (let action of hits) {
+            if (action._source.block_num === highestBlockNum) {
+                action = action._source;
+                mergeActionMeta(action);
+                response.actions.push(action);
+            }
+        }
+
+        response.executed = true;
+    }
+
+    const hits2 = genTrxRes['body']['hits']['hits'];
+
+    if (hits2 && hits2.length > 0) {
+        if (hits2[0]._source['@timestamp']) {
+            hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
+            delete hits2[0]._source['@timestamp'];
+        }
+        response.generated = hits2[0]._source;
+    }
+
+    return response;
+}
+
+export function getTransactionHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getTransaction, fastify, request, route));
+    }
+}

api/routes/v2-history/get_transaction_legacy/index.ts

@@ -0,0 +1,29 @@
+import {FastifyInstance} from "fastify";
+import {getTransactionHandler} from "./get_transaction_legacy";
+import {addApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get all actions belonging to the same transaction',
+        summary: 'get transaction by id',
+        tags: ['history'],
+        querystring: {
+            type: 'object',
+            properties: {
+                "id": {
+                    description: 'transaction id',
+                    type: 'string'
+                }
+            },
+            required: ["id"]
+        }
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getTransactionHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-state/get_account/get_account.ts

@@ -1,10 +1,12 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import got from "got";
 import {timedQuery} from "../../../helpers/functions";
 
 async function getAccount(fastify: FastifyInstance, request: FastifyRequest) {
 
+
+    const query: any = request.query;
+
     const response = {
         account: null,
         actions: null,
@@ -13,7 +15,7 @@ async function getAccount(fastify: FastifyInstance, request: FastifyRequest) {
         links: null
     };
 
-    const account = request.query.account;
+    const account = query.account;
     const reqQueue = [];
 
     try {
@@ -45,7 +47,7 @@ async function getAccount(fastify: FastifyInstance, request: FastifyRequest) {
 }
 
 export function getAccountHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getAccount, fastify, request, route));
     }
 }

api/routes/v2-state/get_key_accounts/get_key_accounts.ts

@@ -1,137 +1,139 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {Numeric} from "eosjs/dist";
 import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 function invalidKey() {
-    const err: any = new Error();
-    err.statusCode = 400;
-    err.message = 'invalid public key';
-    throw err;
+	const err: any = new Error();
+	err.statusCode = 400;
+	err.message = 'invalid public key';
+	throw err;
 }
 
 async function getKeyAccounts(fastify: FastifyInstance, request: FastifyRequest) {
 
-    let publicKey;
-    if (typeof request.body === 'string' && request.req.method === 'POST') {
-        request.body = JSON.parse(request.body);
-    }
+	let publicKey;
+	if (typeof request.body === 'string' && request.method === 'POST') {
+		request.body = JSON.parse(request.body);
+	}
 
-    const {skip, limit} = getSkipLimit(request.query);
-    const maxDocs = fastify.manager.config.api.limits.get_key_accounts ?? 1000;
+	const body: any = request.body;
+	const query: any = request.query;
 
-    const public_Key = request.req.method === 'POST' ? request.body.public_key : request.query.public_key;
+	const {skip, limit} = getSkipLimit(request.query);
+	const maxDocs = fastify.manager.config.api.limits.get_key_accounts ?? 1000;
 
-    if (public_Key.startsWith("PUB_")) {
-        publicKey = public_Key;
-    } else if (public_Key.startsWith("EOS")) {
-        try {
-            publicKey = Numeric.convertLegacyPublicKey(public_Key);
-        } catch (e) {
-            console.log(e.message);
-            invalidKey();
-        }
-    } else {
-        invalidKey();
-    }
+	const public_Key = request.method === 'POST' ? body.public_key : query.public_key;
 
-    const response = {
-        account_names: []
-    } as any;
+	if (public_Key.startsWith("PUB_")) {
+		publicKey = public_Key;
+	} else if (public_Key.startsWith("EOS")) {
+		try {
+			publicKey = Numeric.convertLegacyPublicKey(public_Key);
+		} catch (e) {
+			console.log(e.message);
+			invalidKey();
+		}
+	} else {
+		invalidKey();
+	}
 
-    if (request.req.method === 'GET' && request.query.details) {
-        response.permissions = [];
-    }
+	const response = {
+		account_names: []
+	} as any;
 
-    try {
+	if (request.method === 'GET' && query.details) {
+		response.permissions = [];
+	}
 
-        const permTableResults = await fastify.elastic.search({
-            index: fastify.manager.chain + '-perm-*',
-            size: (limit > maxDocs ? maxDocs : limit) || 100,
-            from: skip || 0,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                term: {
-                                    "auth.keys.key.keyword": publicKey
-                                }
-                            }
-                        ],
-                    }
-                }
-            }
-        });
+	try {
 
-        if (permTableResults.body.hits.hits.length > 0) {
-            for (const perm of permTableResults.body.hits.hits) {
-                response.account_names.push(perm._source.owner);
-                if (request.req.method === 'GET' && request.query.details) {
-                    response.permissions.push(perm._source);
-                }
-            }
-        }
+		const permTableResults = await fastify.elastic.search({
+			index: fastify.manager.chain + '-perm-*',
+			size: (limit > maxDocs ? maxDocs : limit) || 100,
+			from: skip || 0,
+			body: {
+				query: {
+					bool: {
+						must: [
+							{
+								term: {
+									"auth.keys.key.keyword": publicKey
+								}
+							}
+						],
+					}
+				}
+			}
+		});
 
-    } catch (e) {
-        console.log(e);
-    }
+		if (permTableResults.body.hits.hits.length > 0) {
+			for (const perm of permTableResults.body.hits.hits) {
+				response.account_names.push(perm._source.owner);
+				if (request.method === 'GET' && query.details) {
+					response.permissions.push(perm._source);
+				}
+			}
+		}
 
-    if (response.account_names.length > 0) {
-        response.account_names = [...(new Set(response.account_names))];
-        return response;
-    }
+	} catch (e) {
+		console.log(e);
+	}
 
-    // Fallback to action search
-    const _body = {
-        query: {
-            bool: {
-                should: [
-                    {term: {"@updateauth.auth.keys.key.keyword": publicKey}},
-                    {term: {"@newaccount.active.keys.key.keyword": publicKey}},
-                    {term: {"@newaccount.owner.keys.key.keyword": publicKey}}
-                ],
-                minimum_should_match: 1
-            }
-        },
-        sort: [{"global_sequence": {"order": "desc"}}]
-    };
+	if (response.account_names.length > 0) {
+		response.account_names = [...(new Set(response.account_names))];
+		return response;
+	}
 
-    const results = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        size: (limit > maxDocs ? maxDocs : limit) || 100,
-        from: skip || 0,
-        body: _body
-    });
+	// Fallback to action search
+	const _body = {
+		query: {
+			bool: {
+				should: [
+					{term: {"@updateauth.auth.keys.key.keyword": publicKey}},
+					{term: {"@newaccount.active.keys.key.keyword": publicKey}},
+					{term: {"@newaccount.owner.keys.key.keyword": publicKey}}
+				],
+				minimum_should_match: 1
+			}
+		},
+		sort: [{"global_sequence": {"order": "desc"}}]
+	};
 
-    if (results['body']['hits']['hits'].length > 0) {
-        response.account_names = results['body']['hits']['hits'].map((v) => {
-            if (v._source.act.name === 'newaccount') {
-                if (v._source['@newaccount'].newact) {
-                    return v._source['@newaccount'].newact;
-                } else if (v._source.act.data.newact) {
-                    return v._source.act.data.newact;
-                } else {
-                    return null;
-                }
-            } else if (v._source.act.name === 'updateauth') {
-                return v._source.act.data.account;
-            } else {
-                return null;
-            }
-        });
-    }
+	const results = await fastify.elastic.search({
+		index: fastify.manager.chain + '-action-*',
+		size: (limit > maxDocs ? maxDocs : limit) || 100,
+		from: skip || 0,
+		body: _body
+	});
 
-    if (response.account_names.length > 0) {
-        response.account_names = [...(new Set(response.account_names))];
-    }
+	if (results['body']['hits']['hits'].length > 0) {
+		response.account_names = results['body']['hits']['hits'].map((v) => {
+			if (v._source.act.name === 'newaccount') {
+				if (v._source['@newaccount'].newact) {
+					return v._source['@newaccount'].newact;
+				} else if (v._source.act.data.newact) {
+					return v._source.act.data.newact;
+				} else {
+					return null;
+				}
+			} else if (v._source.act.name === 'updateauth') {
+				return v._source.act.data.account;
+			} else {
+				return null;
+			}
+		});
+	}
 
-    return response;
+	if (response.account_names.length > 0) {
+		response.account_names = [...(new Set(response.account_names))];
+	}
+
+	return response;
 }
 
 export function getKeyAccountsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getKeyAccounts, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getKeyAccounts, fastify, request, route));
+	}
 }

api/routes/v2-state/get_key_accounts/index.ts

@@ -38,7 +38,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                                 last_updated: {type: 'string'},
                                 auth: {},
                                 name: {type: 'string'},
-                                present: {type: 'boolean'}
+                                present: {type: 'number'}
                             }
                         }
                     }

api/routes/v2-state/get_links/get_links.ts

@@ -1,71 +1,71 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {getTrackTotalHits, timedQuery} from "../../../helpers/functions";
 import {ApiResponse} from "@elastic/elasticsearch";
 import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 async function getLinks(fastify: FastifyInstance, request: FastifyRequest) {
-    const query = request.query;
-    const {account, code, action, permissions} = query;
-    const {skip, limit} = getSkipLimit(query);
+	const query: any = request.query;
+	const {account, code, action, permissions} = query;
+	const {skip, limit} = getSkipLimit(query);
 
-    const queryStruct = {
-        "bool": {
-            must: []
-        }
-    };
+	const queryStruct = {
+		"bool": {
+			must: [],
+			must_not: []
+		}
+	};
 
-    if (account) {
-        queryStruct.bool.must.push({'term': {'account': account}});
-    }
+	if (account) {
+		queryStruct.bool.must.push({'term': {'account': account}});
+	}
 
-    if (code) {
-        queryStruct.bool.must.push({'term': {'code': code}});
-    }
+	if (code) {
+		queryStruct.bool.must.push({'term': {'code': code}});
+	}
 
-    if (action) {
-        queryStruct.bool.must.push({'term': {'action': action}});
-    }
+	if (action) {
+		queryStruct.bool.must.push({'term': {'action': action}});
+	}
 
-    if (permissions) {
-        queryStruct.bool.must.push({'term': {'permissions': permissions}});
-    }
+	if (permissions) {
+		queryStruct.bool.must.push({'term': {'permissions': permissions}});
+	}
 
-    // only present deltas
-    queryStruct.bool.must.push({'term': {'present': true}});
+	// only present deltas
+	queryStruct.bool.must_not.push({'term': {'present': 0}});
 
-    // Prepare query body
-    const query_body = {
-        track_total_hits: getTrackTotalHits(query),
-        query: queryStruct,
-        sort: {block_num: 'desc'}
-    };
+	// Prepare query body
+	const query_body = {
+		track_total_hits: getTrackTotalHits(query),
+		query: queryStruct,
+		sort: {block_num: 'desc'}
+	};
 
-    const maxLinks = fastify.manager.config.api.limits.get_links;
-    const results: ApiResponse = await fastify.elastic.search({
-        index: fastify.manager.chain + '-link-*',
-        from: skip || 0,
-        size: (limit > maxLinks ? maxLinks : limit) || 50,
-        body: query_body
-    });
-    const hits = results.body.hits.hits;
-    const response: any = {
-        cached: false,
-        total: results.body.hits.total,
-        links: []
-    };
+	const maxLinks = fastify.manager.config.api.limits.get_links;
+	const results: ApiResponse = await fastify.elastic.search({
+		index: fastify.manager.chain + '-link-*',
+		from: skip || 0,
+		size: (limit > maxLinks ? maxLinks : limit) || 50,
+		body: query_body
+	});
+	const hits = results.body.hits.hits;
+	const response: any = {
+		cached: false,
+		total: results.body.hits.total,
+		links: []
+	};
 
-    for (const hit of hits) {
-        const link = hit._source;
-        link.timestamp = link['@timestamp'];
-        delete link['@timestamp'];
-        response.links.push(link);
-    }
-    return response;
+	for (const hit of hits) {
+		const link = hit._source;
+		link.timestamp = link['@timestamp'];
+		delete link['@timestamp'];
+		response.links.push(link);
+	}
+	return response;
 }
 
 export function getLinksHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getLinks, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getLinks, fastify, request, route));
+	}
 }

api/routes/v2-state/get_proposals/get_proposals.ts

@@ -1,16 +1,17 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {getTrackTotalHits, timedQuery} from "../../../helpers/functions";
 
 async function getProposals(fastify: FastifyInstance, request: FastifyRequest) {
 
+    const query: any = request.query;
+
     // Pagination
     let skip, limit;
-    skip = parseInt(request.query.skip, 10);
+    skip = parseInt(query.skip, 10);
     if (skip < 0) {
         return 'invalid skip parameter';
     }
-    limit = parseInt(request.query.limit, 10);
+    limit = parseInt(query.limit, 10);
     if (limit < 1) {
         return 'invalid limit parameter';
     }
@@ -22,8 +23,8 @@ async function getProposals(fastify: FastifyInstance, request: FastifyRequest) {
     };
 
     // Filter by accounts
-    if (request.query.account) {
-        const accounts = request.query.account.split(',');
+    if (query.account) {
+        const accounts = query.account.split(',');
         for(const acc of accounts) {
             queryStruct.bool.must.push({
                 "bool": {
@@ -37,28 +38,28 @@ async function getProposals(fastify: FastifyInstance, request: FastifyRequest) {
     }
 
     // Filter by proposer account
-    if (request.query.proposer) {
-        queryStruct.bool.must.push({"term": {"proposer": request.query.proposer}});
+    if (query.proposer) {
+        queryStruct.bool.must.push({"term": {"proposer": query.proposer}});
     }
 
     // Filter by proposal name
-    if (request.query.proposal) {
-        queryStruct.bool.must.push({"term": {"proposal_name": request.query.proposal}});
+    if (query.proposal) {
+        queryStruct.bool.must.push({"term": {"proposal_name": query.proposal}});
     }
 
     // Filter by execution status
-    if (typeof request.query.executed !== 'undefined') {
-        queryStruct.bool.must.push({"term": {"executed": request.query.executed}});
+    if (typeof query.executed !== 'undefined') {
+        queryStruct.bool.must.push({"term": {"executed": query.executed}});
     }
 
     // Filter by requested actors
-    if (request.query.requested) {
-        queryStruct.bool.must.push({"term": {"requested_approvals.actor": request.query.requested}});
+    if (query.requested) {
+        queryStruct.bool.must.push({"term": {"requested_approvals.actor": query.requested}});
     }
 
     // Filter by provided actors
-    if (request.query.provided) {
-        queryStruct.bool.must.push({"term": {"provided_approvals.actor": request.query.provided}});
+    if (query.provided) {
+        queryStruct.bool.must.push({"term": {"provided_approvals.actor": query.provided}});
     }
 
     // If no filter switch to full match
@@ -97,7 +98,7 @@ async function getProposals(fastify: FastifyInstance, request: FastifyRequest) {
 }
 
 export function getProposalsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getProposals, fastify, request, route));
     }
 }

api/routes/v2-state/get_tokens/get_tokens.ts

@@ -1,4 +1,3 @@
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {getSkipLimit} from "../../v2-history/get_actions/functions";
@@ -6,7 +5,12 @@ import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
 
-    const response = {'account': request.query.account, 'tokens': []};
+    const query: any = request.query;
+
+    const response = {
+        account: query.account,
+        tokens: []
+    };
 
     const {skip, limit} = getSkipLimit(request.query);
     const maxDocs = fastify.manager.config.api.limits.get_tokens ?? 100;
@@ -18,55 +22,69 @@ async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
         "body": {
             query: {
                 bool: {
-                    filter: [{term: {"scope": request.query.account}}]
+                    filter: [{term: {scope: query.account}}]
                 }
             }
         }
     });
 
+    const testSet = new Set();
     for (const hit of stateResult.body.hits.hits) {
         const data = hit._source;
-        if (typeof data.present !== "undefined" && data.present === false) {
+        if (typeof data.present !== "undefined" && data.present === 0) {
             continue;
         }
         let precision;
+        let token_data;
+        let errorMsg;
+
         const key = `${data.code}_${data.symbol}`;
+
+        if (testSet.has(key)) {
+            continue;
+        }
+
+        testSet.add(key);
         if (!fastify.tokenCache) {
             fastify.tokenCache = new Map<string, any>();
         }
         if (fastify.tokenCache.has(key)) {
             precision = fastify.tokenCache.get(key).precision;
         } else {
-            let token_data;
             try {
-                token_data = await fastify.eosjs.rpc.get_currency_balance(data.code, request.query.account, data.symbol);
+                token_data = await fastify.eosjs.rpc.get_currency_balance(data.code, query.account, data.symbol);
                 if (token_data.length > 0) {
                     const [amount, symbol] = token_data[0].split(" ");
                     const amount_arr = amount.split(".");
                     if (amount_arr.length === 2) {
                         precision = amount_arr[1].length;
                         fastify.tokenCache.set(key, {precision});
-                        // console.log('Caching token precision -', key, precision);
                     }
                 }
             } catch (e) {
-                console.log(`get_currency_balance error - contract:${data.code} - account:${request.query.account}`);
+                errorMsg = e.message;
             }
         }
 
-        response.tokens.push({
+        const resp: Record<string, any> = {
             symbol: data.symbol,
             precision: precision,
             amount: parseFloat(data.amount),
-            contract: data.code
-        });
+            contract: data.code,
+        };
+
+        if (errorMsg) {
+            resp.error = errorMsg;
+        }
+
+        response.tokens.push(resp);
     }
 
     return response;
 }
 
 export function getTokensHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getTokens, fastify, request, route));
     }
 }

api/routes/v2-state/get_voters/get_voters.ts

@@ -1,63 +1,63 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 async function getVoters(fastify: FastifyInstance, request: FastifyRequest) {
 
-    const {skip, limit} = getSkipLimit(request.query);
+	const query: any = request.query;
+	const {skip, limit} = getSkipLimit(request.query);
 
-    const response = {
-        voter_count: 0,
-        'voters': []
-    };
-    let queryStruct: any = {
-        "bool": {
-            "must": []
-        }
-    };
+	const response = {
+		voter_count: 0,
+		'voters': []
+	};
+	let queryStruct: any = {
+		"bool": {
+			"must": []
+		}
+	};
 
-    if (request.query.producer) {
-        for (const bp of request.query.producer.split(",")) {
-            queryStruct.bool.must.push({"term": {"producers": bp}});
-        }
-    }
+	if (query.producer) {
+		for (const bp of query.producer.split(",")) {
+			queryStruct.bool.must.push({"term": {"producers": bp}});
+		}
+	}
 
-    if (request.query.proxy === 'true') {
-        queryStruct.bool.must.push({"term": {"is_proxy": true}});
-    }
+	if (query.proxy === 'true') {
+		queryStruct.bool.must.push({"term": {"is_proxy": true}});
+	}
 
-    if (queryStruct.bool.must.length === 0) {
-        queryStruct = {
-            "match_all": {}
-        };
-    }
+	if (queryStruct.bool.must.length === 0) {
+		queryStruct = {
+			"match_all": {}
+		};
+	}
 
-    const maxDocs = fastify.manager.config.api.limits.get_voters ?? 100;
-    const results = await fastify.elastic.search({
-        "index": fastify.manager.chain + '-table-voters-*',
-        "from": skip || 0,
-        "size": (limit > maxDocs ? maxDocs : limit) || 10,
-        "body": {
-            "query": queryStruct,
-            "sort": [{"last_vote_weight": "desc"}]
-        }
-    });
-    const hits = results['body']['hits']['hits'];
-    for (const hit of hits) {
-        const voter = hit._source;
-        response.voters.push({
-            account: voter.voter,
-            weight: voter.last_vote_weight,
-            last_vote: voter.block_num
-        });
-    }
-    response.voter_count = results['body']['hits']['total']['value'];
-    return response;
+	const maxDocs = fastify.manager.config.api.limits.get_voters ?? 100;
+	const results = await fastify.elastic.search({
+		"index": fastify.manager.chain + '-table-voters-*',
+		"from": skip || 0,
+		"size": (limit > maxDocs ? maxDocs : limit) || 10,
+		"body": {
+			"query": queryStruct,
+			"sort": [{"last_vote_weight": "desc"}]
+		}
+	});
+	const hits = results['body']['hits']['hits'];
+	for (const hit of hits) {
+		const voter = hit._source;
+		response.voters.push({
+			account: voter.voter,
+			weight: voter.last_vote_weight,
+			last_vote: voter.block_num
+		});
+	}
+	response.voter_count = results['body']['hits']['total']['value'];
+	return response;
 }
 
 export function getVotersHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getVoters, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getVoters, fastify, request, route));
+	}
 }

api/routes/v2-stats/get_action_usage/get_action_usage.ts

@@ -1,121 +1,120 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 
 async function getLastSeq(fastify: FastifyInstance, date: string) {
-    const req = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        body: {
-            "size": 1,
-            "query": {
-                "bool": {
-                    "must": [
-                        {"range": {"@timestamp": {"lt": date}}}
-                    ]
-                }
-            },
-            "_source": {
-                "includes": ["global_sequence"]
-            },
-            "sort": [{"global_sequence": "desc"}]
-        }
-    });
-    return req.body.hits.hits[0]._source.global_sequence;
+	const req = await fastify.elastic.search({
+		index: fastify.manager.chain + '-action-*',
+		body: {
+			"size": 1,
+			"query": {
+				"bool": {
+					"must": [
+						{"range": {"@timestamp": {"lt": date}}}
+					]
+				}
+			},
+			"_source": {
+				"includes": ["global_sequence"]
+			},
+			"sort": [{"global_sequence": "desc"}]
+		}
+	});
+	return req.body.hits.hits[0]._source.global_sequence;
 }
 
 async function getTxCount(fastify: FastifyInstance, dateFrom: string, dateTo: string) {
-    const req = await fastify.elastic.count({
-        index: fastify.manager.chain + '-action-*',
-        body: {
-            "query": {
-                "bool": {
-                    "must": [
-                        {"term": {"creator_action_ordinal": 0}},
-                        {"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}
-                    ]
-                }
-            }
-        }
-    });
-    return req.body.count;
+	const req = await fastify.elastic.count({
+		index: fastify.manager.chain + '-action-*',
+		body: {
+			"query": {
+				"bool": {
+					"must": [
+						{"term": {"creator_action_ordinal": 0}},
+						{"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}
+					]
+				}
+			}
+		}
+	});
+	return req.body.count;
 }
 
 async function getUniqueActors(fastify: FastifyInstance, dateFrom: string, dateTo: string) {
-    const req = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        size: 0,
-        body: {
-            "aggs": {
-                "unique_actors": {
-                    "cardinality": {
-                        "field": "act.authorization.actor"
-                    }
-                }
-            },
-            "query": {
-                "bool": {
-                    "filter": [{"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}]
-                }
-            }
-        }
-    });
-    return req.body.aggregations.unique_actors.value;
+	const req = await fastify.elastic.search({
+		index: fastify.manager.chain + '-action-*',
+		size: 0,
+		body: {
+			"aggs": {
+				"unique_actors": {
+					"cardinality": {
+						"field": "act.authorization.actor"
+					}
+				}
+			},
+			"query": {
+				"bool": {
+					"filter": [{"range": {"@timestamp": {"gt": dateFrom, "lt": dateTo}}}]
+				}
+			}
+		}
+	});
+	return req.body.aggregations.unique_actors.value;
 }
 
 async function getActionUsage(fastify: FastifyInstance, request: FastifyRequest) {
-    const query = request.query;
-    let now = new Date();
-    let expiration = 86400;
-    if (query.end_date) {
-        now = new Date(query.end_date);
-    }
-    now.setMinutes(0, 0, 0);
-    let period;
-    if (query.period === '1h') {
-        period = 3600000;
-    } else if (query.period === '24h' || query.period === '1d') {
-        period = 86400000;
-    } else {
-        return {};
-    }
+	const query: any = request.query;
+	let now = new Date();
+	let expiration = 86400;
+	if (query.end_date) {
+		now = new Date(query.end_date);
+	}
+	now.setMinutes(0, 0, 0);
+	let period;
+	if (query.period === '1h') {
+		period = 3600000;
+	} else if (query.period === '24h' || query.period === '1d') {
+		period = 86400000;
+	} else {
+		return {};
+	}
 
-    if (!query.end_date) {
-        expiration = Math.floor((period - (Date.now() - now.getTime())) / 1000);
-    }
+	if (!query.end_date) {
+		expiration = Math.floor((period - (Date.now() - now.getTime())) / 1000);
+	}
 
-    const response = {} as any;
-    const periodStart = new Date(now.getTime() - period);
-    const pStart = periodStart.toISOString();
-    const pEnd = now.toISOString();
+	const response = {} as any;
+	const periodStart = new Date(now.getTime() - period);
+	const pStart = periodStart.toISOString();
+	const pEnd = now.toISOString();
 
-    const cacheKey = `${fastify.manager.chain}_act_stats_${pStart}_${pEnd}_${query.unique_actors}`;
-    const cachedValue = await fastify.redis.get(cacheKey);
-    if (cachedValue) {
-        const cachedResponse = JSON.parse(cachedValue);
-        cachedResponse.cached = true;
-        cachedResponse.cache_exp = expiration;
-        return cachedResponse;
-    }
+	const cacheKey = `${fastify.manager.chain}_act_stats_${pStart}_${pEnd}_${query.unique_actors}`;
+	const cachedValue = await fastify.redis.get(cacheKey);
+	if (cachedValue) {
+		const cachedResponse = JSON.parse(cachedValue);
+		cachedResponse.cached = true;
+		cachedResponse.cache_exp = expiration;
+		return cachedResponse;
+	}
 
-    const firstReq = await getLastSeq(fastify, pStart);
-    const lastReq = await getLastSeq(fastify, pEnd);
-    response.action_count = lastReq - firstReq;
-    response.tx_count = await getTxCount(fastify, pStart, pEnd);
+	const firstReq = await getLastSeq(fastify, pStart);
+	const lastReq = await getLastSeq(fastify, pEnd);
+	response.action_count = lastReq - firstReq;
+	response.tx_count = await getTxCount(fastify, pStart, pEnd);
 
-    if (query.unique_actors) {
-        response.unique_actors = await getUniqueActors(fastify, pStart, pEnd);
-    }
+	if (query.unique_actors) {
+		response.unique_actors = await getUniqueActors(fastify, pStart, pEnd);
+	}
 
-    response.period = query.period;
-    response.from = pStart;
-    response.to = pEnd;
+	response.period = query.period;
+	response.from = pStart;
+	response.to = pEnd;
 
-    await fastify.redis.set(cacheKey, JSON.stringify(response), 'EX', expiration);
-    return response;
+	await fastify.redis.set(cacheKey, JSON.stringify(response), 'EX', expiration);
+	return response;
 }
 
 export function getActionUsageHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getActionUsage, fastify, request, route));
-    }
-}
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getActionUsage, fastify, request, route));
+	}
+}

api/routes/v2-stats/get_missed_blocks/get_missed_blocks.ts

@@ -1,64 +1,64 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {timedQuery} from "../../../helpers/functions";
-import {ServerResponse} from "http";
 import {Search} from "@elastic/elasticsearch/api/requestParams";
 import {applyTimeFilter} from "../../v2-history/get_actions/functions";
 
 async function getMissedBlocks(fastify: FastifyInstance, request: FastifyRequest) {
-    const response = {
-        stats: {
-            by_producer: {}
-        },
-        events: []
-    };
-    const searchParams: Search<any> = {
-        track_total_hits: true,
-        index: fastify.manager.chain + "-logs-*",
-        size: 100,
-        body: {
-            query: {
-                bool: {
-                    must: [
-                        {term: {"type": "missed_blocks"}}
-                    ]
-                }
-            },
-            sort: {
-                "@timestamp": "desc"
-            }
-        }
-    };
+	const query: any = request.query;
+	const response = {
+		stats: {
+			by_producer: {}
+		},
+		events: []
+	};
+	const searchParams: Search<any> = {
+		track_total_hits: true,
+		index: fastify.manager.chain + "-logs-*",
+		size: 100,
+		body: {
+			query: {
+				bool: {
+					must: [
+						{term: {"type": "missed_blocks"}}
+					]
+				}
+			},
+			sort: {
+				"@timestamp": "desc"
+			}
+		}
+	};
 
-    let minBlocks = 0;
-    if (request.query.min_blocks) {
-        minBlocks = parseInt(request.query.min_blocks);
-        searchParams.body.query.bool.must.push({range: {"missed_blocks.size": {gte: minBlocks}}})
-    }
+	let minBlocks = 0;
+	if (query.min_blocks) {
+		minBlocks = parseInt(query.min_blocks);
+		searchParams.body.query.bool.must.push({range: {"missed_blocks.size": {gte: minBlocks}}})
+	}
 
-    if (request.query.producer) {
-        searchParams.body.query.bool.must.push({term: {"missed_blocks.producer": request.query.producer}});
-    }
+	if (query.producer) {
+		searchParams.body.query.bool.must.push({term: {"missed_blocks.producer": query.producer}});
+	}
 
-    applyTimeFilter(request.query, searchParams.body.query);
+	applyTimeFilter(query, searchParams.body.query);
 
-    const apiResponse = await fastify.elastic.search(searchParams);
-    apiResponse.body.hits.hits.forEach(v => {
-        const ev = v._source;
-        response.events.push({
-            "@timestamp": ev["@timestamp"],
-            ...ev.missed_blocks
-        });
-        if (!response.stats.by_producer[ev.missed_blocks.producer]) {
-            response.stats.by_producer[ev.missed_blocks.producer] = ev.missed_blocks.size;
-        } else {
-            response.stats.by_producer[ev.missed_blocks.producer] += ev.missed_blocks.size;
-        }
-    });
-    return response;
+	const apiResponse = await fastify.elastic.search(searchParams);
+	apiResponse.body.hits.hits.forEach(v => {
+		const ev = v._source;
+		response.events.push({
+			"@timestamp": ev["@timestamp"],
+			...ev.missed_blocks
+		});
+		if (!response.stats.by_producer[ev.missed_blocks.producer]) {
+			response.stats.by_producer[ev.missed_blocks.producer] = ev.missed_blocks.size;
+		} else {
+			response.stats.by_producer[ev.missed_blocks.producer] += ev.missed_blocks.size;
+		}
+	});
+	return response;
 }
 
 export function getMissedBlocksHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getMissedBlocks, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getMissedBlocks, fastify, request, route));
+	}
 }

api/routes/v2-stats/get_resource_usage/get_resource_usage.ts

@@ -1,16 +1,16 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {timedQuery} from "../../../helpers/functions";
 
 const percentiles = [1, 5, 25, 50, 75, 95, 99];
 
 async function getResourceUsage(fastify: FastifyInstance, request: FastifyRequest) {
+	const query: any = request.query;
 	const searchBody: any = {
 		query: {
 			bool: {
 				must: [
-					{term: {"act.account": request.query.code}},
-					{term: {"act.name": request.query.action}},
+					{term: {"act.account": query.code}},
+					{term: {"act.name": query.action}},
 					{term: {"action_ordinal": {"value": 1}}},
 					{
 						range: {
@@ -69,7 +69,7 @@ async function getResourceUsage(fastify: FastifyInstance, request: FastifyReques
 }
 
 export function getResourceUsageHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+	return async (request: FastifyRequest, reply: FastifyReply) => {
 		reply.send(await timedQuery(getResourceUsage, fastify, request, route));
 	}
 }

api/routes/v2/health/health.ts

@@ -1,10 +1,10 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {ServerResponse} from "http";
 import {connect} from "amqplib";
 import {timedQuery} from "../../../helpers/functions";
-import {getLastIndexedBlockWithTotalBlocks} from "../../../../helpers/common_functions";
+import {getFirstIndexedBlock, getLastIndexedBlockWithTotalBlocks} from "../../../../helpers/common_functions";
 
-async function checkRabbit(fastify: FastifyInstance) {
+
+async function checkRabbit(fastify: FastifyInstance): Promise<ServiceResponse<any>> {
     try {
         const connection = await connect(fastify.manager.ampqUrl);
         await connection.close();
@@ -15,7 +15,15 @@ async function checkRabbit(fastify: FastifyInstance) {
     }
 }
 
-async function checkNodeos(fastify: FastifyInstance) {
+interface NodeosService {
+    head_block_num: number;
+    head_block_time: string;
+    time_offset: number;
+    last_irreversible_block: number;
+    chain_id: string;
+}
+
+async function checkNodeos(fastify: FastifyInstance): Promise<ServiceResponse<NodeosService>> {
     const rpc = fastify.manager.nodeosJsonRPC;
     try {
         const results = await rpc.get_info();
@@ -36,14 +44,47 @@ async function checkNodeos(fastify: FastifyInstance) {
     }
 }
 
-async function checkElastic(fastify: FastifyInstance) {
+interface ESService {
+    first_indexed_block: number;
+    last_indexed_block: number;
+    total_indexed_blocks: number;
+    active_shards: string;
+    missing_blocks: number;
+    missing_pct: string;
+    head_offset: number;
+}
+
+interface ServiceResponse<T> {
+    service: string;
+    time: number;
+    status: any
+    service_data?: T;
+}
+
+async function checkElastic(fastify: FastifyInstance): Promise<ServiceResponse<ESService>> {
     try {
         let esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
+        let firstIndexedBlock: number;
+        const fib = await fastify.redis.get(`${fastify.manager.chain}::fib`);
+        if (fib) {
+            firstIndexedBlock = parseInt(fib);
+        } else {
+            firstIndexedBlock = await getFirstIndexedBlock(fastify.elastic, fastify.manager.chain);
+            await fastify.redis.set(`${fastify.manager.chain}::fib`, firstIndexedBlock);
+        }
         let indexedBlocks = await getLastIndexedBlockWithTotalBlocks(fastify.elastic, fastify.manager.chain);
-        const data = {
-            last_indexed_block: indexedBlocks[0],
-            total_indexed_blocks: indexedBlocks[1],
-            active_shards: esStatus.body[0]['active_shards_percent']
+        const lastIndexedBlock = indexedBlocks[0];
+        const totalIndexed = indexedBlocks[1] - 1;
+        const missingCounter = (lastIndexedBlock - firstIndexedBlock) - totalIndexed;
+        const missingPct = (missingCounter * 100 / indexedBlocks[1]).toFixed(2) + "%";
+        const data: ESService = {
+            active_shards: esStatus.body[0]['active_shards_percent'],
+            head_offset: null,
+            first_indexed_block: firstIndexedBlock,
+            last_indexed_block: lastIndexedBlock,
+            total_indexed_blocks: totalIndexed,
+            missing_blocks: missingCounter,
+            missing_pct: missingPct
         };
         let stat = 'OK';
         esStatus.body.forEach(status => {
@@ -60,7 +101,7 @@ async function checkElastic(fastify: FastifyInstance) {
     }
 }
 
-function createHealth(name: string, status, data?: any) {
+function createHealth(name: string, status, data?: any): ServiceResponse<any> {
     let time = Date.now();
     return {
         service: name,
@@ -70,22 +111,27 @@ function createHealth(name: string, status, data?: any) {
     }
 }
 
-async function getHealthQuery(fastify: FastifyInstance, request: FastifyRequest) {
+async function getHealthQuery(fastify: FastifyInstance) {
     let response = {
         version: fastify.manager.current_version,
         version_hash: fastify.manager.getServerHash(),
         host: fastify.manager.config.api.server_name,
-        features: fastify.manager.config.features,
-        health: []
+        health: [],
+        features: fastify.manager.config.features
     };
-    response.health.push(await checkRabbit(fastify));
-    response.health.push(await checkNodeos(fastify));
-    response.health.push(await checkElastic(fastify));
+    response.health = await Promise.all([
+        checkRabbit(fastify),
+        checkNodeos(fastify),
+        checkElastic(fastify)
+    ]);
+    const es = response.health.find(value => value.service === 'Elasticsearch');
+    const nodeos = response.health.find(value => value.service === 'NodeosRPC');
+    es.service_data.head_offset = nodeos.service_data.head_block_num - es.service_data.last_indexed_block;
     return response;
 }
 
 export function healthHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
         reply.send(await timedQuery(getHealthQuery, fastify, request, route));
     }
 }

api/routes/v2/health/index.ts

@@ -1,18 +1,18 @@
-import {FastifyInstance, RouteSchema} from "fastify";
+import {FastifyInstance, FastifySchema} from "fastify";
 import {addApiRoute, getRouteName} from "../../../helpers/functions";
 import {healthHandler} from "./health";
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: RouteSchema = {
-        tags: ['status'],
-        summary: "API Service Health Report"
-    };
-    addApiRoute(
-        fastify,
-        'GET',
-        getRouteName(__filename),
-        healthHandler,
-        schema
-    );
-    next();
+	const schema: FastifySchema = {
+		tags: ['status'],
+		summary: "API Service Health Report"
+	};
+	addApiRoute(
+		fastify,
+		'GET',
+		getRouteName(__filename),
+		healthHandler,
+		schema
+	);
+	next();
 }

api/schemas/get_abi_snapshot.js

@@ -1,26 +0,0 @@
-exports.GET = {
-    description: 'fetch contract abi at specific block',
-    summary: 'fetch abi at specific block',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "contract": {
-                description: 'contract account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "block": {
-                description: 'target block',
-                type: 'integer',
-                minimum: 1
-            },
-            "fetch": {
-                description: 'should fetch the ABI',
-                type: 'boolean'
-            }
-        },
-        required: ["contract"]
-    }
-};

api/schemas/get_account.js

@@ -1,14 +0,0 @@
-exports.GET = {
-    description: 'get account data',
-    summary: 'get account summary',
-    tags: ['accounts', 'state'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'account name',
-                type: 'string'
-            }
-        }
-    }
-};

api/schemas/get_actions.js

@@ -1,132 +0,0 @@
-exports.GET = {
-    description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
-        ' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
-    summary: 'get root actions',
-    tags: ['actions', 'history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'notified account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "track": {
-                description: 'total results to track (count) [number or true]',
-                type: 'string'
-            },
-            "filter": {
-                description: 'code:name filter',
-                type: 'string',
-                minLength: 3
-            },
-            "skip": {
-                description: 'skip [n] actions (pagination)',
-                type: 'integer',
-                minimum: 0
-            },
-            "limit": {
-                description: 'limit of [n] actions per page',
-                type: 'integer',
-                minimum: 1
-            },
-            "sort": {
-                description: 'sort direction',
-                enum: ['desc', 'asc', '1', '-1'],
-                type: 'string'
-            },
-            "after": {
-                description: 'filter after specified date (ISO8601)',
-                type: 'string'
-            },
-            "before": {
-                description: 'filter before specified date (ISO8601)',
-                type: 'string'
-            },
-            "simple": {
-                description: 'simplified output mode',
-                type: 'boolean'
-            }
-        }
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "query_time": {
-                    type: "number"
-                },
-                "cached": {type: "boolean"},
-                "lib": {
-                    type: "number"
-                },
-                "total": {
-                    type: "object",
-                    properties: {
-                        "value": {type: "number"},
-                        "relation": {type: "string"}
-                    }
-                },
-                "simple_actions": {
-                    type: "array",
-                    items: {
-                        type: "object",
-                        properties: {
-                            "block": {type: "number"},
-                            "timestamp": {type: "string"},
-                            "irreversible": {type: "boolean"},
-                            "contract": {type: "string"},
-                            "action": {type: "string"},
-                            "actors": {type: "string"},
-                            "notified": {type: "string"},
-                            "transaction_id": {type: "string"},
-                            "data": {
-                                additionalProperties: true
-                            }
-                        }
-                    }
-                },
-                "actions": {
-                    type: "array",
-                    items: {
-                        type: 'object',
-                        properties: {
-                            "act": {
-                                type: 'object',
-                                properties: {
-                                    "account": {type: "string"},
-                                    "name": {type: "string"}
-                                },
-                                additionalProperties: true
-                            },
-                            "cpu_usage_us": {type: "number"},
-                            "net_usage_words": {type: "number"},
-                            "account_ram_deltas": {
-                                type: "array",
-                                items: {
-                                    type: "object",
-                                    properties: {
-                                        "account": {type: "string"},
-                                        "delta": {type: "number"}
-                                    },
-                                    additionalProperties: true
-                                }
-                            },
-                            "global_sequence": {type: "number"},
-                            "receiver": {type: 'string'},
-                            "@timestamp": {type: "string"},
-                            "block_num": {type: "number"},
-                            "producer": {type: "string"},
-                            "trx_id": {type: "string"},
-                            "parent": {type: "number"},
-                            "action_ordinal": {type: 'number'},
-                            "creator_action_ordinal": {type: 'number'},
-                            "notified": {type: "array", items: {type: "string"}}
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_blocks.js

@@ -1,20 +0,0 @@
-exports.GET = {
-    description: 'get block range',
-    summary: 'get block range',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "from": {
-                description: 'starting block',
-                type: 'integer',
-                minimum: 1
-            },
-            "to": {
-                description: 'last block',
-                type: 'integer',
-                minimum: 1
-            }
-        }
-    }
-};

api/schemas/get_created_accounts.js

@@ -1,44 +0,0 @@
-exports.GET = {
-    description: 'get all accounts created by one creator',
-    summary: 'get created accounts',
-    tags: ['accounts', 'history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'creator account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            }
-        }
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "query_time": {
-                    type: "number"
-                },
-                "total": {
-                    type: "object",
-                    properties: {
-                        "value": {type: "number"},
-                        "relation": {type: "string"}
-                    }
-                },
-                "accounts": {
-                    type: "array",
-                    items: {
-                        type: 'object',
-                        properties: {
-                            'name': {type: 'string'},
-                            'timestamp': {type: 'string'},
-                            'trx_id': {type: 'string'}
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_creator.js

@@ -1,41 +0,0 @@
-exports.GET = {
-    description: 'get account creator',
-    summary: 'get account creator',
-    tags: ['accounts', 'history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'created account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            }
-        }
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "account": {
-                    type: "string"
-                },
-                "creator": {
-                    type: "string"
-                },
-                "timestamp": {
-                    type: "string"
-                },
-                "block_num": {
-                    type: "integer"
-                },
-                "trx_id": {
-                    type: "string"
-                },
-                "indirect_creator": {
-                    type: "string"
-                }
-            }
-        }
-    }
-};

api/schemas/get_deltas.js

@@ -1,26 +0,0 @@
-exports.GET = {
-    description: 'get state deltas',
-    summary: 'get state deltas',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "code": {
-                description: 'contract account',
-                type: 'string'
-            },
-            "scope": {
-                description: 'table scope',
-                type: 'string'
-            },
-            "table": {
-                description: 'table name',
-                type: 'string'
-            },
-            "payer": {
-                description: 'payer account',
-                type: 'string'
-            }
-        }
-    }
-};

api/schemas/get_key_accounts.js

@@ -1,58 +0,0 @@
-module.exports = {
-    GET: {
-        description: 'get accounts by public key',
-        summary: 'get accounts by public key',
-        tags: ['accounts','state'],
-        querystring: {
-            type: 'object',
-            properties: {
-                "public_key": {
-                    description: 'public key',
-                    type: 'string'
-                },
-            },
-            required: ["public_key"]
-        },
-        response: {
-            200: {
-                type: 'object',
-                properties: {
-                    "account_names": {
-                        type: "array",
-                        items: {
-                            type: "string"
-                        }
-                    }
-                }
-            }
-        }
-    },
-    POST: {
-        description: 'get accounts by public key',
-        summary: 'get accounts by public key',
-        tags: ['accounts','state'],
-        body: {
-            type: 'object',
-            properties: {
-                "public_key": {
-                    description: 'public key',
-                    type: 'string'
-                },
-            },
-            required: ["public_key"]
-        },
-        response: {
-            200: {
-                type: 'object',
-                properties: {
-                    "account_names": {
-                        type: "array",
-                        items: {
-                            type: "string"
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_proposals.js

@@ -1,48 +0,0 @@
-exports.GET = {
-    description: 'get proposals',
-    summary: 'get proposals',
-    tags: ['state'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "proposer": {
-                description: 'filter by proposer',
-                type: 'string'
-            },
-            "proposal": {
-                description: 'filter by proposal name',
-                type: 'string'
-            },
-            "account": {
-                description: 'filter by either requested or provided account',
-                type: 'string'
-            },
-            "requested": {
-                description: 'filter by requested account',
-                type: 'string'
-            },
-            "provided": {
-                description: 'filter by provided account',
-                type: 'string'
-            },
-            "executed": {
-                description: 'filter by execution status',
-                type: 'boolean'
-            },
-            "track": {
-                description: 'total results to track (count) [number or true]',
-                type: 'string'
-            },
-            "skip": {
-                description: 'skip [n] actions (pagination)',
-                type: 'integer',
-                minimum: 0
-            },
-            "limit": {
-                description: 'limit of [n] actions per page',
-                type: 'integer',
-                minimum: 1
-            }
-        }
-    }
-};

api/schemas/get_tokens.js

@@ -1,37 +0,0 @@
-exports.GET = {
-    description: 'get tokens from account',
-    summary: 'get tokens from account',
-    tags: ['accounts', 'state'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'account',
-                type: 'string'
-            },
-        },
-        required: ["account"]
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "query_time": {type: "number"},
-                "cached": {type: "boolean"},
-                "account": {type: "string"},
-                "tokens": {
-                    type: "array",
-                    items: {
-                        type: "object",
-                        properties: {
-                            "symbol": {type: "string"},
-                            "precision": {type: "number"},
-                            "amount": {type: "number"},
-                            "contract": {type: "string"}
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_transacted_accounts.js

@@ -1,52 +0,0 @@
-exports.GET = {
-    description: 'get all account that interacted with the source account provided',
-    summary: 'get interactions based on transfers',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "account": {
-                description: 'source account',
-                type: 'string'
-            },
-            "symbol": {
-                description: 'token symbol',
-                type: 'string',
-                minLength: 1,
-                maxLength: 7
-            },
-            "contract": {
-                description: 'token contract',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "direction": {
-                description: 'search direction',
-                enum: ['in', 'out', 'both'],
-                type: 'string'
-            },
-            "min": {
-                description: 'minimum value',
-                type: 'number'
-            },
-            "max": {
-                description: 'maximum value',
-                type: 'number'
-            },
-            "limit": {
-                description: 'query limit',
-                type: 'number'
-            },
-            "after": {
-                description: 'filter after specified date (ISO8601) or block number',
-                type: 'string'
-            },
-            "before": {
-                description: 'filter before specified date (ISO8601) or block number',
-                type: 'string'
-            }
-        },
-        required: ["account", "direction"]
-    }
-};

api/schemas/get_transaction.js

@@ -1,15 +0,0 @@
-exports.GET = {
-    description: 'get all actions belonging to the same transaction',
-    summary: 'get transaction by id',
-    tags: ['transactions','history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "id": {
-                description: 'transaction id',
-                type: 'string'
-            }
-        },
-        required: ["id"]
-    }
-};

api/schemas/get_transfers.js

@@ -1,96 +0,0 @@
-exports.GET = {
-    description: 'get token transfers utilizing the eosio.token standard',
-    summary: 'get token transfers',
-    tags: ['history'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "from": {
-                description: 'source account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "to": {
-                description: 'destination account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "symbol": {
-                description: 'token symbol',
-                type: 'string',
-                minLength: 1,
-                maxLength: 7
-            },
-            "contract": {
-                description: 'token contract',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "skip": {
-                description: 'skip [n] actions (pagination)',
-                type: 'integer',
-                minimum: 0
-            },
-            "limit": {
-                description: 'limit of [n] actions per page',
-                type: 'integer',
-                minimum: 1
-            },
-            "after": {
-                description: 'filter after specified date (ISO8601)',
-                type: 'string',
-                format: 'date-time'
-            },
-            "before": {
-                description: 'filter before specified date (ISO8601)',
-                type: 'string',
-                format: 'date-time'
-            },
-        },
-        "anyOf": [
-            {required: ["from"]},
-            {required: ["to"]},
-            {required: ["symbol"]},
-            {required: ["contract"]}
-        ]
-    },
-    response: {
-        200: {
-            type: 'object',
-            properties: {
-                "action_count": {
-                    type: "number"
-                },
-                "total_amount": {
-                    type: "number"
-                },
-                "actions": {
-                    type: "array",
-                    items: {
-                        type: 'object',
-                        properties: {
-                            "act": {
-                                type: 'object',
-                                properties: {
-                                    "account": {type: "string"},
-                                    "name": {type: "string"}
-                                },
-                                additionalProperties: true
-                            },
-                            "@timestamp": {type: "string"},
-                            "block_num": {type: "number"},
-                            "producer": {type: "string"},
-                            "trx_id": {type: "string"},
-                            "parent": {type: "number"},
-                            "global_sequence": {type: "number"},
-                            "notified": {type: "array", items: {type: "string"}}
-                        }
-                    }
-                }
-            }
-        }
-    }
-};

api/schemas/get_voters.js

@@ -1,24 +0,0 @@
-exports.GET = {
-    description: 'get voters',
-    summary: 'get voters',
-    tags: ['accounts','state'],
-    querystring: {
-        type: 'object',
-        properties: {
-            "producer": {
-                description: 'filter by voted producer (comma separated)',
-                type: 'string'
-            },
-            "skip": {
-                description: 'skip [n] actions (pagination)',
-                type: 'integer',
-                minimum: 0
-            },
-            "limit": {
-                description: 'limit of [n] actions per page',
-                type: 'integer',
-                minimum: 1
-            }
-        }
-    }
-};

api/schemas/index.js

@@ -1,16 +0,0 @@
-module.exports = {
-    getTransfersSchema: require('./get_transfers'),
-    getAbiSnapshotSchema: require('./get_abi_snapshot'),
-    getKeyAccountsSchema: require('./get_key_accounts'),
-    getActionsSchema: require('./get_actions'),
-    getTransactedAccountsSchema: require('./get_transacted_accounts'),
-    getTransactionSchema: require('./get_transaction'),
-    getCreatorSchema: require('./get_creator'),
-    getTokensSchema: require('./get_tokens'),
-    getDeltasSchema: require('./get_deltas'),
-    getCreatedAccountsSchema: require('./get_created_accounts'),
-    getVotersSchema: require('./get_voters'),
-    getProposalsSchema: require('./get_proposals'),
-    getBlocksSchema: require('./get_blocks'),
-    getAccountSchema: require('./get_account')
-};

api/server.ts

@@ -2,103 +2,145 @@ import {hLog} from "../helpers/common_functions";
 import {ConfigurationModule} from "../modules/config";
 import {ConnectionManager} from "../connections/manager.class";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
-import {IncomingMessage, Server, ServerResponse} from "http";
-import * as Fastify from 'fastify';
-import * as Redis from 'ioredis';
+import IORedis from 'ioredis';
+import fastify from 'fastify'
 import {registerPlugins} from "./plugins";
 import {AddressInfo} from "net";
 import {registerRoutes} from "./routes";
 import {generateOpenApiConfig} from "./config/open_api";
-import {createWriteStream, existsSync, mkdirSync, writeFileSync} from "fs";
+import {createWriteStream, existsSync, mkdirSync, readFileSync} from "fs";
 import {SocketManager} from "./socketManager";
-import got from "got";
-import {join} from "path";
-import * as io from 'socket.io-client';
+import {HyperionModuleLoader} from "../modules/loader";
+import {extendedActions} from "./routes/v2-history/get_actions/definitions";
+import {io, Socket} from "socket.io-client";
+import {CacheManager} from "./helpers/cacheManager";
+
+import {bootstrap} from 'global-agent';
+bootstrap();
 
 class HyperionApiServer {
 
-    private conf: HyperionConfig;
-    private readonly manager: ConnectionManager;
-    private readonly fastify: Fastify.FastifyInstance<Server, IncomingMessage, ServerResponse>;
+    private hub: Socket;
+    private readonly fastify;
     private readonly chain: string;
-    socketManager: SocketManager;
+    private readonly conf: HyperionConfig;
+    private readonly manager: ConnectionManager;
+    private readonly cacheManager: CacheManager;
 
-    private hub: SocketIOClient.Socket;
+    socketManager: SocketManager;
+    mLoader: HyperionModuleLoader;
 
     constructor() {
+
+        const package_json = JSON.parse(readFileSync('./package.json').toString());
+        hLog(`--------- Hyperion API ${package_json.version} ---------`);
+
         const cm = new ConfigurationModule();
         this.conf = cm.config;
         this.chain = this.conf.settings.chain;
         process.title = `hyp-${this.chain}-api`;
         this.manager = new ConnectionManager(cm);
         this.manager.calculateServerHash();
-        this.manager.getHyperionVersion();
+        this.mLoader = new HyperionModuleLoader(cm);
+        this.cacheManager = new CacheManager(this.conf);
 
         if (!existsSync('./logs/' + this.chain)) {
             mkdirSync('./logs/' + this.chain, {recursive: true});
         }
 
         const logStream = createWriteStream('./logs/' + this.chain + '/api.access.log');
-        this.fastify = Fastify({
+
+        const loggerOpts = {
+            stream: logStream,
+            redact: ['req.headers.authorization'],
+            level: 'info',
+            prettyPrint: true,
+            serializers: {
+                res: (reply) => {
+                    return {
+                        statusCode: reply.statusCode
+                    };
+                },
+                req: (request) => {
+                    return {
+                        method: request.method,
+                        url: request.url,
+                        ip: request.headers['x-real-ip']
+                    }
+                }
+            }
+        };
+
+        this.fastify = fastify({
             ignoreTrailingSlash: false,
             trustProxy: true,
             pluginTimeout: 5000,
-            logger: this.conf.api.access_log ? {
-                stream: logStream,
-                redact: ['req.headers.authorization'],
-                level: 'info',
-                serializers: {
-                    res: (res) => {
-                        return {
-                            status: res.statusCode
-                        };
-                    },
-                    req: (req) => {
-                        return {
-                            method: req.method,
-                            url: req.url,
-                            ip: req.headers['x-real-ip']
-                        }
-                    }
-                }
-            } : false
+            logger: this.conf.api.access_log ? loggerOpts : false
         });
+
+        this.fastify.decorate('cacheManager', this.cacheManager);
+
         this.fastify.decorate('manager', this.manager);
 
-        if (this.conf.api.chain_api && this.conf.api.chain_api !== "") {
-            this.fastify.decorate('chain_api', this.conf.api.chain_api);
-        } else {
-            this.fastify.decorate('chain_api', this.manager.conn.chains[this.chain].http);
+        // import get_actions query params from custom modules
+        const extendedActionsSet: Set<string> = new Set([...extendedActions]);
+        for (const qPrefix of this.mLoader.extendedActions) {
+            extendedActionsSet.add(qPrefix);
         }
+        this.fastify.decorate('allowedActionQueryParamSet', extendedActionsSet);
 
-        if (this.conf.api.push_api && this.conf.api.push_api !== "") {
+        // define chain api url for /v1/chain/ redirects
+        let chainApiUrl: string = this.conf.api.chain_api;
+        if (chainApiUrl === null || chainApiUrl === "") {
+            chainApiUrl = this.manager.conn.chains[this.chain].http;
+        }
+        this.fastify.decorate('chain_api', chainApiUrl);
+
+        // define optional push api url for /v1/chain/push_transaction
+        if (this.conf.api.push_api) {
             this.fastify.decorate('push_api', this.conf.api.push_api);
         }
 
-        console.log(`Chain API URL: ${this.fastify.chain_api}`);
-        console.log(`Push API URL: ${this.fastify.push_api}`);
+        hLog(`Chain API URL: "${this.fastify.chain_api}" | Push API URL: "${this.fastify.push_api}"`);
 
-        const ioRedisClient = new Redis(this.manager.conn.redis);
-        const api_rate_limit = {
-            max: 1000,
-            whitelist: ['127.0.0.1'],
-            timeWindow: '1 minute',
-            redis: ioRedisClient
-        };
+        const ioRedisClient = new IORedis(this.manager.conn.redis);
+
+        const pluginParams = {
+            fastify_elasticsearch: {
+                client: this.manager.elasticsearchClient
+            },
+            fastify_redis: this.manager.conn.redis,
+            fastify_eosjs: this.manager,
+        } as any;
+
+        if (!this.conf.api.disable_rate_limit) {
+            let rateLimiterWhitelist = ['127.0.0.1'];
+            if (this.conf.api.rate_limit_allow && this.conf.api.rate_limit_allow.length > 0) {
+                const tempSet = new Set<string>(['127.0.0.1', ...this.conf.api.rate_limit_allow]);
+                rateLimiterWhitelist = [...tempSet];
+            }
+            let rateLimiterRPM = 1000;
+            if (this.conf.api.rate_limit_rpm) {
+                rateLimiterRPM = this.conf.api.rate_limit_rpm;
+            }
+            pluginParams.fastify_rate_limit = {
+                max: rateLimiterRPM,
+                allowList: rateLimiterWhitelist,
+                timeWindow: '1 minute',
+                redis: ioRedisClient
+            }
+        }
 
         if (this.conf.features.streaming.enable) {
             this.activateStreaming();
         }
 
-        registerPlugins(this.fastify, {
-            fastify_elasticsearch: {
-                client: this.manager.elasticsearchClient
-            },
-            fastify_oas: generateOpenApiConfig(this.manager.config),
-            fastify_rate_limit: api_rate_limit,
-            fastify_redis: this.manager.conn.redis,
-            fastify_eosjs: this.manager,
-        });
+        const docsConfig = generateOpenApiConfig(this.manager.config);
+        if (docsConfig) {
+            pluginParams.fastify_swagger = docsConfig;
+        }
+
+        registerPlugins(this.fastify, pluginParams);
 
         this.addGenericTypeParsing();
     }
@@ -128,15 +170,15 @@ class HyperionApiServer {
     }
 
     private addGenericTypeParsing() {
-        this.fastify.addContentTypeParser('*', (req, done) => {
+        this.fastify.addContentTypeParser('*', (request, payload, done) => {
             let data = '';
-            req.on('data', chunk => {
+            payload.on('data', chunk => {
                 data += chunk;
             });
-            req.on('end', () => {
+            payload.on('end', () => {
                 done(null, data);
             });
-            req.on('error', (err) => {
+            payload.on('error', (err) => {
                 console.log('---- Content Parsing Error -----');
                 console.log(err);
             });
@@ -145,15 +187,24 @@ class HyperionApiServer {
 
     async init() {
 
-        await this.fetchChainLogo();
+        await this.mLoader.init();
+
+        // add custom plugin routes
+        for (const plugin of this.mLoader.plugins) {
+            if (plugin.hasApiRoutes) {
+                hLog(`Adding routes for plugin: ${plugin.internalPluginName}`);
+                plugin.addRoutes(this.fastify);
+                plugin.chainName = this.chain;
+            }
+        }
 
         registerRoutes(this.fastify);
 
+        // register documentation when ready
         this.fastify.ready().then(async () => {
-            await this.fastify.oas();
-            console.log(this.chain + ' api ready!');
+            await this.fastify.swagger();
         }, (err) => {
-            console.log('an error happened', err)
+            hLog('an error happened', err)
         });
 
         try {
@@ -161,28 +212,14 @@ class HyperionApiServer {
                 host: this.conf.api.server_addr,
                 port: this.conf.api.server_port
             });
-            console.log(`server listening on ${(this.fastify.server.address() as AddressInfo).port}`);
+            hLog(`${this.chain} hyperion api ready and listening on port ${(this.fastify.server.address() as AddressInfo).port}`);
             this.startHyperionHub();
         } catch (err) {
-            console.log(err);
+            hLog(err);
             process.exit(1)
         }
     }
 
-    async fetchChainLogo() {
-        try {
-            if (this.conf.api.chain_logo_url && this.conf.api.enable_explorer) {
-                console.log(`Downloading chain logo from ${this.conf.api.chain_logo_url}...`);
-                const chainLogo = await got(this.conf.api.chain_logo_url);
-                const path = join(__dirname, '..', 'hyperion-explorer', 'dist', 'assets', this.chain + '_logo.png');
-                writeFileSync(path, chainLogo.rawBody);
-                this.conf.api.chain_logo_url = 'https://' + this.conf.api.server_name + '/v2/explore/assets/' + this.chain + '_logo.png';
-            }
-        } catch (e) {
-            console.log(e);
-        }
-    }
-
     startHyperionHub() {
         if (this.conf.hub) {
             const url = this.conf.hub.inform_url;
@@ -190,17 +227,13 @@ class HyperionApiServer {
             this.hub = io(url, {
                 query: {
                     key: this.conf.hub.publisher_key,
-                    client_mode: false
+                    client_mode: 'false'
                 }
             });
             this.hub.on('connect', () => {
                 hLog(`Hyperion Hub connected!`);
                 this.emitHubApiUpdate();
             });
-            // this.hub.on('reconnect', () => {
-            //     hLog(`Reconnecting...`);
-            //     this.emitHubApiUpdate();
-            // });
         }
     }
 
@@ -211,7 +244,7 @@ class HyperionApiServer {
             location: this.conf.hub.location,
             chainId: this.manager.conn.chains[this.chain].chain_id,
             providerName: this.conf.api.provider_name,
-            explorerEnabled: this.conf.api.enable_explorer,
+            explorerEnabled: this.conf.plugins.explorer?.enabled,
             providerUrl: this.conf.api.provider_url,
             providerLogo: this.conf.api.provider_logo,
             chainLogo: this.conf.api.chain_logo_url,

api/socketManager.ts

@@ -1,7 +1,11 @@
-import {checkFilter, hLog} from '../helpers/common_functions';
-import * as sockets from 'socket.io';
-import * as IOClient from 'socket.io-client';
+import {hLog} from '../helpers/common_functions';
+import {Server, Socket} from 'socket.io';
+import {createAdapter} from 'socket.io-redis';
+import {io} from 'socket.io-client';
 import {FastifyInstance} from "fastify";
+import IORedis from "ioredis";
+import {App, TemplatedApp} from 'uWebSockets.js';
+import {streamPastActions, streamPastDeltas} from "./helpers/functions";
 
 export interface StreamDeltasRequest {
     code: string;
@@ -26,234 +30,43 @@ export interface StreamActionsRequest {
     read_until: number | string;
 }
 
-async function addBlockRangeOpts(data, search_body, fastify: FastifyInstance) {
-
-    let timeRange;
-    let blockRange;
-    let head;
-
-    if (typeof data['start_from'] === 'string' && data['start_from'] !== '') {
-        if (!timeRange) {
-            timeRange = {"@timestamp": {}};
-        }
-        timeRange["@timestamp"]['gte'] = data['start_from'];
-    }
-
-    if (typeof data['read_until'] === 'string' && data['read_until'] !== '') {
-        if (!timeRange) {
-            timeRange = {"@timestamp": {}};
-        }
-        timeRange["@timestamp"]['lte'] = data['read_until'];
-    }
-
-    if (typeof data['start_from'] === 'number' && data['start_from'] !== 0) {
-        if (!blockRange) {
-            blockRange = {"block_num": {}};
-        }
-        if (data['start_from'] < 0) {
-            if (!head) {
-                head = (await fastify.eosjs.rpc.get_info()).head_block_num;
-            }
-            blockRange["block_num"]['gte'] = head + data['start_from'];
-        } else {
-            blockRange["block_num"]['gte'] = data['start_from'];
-        }
-    }
-
-    if (typeof data['read_until'] === 'number' && data['read_until'] !== 0) {
-        if (!blockRange) {
-            blockRange = {"block_num": {}};
-        }
-        if (data['read_until'] < 0) {
-            if (!head) {
-                head = (await fastify.eosjs.rpc.get_info()).head_block_num;
-            }
-            blockRange["block_num"]['lte'] = head + data['read_until'];
-        } else {
-            blockRange["block_num"]['lte'] = data['read_until'];
-        }
-    }
-
-    if (timeRange) {
-        search_body.query.bool.must.push({
-            range: timeRange,
-        });
-    }
-    if (blockRange) {
-        search_body.query.bool.must.push({
-            range: blockRange,
-        });
-    }
-}
-
-function addTermMatch(data, search_body, field) {
-    if (data[field] !== '*' && data[field] !== '') {
-        const termQuery = {};
-        termQuery[field] = data[field];
-        search_body.query.bool.must.push({'term': termQuery});
-    }
-}
-
-const deltaQueryFields = ['code', 'table', 'scope', 'payer'];
-
-async function streamPastDeltas(fastify: FastifyInstance, socket, data) {
-    const search_body = {
-        query: {bool: {must: []}},
-        sort: {block_num: 'asc'},
-    };
-    await addBlockRangeOpts(data, search_body, fastify);
-    deltaQueryFields.forEach(f => {
-        addTermMatch(data, search_body, f);
-    });
-    const responseQueue = [];
-    let counter = 0;
-    const init_response = await fastify.elastic.search({
-        index: fastify.manager.chain + '-delta-*',
-        scroll: '30s',
-        size: 20,
-        body: search_body,
-    });
-    responseQueue.push(init_response);
-    while (responseQueue.length) {
-        const {body} = responseQueue.shift();
-        counter += body['hits']['hits'].length;
-        if (socket.connected) {
-            socket.emit('message', {
-                type: 'delta_trace',
-                mode: 'history',
-                messages: body['hits']['hits'].map(doc => doc._source),
-            });
-        } else {
-            console.log('LOST CLIENT');
-            break;
-        }
-        if (body['hits'].total.value === counter) {
-            console.log(`${counter} past deltas streamed to ${socket.id}`);
-            break;
-        }
-
-        const next_response = await fastify.elastic.scroll({
-            body: {
-                scroll_id: body['_scroll_id'],
-                scroll: '30s'
-            }
-        });
-        responseQueue.push(next_response);
-    }
-}
-
-async function streamPastActions(fastify: FastifyInstance, socket, data) {
-    const search_body = {
-        query: {bool: {must: []}},
-        sort: {global_sequence: 'asc'},
-    };
-
-    await addBlockRangeOpts(data, search_body, fastify);
-
-    if (data.account !== '') {
-        search_body.query.bool.must.push({
-            bool: {
-                should: [
-                    {term: {'notified': data.account}},
-                    {term: {'act.authorization.actor': data.account}},
-                ],
-            },
-        });
-    }
-
-    if (data.contract !== '*' && data.contract !== '') {
-        search_body.query.bool.must.push({'term': {'act.account': data.contract}});
-    }
-
-    if (data.action !== '*' && data.action !== '') {
-        search_body.query.bool.must.push({'term': {'act.name': data.action}});
-    }
-
-    const onDemandFilters = [];
-    if (data.filters.length > 0) {
-        data.filters.forEach(f => {
-            if (f.field && f.value) {
-                if (f.field.startsWith('@') && !f.field.startsWith('act.data')) {
-                    const _q = {};
-                    _q[f.field] = f.value;
-                    search_body.query.bool.must.push({'term': _q});
-                } else {
-                    onDemandFilters.push(f);
-                }
-            }
-        });
-    }
-
-    const responseQueue = [];
-    let counter = 0;
-
-    const init_response = await fastify.elastic.search({
-        index: fastify.manager.chain + '-action-*',
-        scroll: '30s',
-        size: 20,
-        body: search_body,
-    });
-    responseQueue.push(init_response);
-    while (responseQueue.length) {
-        const {body} = responseQueue.shift();
-        const enqueuedMessages = [];
-        counter += body['hits']['hits'].length;
-        for (const doc of body['hits']['hits']) {
-            let allow = false;
-            if (onDemandFilters.length > 0) {
-                allow = onDemandFilters.every(filter => {
-                    return checkFilter(filter, doc._source);
-                });
-            } else {
-                allow = true;
-            }
-            if (allow) {
-                enqueuedMessages.push(doc._source);
-            }
-        }
-        if (socket.connected) {
-            socket.emit('message', {type: 'action_trace', mode: 'history', messages: enqueuedMessages});
-        } else {
-            console.log('LOST CLIENT');
-            break;
-        }
-        if (body['hits'].total.value === counter) {
-            console.log(`${counter} past actions streamed to ${socket.id}`);
-            break;
-        }
-        if (init_response.body.hits.total.value < 1000) {
-            const next_response = await fastify.elastic.scroll({
-                body: {scroll_id: body['_scroll_id'], scroll: '30s'}
-            });
-            responseQueue.push(next_response);
-        } else {
-            console.log('Request too large!');
-            socket.emit('message', {type: 'action_trace', mode: 'history', messages: []});
-        }
-    }
-}
 
 export class SocketManager {
 
-    private io;
+    private io: Server;
     private relay;
     relay_restored = true;
     relay_down = false;
     private readonly url;
     private readonly server: FastifyInstance;
+    private readonly uwsApp: TemplatedApp;
 
-    constructor(fastify: FastifyInstance, url, redisOpts) {
+    constructor(fastify: FastifyInstance, url, redisOptions) {
         this.server = fastify;
         this.url = url;
+        this.uwsApp = App({});
 
-        this.io = sockets(fastify.server, {
-            transports: ['websocket', 'polling'],
+        // this.io = new Server(fastify.server, {
+        // 	allowEIO3: true,
+        // 	transports: ['websocket', 'polling'],
+        // });
+
+        // WS Server for public access
+        this.io = new Server({
+            transports: ['websocket'],
+            path: '/stream'
         });
 
-        this.io.on('connection', (socket) => {
+        this.io.attachApp(this.uwsApp);
+
+        const pubClient = new IORedis(redisOptions);
+        const subClient = pubClient.duplicate();
+        this.io.adapter(createAdapter({pubClient, subClient}));
+
+        this.io.on('connection', (socket: Socket) => {
 
             if (socket.handshake.headers['x-forwarded-for']) {
-                console.log(`[socket] ${socket.id} connected via ${socket.handshake.headers['x-forwarded-for']}`);
+                hLog(`[socket] ${socket.id} connected via ${socket.handshake.headers['x-forwarded-for']}`);
             }
 
             socket.emit('message', {
@@ -264,7 +77,7 @@ export class SocketManager {
             if (this.relay) {
                 this.relay.emit('event', {
                     type: 'client_count',
-                    counter: Object.keys(this.io.sockets.connected).length,
+                    counter: this.io.sockets.sockets.size,
                 });
             }
 
@@ -295,7 +108,7 @@ export class SocketManager {
             });
 
             socket.on('disconnect', (reason) => {
-                console.log(`[socket] ${socket.id} disconnected - ${reason}`);
+                hLog(`[socket] ${socket.id} disconnected - ${reason}`);
                 this.relay.emit('event', {
                     type: 'client_disconnected',
                     id: socket.id,
@@ -303,16 +116,28 @@ export class SocketManager {
                 });
             });
         });
-        console.log('Websocket manager loaded!');
+
+        try {
+            const port = this.server.manager.config.api.stream_port || 1234;
+            this.uwsApp.listen(port, () => {
+                hLog(`Socket.IO via uWS started on port ${port}`);
+            });
+        } catch (e) {
+            hLog(e.message);
+        }
+
+        hLog('Websocket manager loaded!');
     }
 
+    /*
+    WS Relay will connect to the indexer
+     */
     startRelay() {
-        console.log(`starting relay - ${this.url}`);
-
-        this.relay = IOClient(this.url, {path: '/router'});
+        hLog(`starting relay - ${this.url}`);
+        this.relay = io(this.url, {path: '/router'});
 
         this.relay.on('connect', () => {
-            console.log('Relay Connected!');
+            hLog('Relay Connected!');
             if (this.relay_down) {
                 this.relay_restored = true;
                 this.relay_down = false;
@@ -321,7 +146,7 @@ export class SocketManager {
         });
 
         this.relay.on('disconnect', () => {
-            console.log('Relay disconnected!');
+            hLog('Relay disconnected!');
             this.io.emit('status', 'relay_down');
             this.relay_down = true;
             this.relay_restored = false;
@@ -344,7 +169,7 @@ export class SocketManager {
 
         // Relay LIB info to clients;
         this.relay.on('fork_event', (data) => {
-            console.log(data);
+            hLog(data);
             if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
                 this.io.emit('fork_event', data);
             }
@@ -352,8 +177,8 @@ export class SocketManager {
     }
 
     emitToClient(traceData, type) {
-        if (this.io.sockets.connected[traceData.client]) {
-            this.io.sockets.connected[traceData.client].emit('message', {
+        if (this.io.sockets.sockets.has(traceData.client)) {
+            this.io.sockets.sockets.get(traceData.client).emit('message', {
                 type: type,
                 mode: 'live',
                 message: traceData.message,

auto_setup.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-git clone https://github.com/eosrio/hyperion-history-api.git  || exit
-cd hyperion-history-api || exit
-bash ./install_env.sh

chains/example.config.json

@@ -1,8 +1,12 @@
 {
   "api": {
+    "enabled": true,
+    "pm2_scaling": 1,
+    "node_max_old_space_size": 1024,
     "chain_name": "EXAMPLE Chain",
     "server_addr": "127.0.0.1",
     "server_port": 7000,
+    "stream_port": 1234,
     "server_name": "127.0.0.1:7000",
     "provider_name": "Example Provider",
     "provider_url": "https://example.com",
@@ -19,17 +23,48 @@
       "get_trx_actions": 200
     },
     "access_log": false,
-    "enable_explorer": false,
     "chain_api_error_log": false,
     "custom_core_token": "",
-    "enable_export_action": false
+    "enable_export_action": false,
+    "disable_rate_limit": false,
+    "rate_limit_rpm": 1000,
+    "rate_limit_allow": [],
+    "disable_tx_cache": false,
+    "tx_cache_expiration_sec": 3600,
+    "v1_chain_cache": [
+      {
+        "path": "get_block",
+        "ttl": 3000
+      },
+      {
+        "path": "get_info",
+        "ttl": 500
+      }
+    ]
+  },
+  "indexer": {
+    "enabled": true,
+    "node_max_old_space_size": 4096,
+    "start_on": 0,
+    "stop_on": 0,
+    "rewrite": false,
+    "purge_queues": false,
+    "live_reader": false,
+    "live_only_mode": false,
+    "abi_scan_mode": true,
+    "fetch_block": true,
+    "fetch_traces": true,
+    "disable_reading": false,
+    "disable_indexing": false,
+    "process_deltas": true,
+    "disable_delta_rm": true
   },
   "settings": {
     "preview": false,
     "chain": "eos",
     "eosio_alias": "eosio",
-    "parser": "1.8",
-    "auto_stop": 300,
+    "parser": "3.2",
+    "auto_stop": 0,
     "index_version": "v1",
     "debug": false,
     "bp_logs": false,
@@ -37,12 +72,14 @@
     "ipc_debug_rate": 60000,
     "allow_custom_abi": false,
     "rate_monitoring": true,
-    "max_ws_payload_kb": 256,
+    "max_ws_payload_mb": 256,
     "ds_profiling": false,
     "auto_mode_switch": false,
     "hot_warm_policy": false,
     "custom_policy": "",
-    "bypass_index_map": false
+    "bypass_index_map": false,
+    "index_partition_size": 10000000,
+    "es_replicas": 0
   },
   "blacklists": {
     "actions": [],
@@ -55,34 +92,21 @@
     "root_only": false
   },
   "scaling": {
-    "readers":            1,
-    "ds_queues":          1,
-    "ds_threads":         1,
-    "ds_pool_size":       1,
-    "indexing_queues":    1,
-    "ad_idx_queues":      1,
-    "max_autoscale":      4,
-    "batch_size":         5000,
-    "resume_trigger":     5000,
+    "readers": 1,
+    "ds_queues": 1,
+    "ds_threads": 1,
+    "ds_pool_size": 1,
+    "indexing_queues": 1,
+    "ad_idx_queues": 1,
+    "dyn_idx_queues": 1,
+    "max_autoscale": 4,
+    "batch_size": 5000,
+    "resume_trigger": 5000,
     "auto_scale_trigger": 20000,
-    "block_queue_limit":  10000,
-    "max_queue_limit":    100000,
-    "routing_mode":       "heatmap",
-    "polling_interval":   10000
-  },
-  "indexer": {
-    "start_on": 0,
-    "stop_on": 0,
-    "rewrite": false,
-    "purge_queues": false,
-    "live_reader": false,
-    "live_only_mode": false,
-    "abi_scan_mode": true,
-    "fetch_block": true,
-    "fetch_traces": true,
-    "disable_reading": false,
-    "disable_indexing": false,
-    "process_deltas": true
+    "block_queue_limit": 10000,
+    "max_queue_limit": 100000,
+    "routing_mode": "round_robin",
+    "polling_interval": 10000
   },
   "features": {
     "streaming": {
@@ -96,7 +120,7 @@
       "voters": true
     },
     "index_deltas": true,
-    "index_transfer_memo": false,
+    "index_transfer_memo": true,
     "index_all_deltas": true,
     "deferred_trx": false,
     "failed_trx": false,
@@ -107,5 +131,6 @@
     "read": 50,
     "block": 100,
     "index": 500
-  }
+  },
+  "plugins": {}
 }

connections/amqp.ts

@@ -9,18 +9,22 @@ export async function createConnection(config): Promise<Connection> {
 		debugLog("[AMQP] connection established");
 		return conn;
 	} catch (e) {
-		console.log("[AMQP] failed to connect!");
-		console.log(e.message);
+		hLog("[AMQP] failed to connect!");
+		hLog(e.message);
 		await new Promise(resolve => setTimeout(resolve, 5000));
 		return await createConnection(config);
 	}
 }
 
 export function getAmpqUrl(config): string {
+	let frameMaxValue = '0x10000';
+	if (config.frameMax) {
+		frameMaxValue = config.frameMax;
+	}
 	const u = encodeURIComponent(config.user);
 	const p = encodeURIComponent(config.pass);
 	const v = encodeURIComponent(config.vhost)
-	return `amqp://${u}:${p}@${config.host}/${v}`;
+	return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
 }
 
 
@@ -30,8 +34,8 @@ async function createChannels(connection) {
 		const confirmChannel = await connection.createConfirmChannel();
 		return [channel, confirmChannel];
 	} catch (e) {
-		console.log("[AMQP] failed to create channels");
-		console.error(e);
+		hLog("[AMQP] failed to create channels");
+		hLog(e);
 		return null;
 	}
 }
@@ -66,7 +70,7 @@ export async function amqpConnect(onReconnect, config, onClose) {
 export async function checkQueueSize(q_name, config) {
 	try {
 		const v = encodeURIComponent(config.vhost);
-		const apiUrl = `http://${config.api}/api/queues/${v}/${encodeURIComponent(q_name)}`;
+		const apiUrl = `${config.protocol}://${config.api}/api/queues/${v}/${encodeURIComponent(q_name)}`;
 		const opts = {
 			username: config.user,
 			password: config.pass
@@ -74,11 +78,13 @@ export async function checkQueueSize(q_name, config) {
 		const data = await got.get(apiUrl, opts).json() as any;
 		return data.messages;
 	} catch (e) {
-		hLog('[WARNING] Checking queue size failed, HTTP API is not ready!');
-		if (e instanceof HTTPError) {
-			hLog(e.response);
-		} else {
-			hLog(JSON.stringify(e, null, 2));
+		hLog(`[WARNING] Checking queue size failed! - ${e.message}`);
+		if (e.response && e.response.body) {
+			if (e instanceof HTTPError) {
+				hLog(e.response.body);
+			} else {
+				hLog(JSON.stringify(e.response.body, null, 2));
+			}
 		}
 		return 0;
 	}

connections/manager.class.ts

@@ -1,14 +1,15 @@
 import {ConfigurationModule} from "../modules/config";
-import {JsonRpc} from "eosjs/dist";
+import {JsonRpc} from "eosjs";
 import got from "got";
 import {Client} from '@elastic/elasticsearch'
 import {HyperionConnections} from "../interfaces/hyperionConnections";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
 import {amqpConnect, checkQueueSize, getAmpqUrl} from "./amqp";
 import {StateHistorySocket} from "./state-history";
-import fetch from 'node-fetch';
+import fetch from 'cross-fetch';
 import {exec} from "child_process";
 import {hLog} from "../helpers/common_functions";
+import {readFileSync} from "fs";
 
 export class ConnectionManager {
 
@@ -19,12 +20,15 @@ export class ConnectionManager {
     last_commit_hash: string;
     current_version: string;
 
-    private esIngestClients: Client[];
+    private readonly esIngestClients: Client[];
     private esIngestClient: Client;
 
     constructor(private cm: ConfigurationModule) {
         this.config = cm.config;
         this.conn = cm.connections;
+        if (!this.conn.amqp.protocol) {
+            this.conn.amqp.protocol = 'http';
+        }
         this.chain = this.config.settings.chain;
         this.esIngestClients = [];
         this.prepareESClient();
@@ -39,7 +43,8 @@ export class ConnectionManager {
     async purgeQueues() {
         hLog(`Purging all ${this.chain} queues!`);
         const apiUrl = `http://${this.conn.amqp.api}`;
-        const getAllQueuesFromVHost = apiUrl + `/api/queues/%2F${this.conn.amqp.vhost}`;
+        const vHost = encodeURIComponent(this.conn.amqp.vhost);
+        const getAllQueuesFromVHost = apiUrl + `/api/queues/${vHost}`;
         const opts = {
             username: this.conn.amqp.user,
             password: this.conn.amqp.pass
@@ -61,7 +66,7 @@ export class ConnectionManager {
                     const msg_count = parseInt(queue.messages);
                     if (msg_count > 0) {
                         try {
-                            await got.delete(apiUrl + `/api/queues/%2F${this.conn.amqp.vhost}/${queue.name}/contents`, opts);
+                            await got.delete(apiUrl + `/api/queues/${vHost}/${queue.name}/contents`, opts);
                             hLog(`${queue.messages} messages deleted on queue ${queue.name}`);
                         } catch (e) {
                             console.log(e.message);
@@ -75,21 +80,19 @@ export class ConnectionManager {
     }
 
     prepareESClient() {
-        let es_url;
         const _es = this.conn.elasticsearch;
         if (!_es.protocol) {
             _es.protocol = 'http';
         }
-        if (_es.user !== '') {
-            es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${_es.host}`;
-        } else {
-            es_url = `${_es.protocol}://${_es.host}`
-        }
         this.esIngestClient = new Client({
-            node: es_url,
-            ssl: {
+            node: `${_es.protocol}://${_es.host}`,
+            auth: {
+                username: _es.user,
+                password: _es.pass
+            },
+            ssl: _es.protocol === 'https' ? {
                 rejectUnauthorized: false
-            }
+            } : undefined
         });
     }
 
@@ -105,19 +108,17 @@ export class ConnectionManager {
         if (_es.ingest_nodes) {
             if (_es.ingest_nodes.length > 0) {
                 for (const node of _es.ingest_nodes) {
-                    let es_url;
-                    if (_es.user !== '') {
-                        es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${node}`;
-                    } else {
-                        es_url = `${_es.protocol}://${node}`
-                    }
-                    this.esIngestClients.push(new Client({
-                        node: es_url,
+                    this.esIngestClient = new Client({
+                        node: `${_es.protocol}://${_es.host}`,
+                        auth: {
+                            username: _es.user,
+                            password: _es.pass
+                        },
                         pingTimeout: 100,
-                        ssl: {
+                        ssl: _es.protocol === 'https' ? {
                             rejectUnauthorized: false
-                        }
-                    }));
+                        } : undefined
+                    });
                 }
             }
         }
@@ -140,7 +141,7 @@ export class ConnectionManager {
     }
 
     get shipClient(): StateHistorySocket {
-        return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_kb);
+        return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_mb);
     }
 
     get ampqUrl() {
@@ -149,8 +150,15 @@ export class ConnectionManager {
 
     calculateServerHash() {
         exec('git rev-parse HEAD', (err, stdout) => {
-            console.log('Last commit hash on this branch is:', stdout);
-            this.last_commit_hash = stdout.trim();
+            if (err) {
+                // hLog(`\n ${err.message}\n >>> Failed to check last commit hash. Version hash will be "custom"`);
+                hLog(`Failed to check last commit hash. Version hash will be "custom"`);
+                this.last_commit_hash = 'custom';
+            } else {
+                hLog('Last commit hash on this branch is:', stdout);
+                this.last_commit_hash = stdout.trim();
+            }
+            this.getHyperionVersion();
         });
     }
 
@@ -159,6 +167,9 @@ export class ConnectionManager {
     }
 
     getHyperionVersion() {
-        this.current_version = require('../package.json').version
+        this.current_version = require('../package.json').version;
+        if (this.last_commit_hash === 'custom') {
+            this.current_version = this.current_version + '-dirty';
+        }
     }
 }

connections/state-history.ts

@@ -1,48 +1,57 @@
 import {debugLog, hLog} from "../helpers/common_functions";
 
-const WebSocket = require('ws');
+import WebSocket from 'ws';
 
 export class StateHistorySocket {
-    private ws;
-    private readonly shipUrl;
-    private readonly max_payload_kb;
+	private ws;
+	private readonly shipUrl;
+	private readonly max_payload_mb;
+	retryOnDisconnect = true;
+	connected = false;
 
-    constructor(ship_url, max_payload_kb) {
-        this.shipUrl = ship_url;
-        if (max_payload_kb) {
-            this.max_payload_kb = max_payload_kb;
-        } else {
-            this.max_payload_kb = 256;
-        }
-    }
+	constructor(ship_url, max_payload_mb) {
+		this.shipUrl = ship_url;
+		if (max_payload_mb) {
+			this.max_payload_mb = max_payload_mb;
+		} else {
+			this.max_payload_mb = 256;
+		}
+	}
 
-    connect(onMessage, onDisconnect, onError, onConnected) {
-        debugLog(`Connecting to ${this.shipUrl}...`);
-        this.ws = new WebSocket(this.shipUrl, null, {
-            perMessageDeflate: false,
-            maxPayload: this.max_payload_kb * 1024 * 1024,
-        });
-        this.ws.on('open', () => {
-            hLog('Websocket connected!');
-            if (onConnected) {
-                onConnected();
-            }
-        });
-        this.ws.on('message', onMessage);
-        this.ws.on('close', () => {
-            hLog('Websocket disconnected!');
-            onDisconnect();
-        });
-        this.ws.on('error', (err) => {
-            hLog(`${this.shipUrl} :: ${err.message}`);
-        });
-    }
+	connect(onMessage, onDisconnect, onError, onConnected) {
+		debugLog(`Connecting to ${this.shipUrl}...`);
+		this.ws = new WebSocket(this.shipUrl, {
+			perMessageDeflate: false,
+			maxPayload: this.max_payload_mb * 1024 * 1024,
+		});
+		this.ws.on('open', () => {
+			this.connected = true;
+			hLog('Websocket connected!');
+			if (onConnected) {
+				onConnected();
+			}
+		});
+		this.ws.on('message', (data) => onMessage(data));
+		this.ws.on('close', () => {
+			this.connected = false;
+			hLog('Websocket disconnected!');
+			if(this.retryOnDisconnect) {
+				onDisconnect();
+			}
+		});
+		this.ws.on('error', (err) => {
+			hLog(`${this.shipUrl} :: ${err.message}`);
+		});
+	}
 
-    close() {
-        this.ws.close();
-    }
+	close(graceful: boolean) {
+		if(graceful) {
+			this.retryOnDisconnect = false;
+		}
+		this.ws.close();
+	}
 
-    send(payload) {
-        this.ws.send(payload);
-    }
+	send(payload) {
+		this.ws.send(payload);
+	}
 }

definitions/abi_def.js

@@ -1,260 +1,264 @@
 const AbiDefinitions = {
-    version: "eosio::abi/1.1",
-    structs: [
+  version: 'eosio::abi/1.1',
+  structs: [
+    {
+      name: 'extensions_entry',
+      base: '',
+      fields: [
         {
-            name: "extensions_entry",
-            base: "",
-            fields: [
-                {
-                    name: "tag",
-                    type: "uint16"
-                },
-                {
-                    name: "value",
-                    type: "bytes"
-                }
-            ]
+          name: 'tag',
+          type: 'uint16',
         },
         {
-            name: "type_def",
-            base: "",
-            fields: [
-                {
-                    name: "new_type_name",
-                    type: "string"
-                },
-                {
-                    name: "type",
-                    type: "string"
-                }
-            ]
+          name: 'value',
+          type: 'bytes',
+        },
+      ],
+    },
+    {
+      name: 'type_def',
+      base: '',
+      fields: [
+        {
+          name: 'new_type_name',
+          type: 'string',
         },
         {
-            name: "field_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "string"
-                },
-                {
-                    name: "type",
-                    type: "string"
-                }
-            ]
+          name: 'type',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'field_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'string',
         },
         {
-            name: "struct_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "string"
-                },
-                {
-                    name: "base",
-                    type: "string"
-                },
-                {
-                    name: "fields",
-                    type: "field_def[]"
-                }
-            ]
+          name: 'type',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'struct_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'string',
         },
         {
-            name: "action_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "name"
-                },
-                {
-                    name: "type",
-                    type: "string"
-                },
-                {
-                    name: "ricardian_contract",
-                    type: "string"
-                }
-            ]
+          name: 'base',
+          type: 'string',
         },
         {
-            name: "table_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "name"
-                },
-                {
-                    name: "index_type",
-                    type: "string"
-                },
-                {
-                    name: "key_names",
-                    type: "string[]"
-                },
-                {
-                    name: "key_types",
-                    type: "string[]"
-                },
-                {
-                    name: "type",
-                    type: "string"
-                }
-            ]
+          name: 'fields',
+          type: 'field_def[]',
+        },
+      ],
+    },
+    {
+      name: 'action_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'name',
         },
         {
-            name: "clause_pair",
-            base: "",
-            fields: [
-                {
-                    name: "id",
-                    type: "string"
-                },
-                {
-                    name: "body",
-                    type: "string"
-                }
-            ]
+          name: 'type',
+          type: 'string',
         },
         {
-            name: "error_message",
-            base: "",
-            fields: [
-                {
-                    name: "error_code",
-                    type: "uint64"
-                },
-                {
-                    name: "error_msg",
-                    type: "string"
-                }
-            ]
+          name: 'ricardian_contract',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'table_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'name',
         },
         {
-            name: "variant_def",
-            base: "",
-            fields: [
-                {
-                    name: "name",
-                    type: "string"
-                },
-                {
-                    name: "types",
-                    type: "string[]"
-                }
-            ]
+          name: 'index_type',
+          type: 'string',
         },
         {
-            name: "abi_def",
-            base: "",
-            fields: [
-                {
-                    name: "version",
-                    type: "string"
-                },
-                {
-                    name: "types",
-                    type: "type_def[]"
-                },
-                {
-                    name: "structs",
-                    type: "struct_def[]"
-                },
-                {
-                    name: "actions",
-                    type: "action_def[]"
-                },
-                {
-                    name: "tables",
-                    type: "table_def[]"
-                },
-                {
-                    name: "ricardian_clauses",
-                    type: "clause_pair[]"
-                },
-                {
-                    name: "error_messages",
-                    type: "error_message[]"
-                },
-                {
-                    name: "abi_extensions",
-                    type: "extensions_entry[]"
-                },
-                {
-                    name: "variants",
-                    type: "variant_def[]$"
-                }
-            ]
-        }
-    ]
+          name: 'key_names',
+          type: 'string[]',
+        },
+        {
+          name: 'key_types',
+          type: 'string[]',
+        },
+        {
+          name: 'type',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'clause_pair',
+      base: '',
+      fields: [
+        {
+          name: 'id',
+          type: 'string',
+        },
+        {
+          name: 'body',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'error_message',
+      base: '',
+      fields: [
+        {
+          name: 'error_code',
+          type: 'uint64',
+        },
+        {
+          name: 'error_msg',
+          type: 'string',
+        },
+      ],
+    },
+    {
+      name: 'variant_def',
+      base: '',
+      fields: [
+        {
+          name: 'name',
+          type: 'string',
+        },
+        {
+          name: 'types',
+          type: 'string[]',
+        },
+      ],
+    },
+    {
+      name: 'abi_def',
+      base: '',
+      fields: [
+        {
+          name: 'version',
+          type: 'string',
+        },
+        {
+          name: 'types',
+          type: 'type_def[]',
+        },
+        {
+          name: 'structs',
+          type: 'struct_def[]',
+        },
+        {
+          name: 'actions',
+          type: 'action_def[]',
+        },
+        {
+          name: 'tables',
+          type: 'table_def[]',
+        },
+        {
+          name: 'ricardian_clauses',
+          type: 'clause_pair[]',
+        },
+        {
+          name: 'error_messages',
+          type: 'error_message[]',
+        },
+        {
+          name: 'abi_extensions',
+          type: 'extensions_entry[]',
+        },
+        {
+          name: 'variants',
+          type: 'variant_def[]$',
+        },
+      ],
+    },
+  ],
 };
 
 const RexAbi = {
-    version: "eosio::abi/1.1",
-    types: [],
-    structs: [
+  version: 'eosio::abi/1.1',
+  types: [],
+  structs: [
+    {
+      name: 'buyresult',
+      base: '',
+      fields: [
         {
-            name: "buyresult",
-            base: "",
-            fields: [{
-                name: "rex_received",
-                type: "asset"
-            }
-            ]
+          name: 'rex_received',
+          type: 'asset',
+        },
+      ],
+    }, {
+      name: 'orderresult',
+      base: '',
+      fields: [
+        {
+          name: 'owner',
+          type: 'name',
         }, {
-            name: "orderresult",
-            base: "",
-            fields: [{
-                name: "owner",
-                type: "name"
-            }, {
-                name: "proceeds",
-                type: "asset"
-            }
-            ]
-        }, {
-            name: "rentresult",
-            base: "",
-            fields: [{
-                name: "rented_tokens",
-                type: "asset"
-            }
-            ]
+          name: 'proceeds',
+          type: 'asset',
         },
+      ],
+    }, {
+      name: 'rentresult',
+      base: '',
+      fields: [
         {
-            name: "sellresult",
-            base: "",
-            fields: [{
-                name: "proceeds",
-                type: "asset"
-            }]
-        }
-    ],
-    actions: [
-        {
-            name: "buyresult",
-            type: "buyresult",
-            ricardian_contract: ""
+          name: 'rented_tokens',
+          type: 'asset',
         },
+      ],
+    },
+    {
+      name: 'sellresult',
+      base: '',
+      fields: [
         {
-            name: "orderresult",
-            type: "orderresult",
-            ricardian_contract: ""
-        },
-        {
-            name: "rentresult",
-            type: "rentresult",
-            ricardian_contract: ""
-        },
-        {
-            name: "sellresult",
-            type: "sellresult",
-            ricardian_contract: ""
-        }
-    ]
+          name: 'proceeds',
+          type: 'asset',
+        }],
+    },
+  ],
+  actions: [
+    {
+      name: 'buyresult',
+      type: 'buyresult',
+      ricardian_contract: '',
+    },
+    {
+      name: 'orderresult',
+      type: 'orderresult',
+      ricardian_contract: '',
+    },
+    {
+      name: 'rentresult',
+      type: 'rentresult',
+      ricardian_contract: '',
+    },
+    {
+      name: 'sellresult',
+      type: 'sellresult',
+      ricardian_contract: '',
+    },
+  ],
 };
 
 module.exports = {AbiDefinitions, RexAbi};

definitions/ecosystem_settings.js

@@ -1,37 +1,51 @@
-function addIndexer(chainName) {
-  return {
-    script: './launcher.js',
-    name: chainName + '-indexer',
-    namespace: chainName,
-    interpreter: 'node',
-    interpreter_args: ['--max-old-space-size=4096', '--trace-deprecation'],
-    autorestart: false,
-    kill_timeout: 3600,
-    watch: false,
-    time: true, // include timestamps in pm2 logs
-    env: {
-      CONFIG_JSON: 'chains/' + chainName + '.config.json',
-      TRACE_LOGS: 'false',
-    },
-  };
+function interpreterArgs(heap) {
+    const arr = ['--trace-deprecation', '--trace-warnings'];
+    if (heap) {
+        arr.push('--max-old-space-size=' + heap);
+    } else {
+        arr.push('--max-old-space-size=2048');
+    }
+    if (process.env.INSPECT) {
+        arr.push('--inspect');
+    }
+    return arr;
 }
 
-function addApiServer(chainName, threads) {
-  return {
-    script: './api/server.js',
-    name: chainName + '-api',
-    namespace: chainName,
-    node_args: ['--trace-deprecation'],
-    exec_mode: 'cluster',
-    merge_logs: true,
-    instances: threads,
-    autorestart: true,
-    exp_backoff_restart_delay: 100,
-    watch: false,
-    env: {
-      CONFIG_JSON: 'chains/' + chainName + '.config.json',
-    },
-  };
+function addIndexer(chainName, heap) {
+    return {
+        script: './launcher.js',
+        name: chainName + '-indexer',
+        namespace: chainName,
+        interpreter: 'node',
+        interpreter_args: interpreterArgs(heap),
+        autorestart: false,
+        kill_timeout: 3600,
+        watch: false,
+        time: true,
+        env: {
+            CONFIG_JSON: 'chains/' + chainName + '.config.json',
+            TRACE_LOGS: 'false',
+        },
+    };
+}
+
+function addApiServer(chainName, threads, heap) {
+    return {
+        script: './api/server.js',
+        name: chainName + '-api',
+        namespace: chainName,
+        node_args: interpreterArgs(heap),
+        exec_mode: 'cluster',
+        merge_logs: true,
+        instances: threads,
+        autorestart: true,
+        exp_backoff_restart_delay: 100,
+        watch: false,
+        time: true,
+        env: {
+            CONFIG_JSON: 'chains/' + chainName + '.config.json',
+        },
+    };
 }
 
 module.exports = {addIndexer, addApiServer};

definitions/index-lifecycle-policies.ts

@@ -1,5 +1,6 @@
 import {IlmPutLifecycle} from "@elastic/elasticsearch/api/requestParams";
 
+// noinspection JSUnusedGlobalSymbols
 export const ILPs: IlmPutLifecycle[] = [
     {
         policy: "hyperion-rollover",

definitions/index-templates.ts

@@ -1,9 +1,7 @@
 import {ConfigurationModule} from "../modules/config";
 
 const shards = 2;
-const replicas = 0;
 const refresh = "1s";
-let defaultLifecyclePolicy = "200G";
 
 export * from './index-lifecycle-policies';
 
@@ -15,507 +13,513 @@ const compression = "best_compression";
 const cm = new ConfigurationModule();
 const chain = cm.config.settings.chain;
 
-if (cm.config.settings.hot_warm_policy) {
-    defaultLifecyclePolicy = "hyperion-rollover";
-}
-
-if(cm.config.settings.custom_policy) {
-    defaultLifecyclePolicy = cm.config.settings.custom_policy;
+// update number of replicas if set to larger than 0
+let replicas = 0;
+if (cm.config.settings.es_replicas) {
+	replicas = cm.config.settings.es_replicas;
 }
 
 const defaultIndexSettings = {
-    "index": {
-        "number_of_shards": shards,
-        "refresh_interval": refresh,
-        "number_of_replicas": replicas,
-        "codec": compression
-    }
+	"index": {
+		"number_of_shards": shards,
+		"refresh_interval": refresh,
+		"number_of_replicas": replicas,
+		"codec": compression
+	}
 };
 
 const actionSettings = {
-    index: {
-        lifecycle: {
-            "name": defaultLifecyclePolicy,
-            "rollover_alias": chain + "-action"
-        },
-        codec: compression,
-        refresh_interval: refresh,
-        number_of_shards: shards * 2,
-        number_of_replicas: replicas,
-        sort: {
-            field: "global_sequence",
-            order: "desc"
-        }
-    }
+	index: {
+		codec: compression,
+		refresh_interval: refresh,
+		number_of_shards: shards * 2,
+		number_of_replicas: replicas,
+		sort: {
+			field: "global_sequence",
+			order: "desc"
+		}
+	}
 };
 
+// actionSettings.index["lifecycle"] = {
+//     "name": defaultLifecyclePolicy,
+//     "rollover_alias": chain + "-action"
+// };
+
 if (cm.config.settings.hot_warm_policy) {
-    actionSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
+	actionSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
 }
 
+const transferProps = {
+	"properties": {
+		"from": {"type": "keyword"},
+		"to": {"type": "keyword"},
+		"amount": {"type": "float"},
+		"symbol": {"type": "keyword"},
+		"memo": {"type": "text"}
+	}
+};
+
 export const action = {
-    order: 0,
-    index_patterns: [
-        chain + "-action-*"
-    ],
-    settings: actionSettings,
-    mappings: {
-        properties: {
-            "@timestamp": {"type": "date"},
-            "ds_error": {"type": "boolean"},
-            "global_sequence": {"type": "long"},
-            "account_ram_deltas.delta": {"type": "integer"},
-            "account_ram_deltas.account": {"type": "keyword"},
-            "act.authorization.permission": {"enabled": false},
-            "act.authorization.actor": {"type": "keyword"},
-            "act.account": {"type": "keyword"},
-            "act.name": {"type": "keyword"},
-            "act.data": {"enabled": false},
-            "block_num": {"type": "long"},
-            "action_ordinal": {"type": "long"},
-            "creator_action_ordinal": {"type": "long"},
-            "cpu_usage_us": {"type": "integer"},
-            "net_usage_words": {"type": "integer"},
-            "code_sequence": {"type": "integer"},
-            "abi_sequence": {"type": "integer"},
-            "trx_id": {"type": "keyword"},
-            "producer": {"type": "keyword"},
-            "notified": {"type": "keyword"},
-            "signatures": {"enabled": false},
-            "inline_count": {"type": "short"},
-            "max_inline": {"type": "short"},
-            "inline_filtered": {"type": "boolean"},
-            "receipts": {
-                "properties": {
-                    "global_sequence": {"type": "long"},
-                    "recv_sequence": {"type": "long"},
-                    "receiver": {"type": "keyword"},
-                    "auth_sequence": {
-                        "properties": {
-                            "account": {"type": "keyword"},
-                            "sequence": {"type": "long"}
-                        }
-                    }
-                }
-            },
+	order: 0,
+	index_patterns: [
+		chain + "-action-*"
+	],
+	settings: actionSettings,
+	mappings: {
+		properties: {
+			"@timestamp": {"type": "date"},
+			"ds_error": {"type": "boolean"},
+			"global_sequence": {"type": "long"},
+			"account_ram_deltas.delta": {"type": "integer"},
+			"account_ram_deltas.account": {"type": "keyword"},
+			"act.authorization.permission": {"enabled": false},
+			"act.authorization.actor": {"type": "keyword"},
+			"act.account": {"type": "keyword"},
+			"act.name": {"type": "keyword"},
+			"act.data": {"enabled": false},
+			"block_num": {"type": "long"},
+			"block_id": {"type": "keyword"},
+			"action_ordinal": {"type": "long"},
+			"creator_action_ordinal": {"type": "long"},
+			"cpu_usage_us": {"type": "integer"},
+			"net_usage_words": {"type": "integer"},
+			"code_sequence": {"type": "integer"},
+			"abi_sequence": {"type": "integer"},
+			"trx_id": {"type": "keyword"},
+			"producer": {"type": "keyword"},
+			"signatures": {"enabled": false},
+			"inline_count": {"type": "short"},
+			"max_inline": {"type": "short"},
+			"inline_filtered": {"type": "boolean"},
+			"receipts": {
+				"properties": {
+					"global_sequence": {"type": "long"},
+					"recv_sequence": {"type": "long"},
+					"receiver": {"type": "keyword"},
+					"auth_sequence": {
+						"properties": {
+							"account": {"type": "keyword"},
+							"sequence": {"type": "long"}
+						}
+					}
+				}
+			},
 
-            // eosio::newaccount
-            "@newaccount": {
-                "properties": {
-                    "active": {"type": "object"},
-                    "owner": {"type": "object"},
-                    "newact": {"type": "keyword"}
-                }
-            },
+			// eosio::newaccount
+			"@newaccount": {
+				"properties": {
+					"active": {"type": "object"},
+					"owner": {"type": "object"},
+					"newact": {"type": "keyword"}
+				}
+			},
 
-            // eosio::updateauth
-            "@updateauth": {
-                "properties": {
-                    "permission": {"type": "keyword"},
-                    "parent": {"type": "keyword"},
-                    "auth": {"type": "object"}
-                }
-            },
+			// eosio::updateauth
+			"@updateauth": {
+				"properties": {
+					"permission": {"type": "keyword"},
+					"parent": {"type": "keyword"},
+					"auth": {"type": "object"}
+				}
+			},
 
-            // *::transfer
-            "@transfer": {
-                "properties": {
-                    "from": {"type": "keyword"},
-                    "to": {"type": "keyword"},
-                    "amount": {"type": "float"},
-                    "symbol": {"type": "keyword"},
-                    "memo": {"type": "text"}
-                }
-            },
+			// *::transfer
+			"@transfer": transferProps,
 
-            // eosio::unstaketorex
-            "@unstaketorex": {
-                "properties": {
-                    "owner": {"type": "keyword"},
-                    "receiver": {"type": "keyword"},
-                    "amount": {"type": "float"}
-                }
-            },
+			// eosio::unstaketorex
+			"@unstaketorex": {
+				"properties": {
+					"owner": {"type": "keyword"},
+					"receiver": {"type": "keyword"},
+					"amount": {"type": "float"}
+				}
+			},
 
-            // eosio::buyrex
-            "@buyrex": {
-                "properties": {
-                    "from": {"type": "keyword"},
-                    "amount": {"type": "float"}
-                }
-            },
+			// eosio::buyrex
+			"@buyrex": {
+				"properties": {
+					"from": {"type": "keyword"},
+					"amount": {"type": "float"}
+				}
+			},
 
-            // eosio::buyram
-            "@buyram": {
-                "properties": {
-                    "payer": {"type": "keyword"},
-                    "receiver": {"type": "keyword"},
-                    "quant": {"type": "float"}
-                }
-            },
+			// eosio::buyram
+			"@buyram": {
+				"properties": {
+					"payer": {"type": "keyword"},
+					"receiver": {"type": "keyword"},
+					"quant": {"type": "float"}
+				}
+			},
 
-            // eosio::buyrambytes
-            "@buyrambytes": {
-                "properties": {
-                    "payer": {"type": "keyword"},
-                    "receiver": {"type": "keyword"},
-                    "bytes": {"type": "long"}
-                }
-            },
+			// eosio::buyrambytes
+			"@buyrambytes": {
+				"properties": {
+					"payer": {"type": "keyword"},
+					"receiver": {"type": "keyword"},
+					"bytes": {"type": "long"}
+				}
+			},
 
-            // eosio::delegatebw
-            "@delegatebw": {
-                "properties": {
-                    "from": {"type": "keyword"},
-                    "receiver": {"type": "keyword"},
-                    "stake_cpu_quantity": {"type": "float"},
-                    "stake_net_quantity": {"type": "float"},
-                    "transfer": {"type": "boolean"},
-                    "amount": {"type": "float"}
-                }
-            },
+			// eosio::delegatebw
+			"@delegatebw": {
+				"properties": {
+					"from": {"type": "keyword"},
+					"receiver": {"type": "keyword"},
+					"stake_cpu_quantity": {"type": "float"},
+					"stake_net_quantity": {"type": "float"},
+					"transfer": {"type": "boolean"},
+					"amount": {"type": "float"}
+				}
+			},
 
-            // eosio::undelegatebw
-            "@undelegatebw": {
-                "properties": {
-                    "from": {"type": "keyword"},
-                    "receiver": {"type": "keyword"},
-                    "unstake_cpu_quantity": {"type": "float"},
-                    "unstake_net_quantity": {"type": "float"},
-                    "amount": {"type": "float"}
-                }
-            }
-        }
-    }
+			// eosio::undelegatebw
+			"@undelegatebw": {
+				"properties": {
+					"from": {"type": "keyword"},
+					"receiver": {"type": "keyword"},
+					"unstake_cpu_quantity": {"type": "float"},
+					"unstake_net_quantity": {"type": "float"},
+					"amount": {"type": "float"}
+				}
+			}
+		}
+	}
 };
 
 const deltaSettings = {
-    "index": {
-        "lifecycle": {
-            "name": defaultLifecyclePolicy,
-            "rollover_alias": chain + "-delta"
-        },
-        "codec": compression,
-        "number_of_shards": shards * 2,
-        "refresh_interval": refresh,
-        "number_of_replicas": replicas,
-        "sort.field": ["block_num", "scope", "primary_key"],
-        "sort.order": ["desc", "asc", "asc"]
-    }
+	index: {
+		codec: compression,
+		number_of_shards: shards * 2,
+		refresh_interval: refresh,
+		number_of_replicas: replicas,
+		sort: {
+			field: ["block_num", "scope", "primary_key"],
+			order: ["desc", "asc", "asc"]
+		}
+	}
 };
 
+// deltaSettings.index["lifecycle"] = {
+//     "name": defaultLifecyclePolicy,
+//     "rollover_alias": chain + "-delta"
+// };
+
 if (cm.config.settings.hot_warm_policy) {
-    deltaSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
+	deltaSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
 }
 
 export const delta = {
-    "index_patterns": [chain + "-delta-*"],
-    "settings": deltaSettings,
-    "mappings": {
-        "properties": {
+	"index_patterns": [chain + "-delta-*"],
+	"settings": deltaSettings,
+	"mappings": {
+		"properties": {
 
-            // base fields
-            "@timestamp": {"type": "date"},
-            "ds_error": {"type": "boolean"},
-            "block_id": {"type": "keyword"},
-            "block_num": {"type": "long"},
-            "deleted_at": {"type": "long"},
-            "code": {"type": "keyword"},
-            "scope": {"type": "keyword"},
-            "table": {"type": "keyword"},
-            "payer": {"type": "keyword"},
-            "primary_key": {"type": "keyword"},
-            "data": {"enabled": false},
-            "value": {"enabled": false},
+			// base fields
+			"@timestamp": {"type": "date"},
+			"present": {"type": "byte"},
+			"ds_error": {"type": "boolean"},
+			"block_id": {"type": "keyword"},
+			"block_num": {"type": "long"},
+			"deleted_at": {"type": "long"},
+			"code": {"type": "keyword"},
+			"scope": {"type": "keyword"},
+			"table": {"type": "keyword"},
+			"payer": {"type": "keyword"},
+			"primary_key": {"type": "keyword"},
+			"data": {"enabled": false},
+			"value": {"enabled": false},
 
-            // eosio.msig::approvals
-            "@approvals.proposal_name": {"type": "keyword"},
-            "@approvals.provided_approvals": {"type": "object"},
-            "@approvals.requested_approvals": {"type": "object"},
+			// eosio.msig::approvals
+			"@approvals.proposal_name": {"type": "keyword"},
+			"@approvals.provided_approvals": {"type": "object"},
+			"@approvals.requested_approvals": {"type": "object"},
 
-            // eosio.msig::proposal
-            "@proposal.proposal_name": {"type": "keyword"},
-            "@proposal.transaction": {"enabled": false},
+			// eosio.msig::proposal
+			"@proposal.proposal_name": {"type": "keyword"},
+			"@proposal.transaction": {"enabled": false},
 
-            // *::accounts
-            "@accounts.amount": {"type": "float"},
-            "@accounts.symbol": {"type": "keyword"},
+			// *::accounts
+			"@accounts.amount": {"type": "float"},
+			"@accounts.symbol": {"type": "keyword"},
 
-            // eosio::voters
-            "@voters.is_proxy": {"type": "boolean"},
-            "@voters.producers": {"type": "keyword"},
-            "@voters.last_vote_weight": {"type": "double"},
-            "@voters.proxied_vote_weight": {"type": "double"},
-            "@voters.staked": {"type": "float"},
-            "@voters.proxy": {"type": "keyword"},
+			// eosio::voters
+			"@voters.is_proxy": {"type": "boolean"},
+			"@voters.producers": {"type": "keyword"},
+			"@voters.last_vote_weight": {"type": "double"},
+			"@voters.proxied_vote_weight": {"type": "double"},
+			"@voters.staked": {"type": "float"},
+			"@voters.proxy": {"type": "keyword"},
 
-            // eosio::producers
-            "@producers.total_votes": {"type": "double"},
-            "@producers.is_active": {"type": "boolean"},
-            "@producers.unpaid_blocks": {"type": "long"},
+			// eosio::producers
+			"@producers.total_votes": {"type": "double"},
+			"@producers.is_active": {"type": "boolean"},
+			"@producers.unpaid_blocks": {"type": "long"},
 
-            // eosio::global
-            "@global": {
-                "properties": {
-                    "last_name_close": {"type": "date"},
-                    "last_pervote_bucket_fill": {"type": "date"},
-                    "last_producer_schedule_update": {"type": "date"},
-                    "perblock_bucket": {"type": "double"},
-                    "pervote_bucket": {"type": "double"},
-                    "total_activated_stake": {"type": "double"},
-                    "total_voteshare_change_rate": {"type": "double"},
-                    "total_unpaid_voteshare": {"type": "double"},
-                    "total_producer_vote_weight": {"type": "double"},
-                    "total_ram_bytes_reserved": {"type": "long"},
-                    "total_ram_stake": {"type": "long"},
-                    "total_unpaid_blocks": {"type": "long"},
-                }
-            }
-        }
-    }
+			// eosio::global
+			"@global": {
+				"properties": {
+					"last_name_close": {"type": "date"},
+					"last_pervote_bucket_fill": {"type": "date"},
+					"last_producer_schedule_update": {"type": "date"},
+					"perblock_bucket": {"type": "double"},
+					"pervote_bucket": {"type": "double"},
+					"total_activated_stake": {"type": "double"},
+					"total_voteshare_change_rate": {"type": "double"},
+					"total_unpaid_voteshare": {"type": "double"},
+					"total_producer_vote_weight": {"type": "double"},
+					"total_ram_bytes_reserved": {"type": "long"},
+					"total_ram_stake": {"type": "long"},
+					"total_unpaid_blocks": {"type": "long"},
+				}
+			}
+		}
+	}
 };
 
 export const abi = {
-    "index_patterns": [chain + "-abi-*"],
-    "settings": defaultIndexSettings,
-    "mappings": {
-        "properties": {
-            "@timestamp": {"type": "date"},
-            "block": {"type": "long"},
-            "account": {"type": "keyword"},
-            "abi": {"enabled": false},
-            "abi_hex": {"enabled": false},
-            "actions": {"type": "keyword"},
-            "tables": {"type": "keyword"}
-        }
-    }
+	"index_patterns": [chain + "-abi-*"],
+	"settings": defaultIndexSettings,
+	"mappings": {
+		"properties": {
+			"@timestamp": {"type": "date"},
+			"block": {"type": "long"},
+			"account": {"type": "keyword"},
+			"abi": {"enabled": false},
+			"abi_hex": {"enabled": false},
+			"actions": {"type": "keyword"},
+			"tables": {"type": "keyword"}
+		}
+	}
 };
 
 export const permissionLink = {
-    "index_patterns": [chain + "-link-*"],
-    "settings": defaultIndexSettings,
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "@timestamp": {"type": "date"},
-            "present": {"type": "boolean"},
-            "account": {"type": "keyword"},
-            "code": {"type": "keyword"},
-            "action": {"type": "keyword"},
-            "permission": {"type": "keyword"}
-        }
-    }
+	"index_patterns": [chain + "-link-*"],
+	"settings": defaultIndexSettings,
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"@timestamp": {"type": "date"},
+			"present": {"type": "byte"},
+			"account": {"type": "keyword"},
+			"code": {"type": "keyword"},
+			"action": {"type": "keyword"},
+			"permission": {"type": "keyword"}
+		}
+	}
 };
 
 export const permission = {
-    "index_patterns": [chain + "-perm-*"],
-    "settings": defaultIndexSettings,
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "present": {"type": "boolean"},
-            "owner": {"type": "keyword"},
-            "name": {"type": "keyword"},
-            "parent": {"type": "keyword"},
-            "last_updated": {"type": "date"},
-            "auth": {"type": "object"}
-        }
-    }
+	"index_patterns": [chain + "-perm-*"],
+	"settings": defaultIndexSettings,
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"present": {"type": "byte"},
+			"owner": {"type": "keyword"},
+			"name": {"type": "keyword"},
+			"parent": {"type": "keyword"},
+			"last_updated": {"type": "date"},
+			"auth": {"type": "object"}
+		}
+	}
 };
 
 export const resourceLimits = {
-    "index_patterns": [chain + "-reslimits-*"],
-    "settings": defaultIndexSettings,
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "@timestamp": {"type": "date"},
-            "owner": {"type": "keyword"},
-            "total_weight": {"type": "long"},
-            "net_weight": {"type": "long"},
-            "cpu_weight": {"type": "long"},
-            "ram_bytes": {"type": "long"}
-        }
-    }
+	"index_patterns": [chain + "-reslimits-*"],
+	"settings": defaultIndexSettings,
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"@timestamp": {"type": "date"},
+			"owner": {"type": "keyword"},
+			"total_weight": {"type": "long"},
+			"net_weight": {"type": "long"},
+			"cpu_weight": {"type": "long"},
+			"ram_bytes": {"type": "long"}
+		}
+	}
 };
 
 export const generatedTransaction = {
-    "index_patterns": [chain + "-gentrx-*"],
-    "settings": defaultIndexSettings,
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "@timestamp": {"type": "date"},
-            "sender": {"type": "keyword"},
-            "sender_id": {"type": "keyword"},
-            "payer": {"type": "keyword"},
-            "trx_id": {"type": "keyword"},
-            "actions": {"enabled": false},
-            "packed_trx": {"enabled": false}
-        }
-    }
+	"index_patterns": [chain + "-gentrx-*"],
+	"settings": defaultIndexSettings,
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"@timestamp": {"type": "date"},
+			"sender": {"type": "keyword"},
+			"sender_id": {"type": "keyword"},
+			"payer": {"type": "keyword"},
+			"trx_id": {"type": "keyword"},
+			"actions": {"enabled": false},
+			"packed_trx": {"enabled": false}
+		}
+	}
 };
 
 export const failedTransaction = {
-    "index_patterns": [chain + "-trxerr-*"],
-    "settings": defaultIndexSettings,
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "@timestamp": {"type": "date"},
-            "status": {"type": "short"}
-        }
-    }
+	"index_patterns": [chain + "-trxerr-*"],
+	"settings": defaultIndexSettings,
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"@timestamp": {"type": "date"},
+			"status": {"type": "short"}
+		}
+	}
 };
 
 export const resourceUsage = {
-    "index_patterns": [chain + "-userres-*"],
-    "settings": defaultIndexSettings,
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "@timestamp": {"type": "date"},
-            "owner": {"type": "keyword"},
-            "net_used": {"type": "long"},
-            "net_total": {"type": "long"},
-            "net_pct": {"type": "float"},
-            "cpu_used": {"type": "long"},
-            "cpu_total": {"type": "long"},
-            "cpu_pct": {"type": "float"},
-            "ram": {"type": "long"}
-        }
-    }
+	"index_patterns": [chain + "-userres-*"],
+	"settings": defaultIndexSettings,
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"@timestamp": {"type": "date"},
+			"owner": {"type": "keyword"},
+			"net_used": {"type": "long"},
+			"net_total": {"type": "long"},
+			"net_pct": {"type": "float"},
+			"cpu_used": {"type": "long"},
+			"cpu_total": {"type": "long"},
+			"cpu_pct": {"type": "float"},
+			"ram": {"type": "long"}
+		}
+	}
 };
 
 
 export const logs = {
-    "index_patterns": [chain + "-logs-*"],
-    "settings": defaultIndexSettings
+	"index_patterns": [chain + "-logs-*"],
+	"settings": defaultIndexSettings
 };
 
 export const block = {
-    "index_patterns": [chain + "-block-*"],
-    "settings": {
-        "index": {
-            "codec": compression,
-            "number_of_shards": shards,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas,
-            "sort.field": "block_num",
-            "sort.order": "desc"
-        }
-    },
-    "mappings": {
-        "properties": {
-            "@timestamp": {"type": "date"},
-            "block_num": {"type": "long"},
-            "block_id": {"type": "keyword"},
-            "prev_id": {"type": "keyword"},
-            "producer": {"type": "keyword"},
-            "new_producers.producers.block_signing_key": {"enabled": false},
-            "new_producers.producers.producer_name": {"type": "keyword"},
-            "new_producers.version": {"type": "long"},
-            "schedule_version": {"type": "double"},
-            "cpu_usage": {"type": "integer"},
-            "net_usage": {"type": "integer"}
-        }
-    }
+	"index_patterns": [chain + "-block-*"],
+	"settings": {
+		"index": {
+			"codec": compression,
+			"number_of_shards": shards,
+			"refresh_interval": refresh,
+			"number_of_replicas": replicas,
+			"sort.field": "block_num",
+			"sort.order": "desc"
+		}
+	},
+	"mappings": {
+		"properties": {
+			"@timestamp": {"type": "date"},
+			"block_num": {"type": "long"},
+			"block_id": {"type": "keyword"},
+			"prev_id": {"type": "keyword"},
+			"producer": {"type": "keyword"},
+			"new_producers.producers.block_signing_key": {"enabled": false},
+			"new_producers.producers.producer_name": {"type": "keyword"},
+			"new_producers.version": {"type": "long"},
+			"schedule_version": {"type": "double"},
+			"cpu_usage": {"type": "integer"},
+			"net_usage": {"type": "integer"}
+		}
+	}
 };
 
 export const tableProposals = {
-    "index_patterns": [chain + "-table-proposals-*"],
-    "settings": {
-        "index": {
-            "codec": compression,
-            "number_of_shards": shards,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas,
-            "sort.field": "block_num",
-            "sort.order": "desc"
-        }
-    },
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "present": {"type": "boolean"},
-            "proposal_name": {"type": "keyword"},
-            "requested_approvals": {"type": "object"},
-            "provided_approvals": {"type": "object"},
-            "executed": {"type": "boolean"}
-        }
-    }
+	"index_patterns": [chain + "-table-proposals-*"],
+	"settings": {
+		"index": {
+			"codec": compression,
+			"number_of_shards": shards,
+			"refresh_interval": refresh,
+			"number_of_replicas": replicas,
+			"sort.field": "block_num",
+			"sort.order": "desc"
+		}
+	},
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"present": {"type": "byte"},
+			"proposal_name": {"type": "keyword"},
+			"requested_approvals": {"type": "object"},
+			"provided_approvals": {"type": "object"},
+			"executed": {"type": "boolean"}
+		}
+	}
 };
 
 export const tableAccounts = {
-    "index_patterns": [chain + "-table-accounts-*"],
-    "settings": {
-        "index": {
-            "codec": compression,
-            "number_of_shards": shards,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas,
-            "sort.field": "amount",
-            "sort.order": "desc"
-        }
-    },
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "present": {"type": "boolean"},
-            "code": {"type": "keyword"},
-            "scope": {"type": "keyword"},
-            "amount": {"type": "float"},
-            "symbol": {"type": "keyword"}
-        }
-    }
+	"index_patterns": [chain + "-table-accounts-*"],
+	"settings": {
+		"index": {
+			"codec": compression,
+			"number_of_shards": shards,
+			"refresh_interval": refresh,
+			"number_of_replicas": replicas,
+			"sort.field": "amount",
+			"sort.order": "desc"
+		}
+	},
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"present": {"type": "byte"},
+			"code": {"type": "keyword"},
+			"scope": {"type": "keyword"},
+			"amount": {"type": "float"},
+			"symbol": {"type": "keyword"}
+		}
+	}
 };
 
+// noinspection JSUnusedGlobalSymbols
 export const tableDelBand = {
-    "index_patterns": [chain + "-table-delband-*"],
-    "settings": {
-        "index": {
-            "codec": compression,
-            "number_of_shards": shards,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas,
-            "sort.field": "total_weight",
-            "sort.order": "desc"
-        }
-    },
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "from": {"type": "keyword"},
-            "to": {"type": "keyword"},
-            "total_weight": {"type": "float"},
-            "net_weight": {"type": "float"},
-            "cpu_weight": {"type": "float"}
-        }
-    }
+	"index_patterns": [chain + "-table-delband-*"],
+	"settings": {
+		"index": {
+			"codec": compression,
+			"number_of_shards": shards,
+			"refresh_interval": refresh,
+			"number_of_replicas": replicas,
+			"sort.field": "total_weight",
+			"sort.order": "desc"
+		}
+	},
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"from": {"type": "keyword"},
+			"to": {"type": "keyword"},
+			"total_weight": {"type": "float"},
+			"net_weight": {"type": "float"},
+			"cpu_weight": {"type": "float"}
+		}
+	}
 };
 
 export const tableVoters = {
-    "index_patterns": [chain + "-table-voters-*"],
-    "settings": {
-        "index": {
-            "codec": compression,
-            "number_of_shards": shards,
-            "refresh_interval": refresh,
-            "number_of_replicas": replicas,
-            "sort.field": "last_vote_weight",
-            "sort.order": "desc"
-        }
-    },
-    "mappings": {
-        "properties": {
-            "block_num": {"type": "long"},
-            "voter": {"type": "keyword"},
-            "producers": {"type": "keyword"},
-            "last_vote_weight": {"type": "double"},
-            "is_proxy": {"type": "boolean"},
-            "proxied_vote_weight": {"type": "double"},
-            "staked": {"type": "double"},
-            "proxy": {"type": "keyword"}
-        }
-    }
+	"index_patterns": [chain + "-table-voters-*"],
+	"settings": {
+		"index": {
+			"codec": compression,
+			"number_of_shards": shards,
+			"refresh_interval": refresh,
+			"number_of_replicas": replicas,
+			"sort.field": "last_vote_weight",
+			"sort.order": "desc"
+		}
+	},
+	"mappings": {
+		"properties": {
+			"block_num": {"type": "long"},
+			"voter": {"type": "keyword"},
+			"producers": {"type": "keyword"},
+			"last_vote_weight": {"type": "double"},
+			"is_proxy": {"type": "boolean"},
+			"proxied_vote_weight": {"type": "double"},
+			"staked": {"type": "double"},
+			"proxy": {"type": "keyword"}
+		}
+	}
 };

docker/elasticsearch/config/elasticsearch.yml

@@ -0,0 +1,5 @@
+cluster.name: docker-cluster
+network.host: 0.0.0.0
+bootstrap.memory_lock: true
+discovery.type: single-node
+node.name: es01

docker/elasticsearch/config/jvm.options.d/heap.options

@@ -0,0 +1,2 @@
+-Xms8g
+-Xmx8g

docker/eosio/Dockerfile

@@ -1,11 +1,5 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get upgrade -y && apt-get autoremove && apt-get install -y wget netcat
-
-RUN wget -nv https://github.com/eosio/eos/releases/download/v2.0.5/eosio_2.0.5-1-ubuntu-18.04_amd64.deb
-RUN apt-get install -y ./eosio_2.0.5-1-ubuntu-18.04_amd64.deb
-
-RUN useradd -m -s /bin/bash eosio
-USER eosio
-
-EXPOSE 8080
+FROM ubuntu:20.04
+LABEL version="2.1.0" description="EOSIO 2.1.0"
+RUN apt-get update && apt-get install -y wget
+RUN wget -nv https://github.com/EOSIO/eos/releases/download/v2.1.0/eosio_2.1.0-1-ubuntu-20.04_amd64.deb
+RUN apt-get install -y ./eosio_2.1.0-1-ubuntu-20.04_amd64.deb && rm -f ./eosio_2.1.0-1-ubuntu-20.04_amd64.deb

docker/eosio/config/config.ini

@@ -1,28 +1,14 @@
-# Enable block production, even if the chain is stale. (eosio::producer_plugin)
+### PRODUCER PLUGIN ###
 enable-stale-production = true
-
-# ID of producer controlled by this node (e.g. inita; may specify multiple times) (eosio::producer_plugin)
 producer-name = eosio
 
-# print contract's output to console (eosio::chain_plugin)
-contracts-console = true
-
-# The local IP and port to listen for incoming http connections; set blank to disable. (eosio::http_plugin)
-http-server-address = 0.0.0.0:8888
-
-# If set to false, then any incoming "Host" header is considered valid (eosio::http_plugin)
-http-validate-host = false
-
-# enable trace history (eosio::state_history_plugin)
+### STATE HISTORY PLUGIN ###
+plugin = eosio::state_history_plugin
+state-history-endpoint = 0.0.0.0:8080
 trace-history = true
-
-# enable chain state history (eosio::state_history_plugin)
 chain-state-history = true
 
-# the endpoint upon which to listen for incoming connections. Caution: only expose this port to your internal network. (eosio::state_history_plugin)
-state-history-endpoint = 0.0.0.0:8080
-
-# Plugin(s) to enable, may be specified multiple times
-plugin = eosio::producer_api_plugin
+### CHAIN API PLUGIN ###
 plugin = eosio::chain_api_plugin
-plugin = eosio::state_history_plugin
+http-server-address = 0.0.0.0:8888
+http-validate-host = false

docker/hyperion/Dockerfile

@@ -1,15 +1,6 @@
-FROM ubuntu:18.04
-
-RUN apt-get update && apt-get upgrade -y && apt-get autoremove && apt-get install -y build-essential git curl netcat
-
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-RUN apt-get install -y nodejs && npm install pm2 -g && git clone https://github.com/eosrio/hyperion-history-api.git
-
-RUN useradd -m -s /bin/bash hyperion && chown -R hyperion:hyperion /hyperion-history-api
-USER hyperion
+FROM node:16.5
+RUN npm install pm2 -g
+RUN git clone https://github.com/eosrio/hyperion-history-api.git
 WORKDIR /hyperion-history-api
-
-RUN npm install
-
-EXPOSE 7001
-
+RUN git checkout dev-3.3
+RUN npm install --production

docker/rabbitmq/data/.gitignore

@@ -1,5 +0,0 @@
-# Ignore everything in this directory
-*
-
-# Except this file
-!.gitignore

docker/rabbitmq/rabbitmq.conf

@@ -0,0 +1,4 @@
+default_vhost = hyperion
+default_user = hyperion
+default_pass = hyp123456
+cluster_name = hyp-docker

docker/template-docker-compose.yml

@@ -1,111 +0,0 @@
-version: '3.8'
-
-services:
-  redis:
-    container_name: redis
-    image: redis:5.0.9-alpine
-    restart: on-failure
-    networks:
-      - hyperion
-
-  rabbitmq:
-    container_name: rabbitmq
-    image: rabbitmq:3.8.3-management
-    restart: on-failure
-    environment:
-      - RABBITMQ_DEFAULT_USER=username
-      - RABBITMQ_DEFAULT_PASS=password
-      - RABBITMQ_DEFAULT_VHOST=/hyperion
-    volumes:
-      - ./rabbitmq/data:/var/lib/rabbitmq
-    ports:
-      - 15672:15672
-    networks:
-      - hyperion
-
-  elasticsearch:
-    container_name: elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.7.1
-    restart: on-failure
-    environment:
-      - discovery.type=single-node
-      - cluster.name=es-cluster
-      - node.name=es01
-      - bootstrap.memory_lock=true
-      - xpack.security.enabled=true
-      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
-      - ELASTIC_USERNAME=elastic
-      - ELASTIC_PASSWORD=password
-    volumes:
-      - ./elasticsearch/data:/usr/share/elasticsearch/data
-    ports:
-      - 9200:9200
-    networks:
-      - hyperion
-
-  kibana:
-    container_name: kibana
-    image: docker.elastic.co/kibana/kibana:7.7.1
-    restart: on-failure
-    environment:
-      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
-      - ELASTICSEARCH_USERNAME=elastic
-      - ELASTICSEARCH_PASSWORD=password
-    ports:
-      - 5601:5601
-    networks:
-      - hyperion
-    depends_on:
-      - elasticsearch
-
-  eosio-node:
-    container_name: eosio-node
-    image: eosrio/hyperion:eosio-2.0.5
-    volumes:
-      - ./eosio/data/:/home/eosio/data/
-      - ./eosio/config/:/home/eosio/config/
-      - ./scripts/:/home/eosio/scripts/
-    ports:
-      - 8888:8888
-    networks:
-      - hyperion
-    command: bash -c "/home/eosio/scripts/run-nodeos.sh ${SCRIPT:-false} ${SNAPSHOT:-""}"
-
-  hyperion-indexer:
-    container_name: hyperion-indexer
-    image: eosrio/hyperion:hyperion-3.1.0-beta.3
-    restart: on-failure
-    depends_on:
-      - elasticsearch
-      - redis
-      - rabbitmq
-      - eosio-node
-    volumes:
-      - ./hyperion/config/connections.json:/hyperion-history-api/connections.json
-      - ./hyperion/config/ecosystem.config.js:/hyperion-history-api/ecosystem.config.js
-      - ./hyperion/config/chains/:/hyperion-history-api/chains/
-      - ./scripts/:/home/hyperion/scripts/
-    networks:
-      - hyperion
-    command: bash -c "/home/hyperion/scripts/run-hyperion.sh ${SCRIPT:-false} eos-indexer"
-
-  hyperion-api:
-    container_name: hyperion-api
-    image: eosrio/hyperion:hyperion-3.0
-    restart: on-failure
-    ports:
-      - 7000:7000
-    depends_on:
-      - hyperion-indexer
-    volumes:
-      - ./hyperion/config/connections.json:/hyperion-history-api/connections.json
-      - ./hyperion/config/ecosystem.config.js:/hyperion-history-api/ecosystem.config.js
-      - ./hyperion/config/chains/:/hyperion-history-api/chains/
-      - ./scripts/:/home/hyperion/scripts/
-    networks:
-      - hyperion
-    command: bash -c "/home/hyperion/scripts/run-hyperion.sh ${SCRIPT:-false} eos-api"
-
-networks:
-  hyperion:
-    driver: bridge

ecosystem.config.js

@@ -0,0 +1,23 @@
+const {addApiServer, addIndexer} = require('./definitions/ecosystem_settings');
+const {readdirSync, readFileSync} = require("fs");
+const path = require('path');
+
+const apps = [];
+const chainsRoot = path.join(path.resolve(), 'chains');
+readdirSync(chainsRoot)
+    .filter(f => f.endsWith('.config.json'))
+    .forEach(value => {
+        const configFile = readFileSync(path.join(chainsRoot, value))
+        const config = JSON.parse(configFile.toString());
+        const chainName = config.settings.chain;
+        if (config.api.enabled) {
+            const apiHeap = config.api.node_max_old_space_size;
+            apps.push(addApiServer(chainName, config.api.pm2_scaling, apiHeap));
+        }
+        if (config.indexer.enabled) {
+            const indexerHeap = config.indexer.node_max_old_space_size;
+            apps.push(addIndexer(chainName, indexerHeap));
+        }
+    });
+
+module.exports = {apps};