v3.3.9

- fix readers stopping before having finished

.gitignore

@@ -31,7 +31,6 @@ node_modules/
 
 .idea
 *.pem
-ecosystem.config.js
 package-lock.json
 connections.json
 connections.json2
@@ -75,6 +74,12 @@ modules/**/*.js.map
 addons/**/*.js
 addons/**/*.js.map
 
+scripts/hpm.js
+scripts/hpm.js.map
+
+scripts/hyp-config.js
+scripts/hyp-config.js.map
+
 plugins/.state.json
 plugins/repos
 plugins/*.js

README.md

@@ -1,54 +1,99 @@
-# Hyperion History API
+<!--suppress HtmlUnknownTarget, HtmlDeprecatedAttribute -->
+<br></br>
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://eosrio.io/hyperion-white.png">
+    <img alt="Hyperion Logo"
+         src="https://eosrio.io/hyperion.png">
+  </picture>
+</p>
 
-<img alt="hyperion logo" height="64" src="https://eosrio.io/hyperion.png">
-<br/>
-Scalable Full History API Solution for EOSIO based blockchains
+<h4 align="center">
+    Scalable Full History API Solution for 
+    <a href="https://antelope.io">
+        Antelope
+    </a>
+    (former EOSIO) based blockchains
+</h4>
 
-Made with ♥ by [EOS Rio](https://eosrio.io/)
+<br>
 
-Official documentation: https://hyperion.docs.eosrio.io/
+Made with ♥ by [Rio Blocks / EOS Rio](https://rioblocks.io/?lang=en)
+
+### [Official documentation](https://hyperion.docs.eosrio.io)
+
+### How to use
+
+ - [For Providers](https://hyperion.docs.eosrio.io/manual_installation/)
+
+ - [For Developers](https://hyperion.docs.eosrio.io/howtouse/)
+
+### Official plugins:
+
+- [Hyperion Lightweight Explorer](https://github.com/eosrio/hyperion-explorer-plugin)
 
 ### 1. Overview
 
-Hyperion is a full history solution for indexing, storing and retrieving EOSIO blockchain`s historical data. EOSIO protocol is highly scalable reaching up to tens of thousands of transactions per second demanding high performance indexing and optimized storage and querying solutions. Hyperion is developed to tackle those challenges providing open source software to be operated by block producers, infrastructure providers and dApp developers.
-
-Focused on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers, Hyperion implements an improved data structure
-actions are stored in a flattened format
-a parent field is added to the inline actions to point to the parent global sequence
-if the inline action data is identical to the parent it is considered a notification and thus removed from the database
-no blocks or transaction data is stored, all information can be reconstructed from actions
+Hyperion is a full history solution for indexing, storing and retrieving Antelope blockchain's historical data.
+Antelope protocol is highly scalable reaching up to tens of thousands of transactions per second demanding high
+performance indexing and optimized storage and querying solutions. Hyperion is developed to tackle those challenges
+providing open source software to be operated by block producers, infrastructure providers and dApp developers.
 
+Focused on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers,
+Hyperion implements an improved data structure. Actions are stored in a flattened format, transaction ids are added to 
+all inline actions, allowing to group by transaction without storing a full transaction index. Besides that if the inline 
+action data is identical to the parent, it is considered a notification and thus removed from the database.
+No full block or transaction data is stored, all information can be reconstructed from actions and deltas, only a block 
+header index is stored.
 
 ### 2. Architecture
-The following components are required in order to have a fully functional Hyperion API deployment, 
-for small use cases its absolutely fine to run all components on a single machine. For larger chains and 
-production environments we recommend setting them up into different servers under a high-speed local network.
+
+The following components are required in order to have a fully functional Hyperion API deployment.
+* For small use cases, it is absolutely fine to run all components on a single machine.
+* For larger chains and production environments, we recommend setting them up into different servers under a high-speed local network.
 
 #### 2.1 Elasticsearch Cluster
+
 The ES cluster is responsible for storing all indexed data.
-Direct access to the Hyperion API and Indexer must be provided. We recommend nodes in the 
-cluster to have at least 32 GB of RAM and 8 cpu cores. SSD/NVME drives are recommended for 
-maximum indexing throughput. For production environments a multi-node cluster is highly recommended.
+Direct access to the Hyperion API and Indexer must be provided. We recommend nodes in the
+cluster to have at least 32 GB of RAM and 8 cpu cores. SSD/NVME drives are recommended for
+maximum indexing throughput, although HDDs can be used for cold storage nodes.
+For production environments, a multi-node cluster is highly recommended.
 
 #### 2.2 Hyperion Indexer
+
 The Indexer is a Node.js based app that process data from the state history plugin and allows it to be indexed.
-The PM2 process manager is used to launch and operate the indexer. The configuration flexibility is very extensive, 
-so system recommendations will depend on the use case and data load. It will require access to at least one ES node, 
+The [PM2 process manager](https://pm2.keymetrics.io) is used to launch and operate the indexer. The configuration
+flexibility is very extensive,
+so system recommendations will depend on the use case and data load. It will require access to at least one ES node,
 RabbitMQ and the state history node.
 
 #### 2.3 Hyperion API
+
 Parallelizable API server that provides the V2 and V1 (legacy history plugin) endpoints.
-It is launched by PM2 and can also operate in cluster mode. It requires direct access to 
+It is launched by PM2 and can also operate in cluster mode. It requires direct access to
 at least one ES node for the queries and all other services for full healthcheck
 
 #### 2.4 RabbitMQ
-Use as messaging queue and data transport between the indexer stages
 
-#### 2.5 EOSIO State History
-Nodeos plugin used to collect action traces and state deltas. Provides data via websocket to the indexer
+Used as messaging queue and data transport between the indexer stages and for real-time data streaming
 
-### 3. How to use
+#### 2.5 Redis
 
-#### 3.1 [For Providers](https://eosrio.github.io/hyperion-docs/quickstart/)
+Used for transient data storage across processes and for the preemptive transaction caching used on
+the `v2/history/get_transaction` and `v2/history/check_transaction` endpoints
 
-#### 3.2 [For Developers](https://eosrio.github.io/hyperion-docs/howtouse/)
+#### 2.6 Leap State History
+
+[Leap / Nodeos](https://github.com/AntelopeIO/leap/tree/main/plugins/state_history_plugin) plugin used
+to collect action traces and state deltas. Provides data via websocket to the indexer
+
+#### 2.7 Hyperion Stream Client (optional)
+
+Web and Node.js client for real-time streaming on enabled hyperion
+providers. [Documentation](https://hyperion.docs.eosrio.io/stream_client/)
+
+#### 2.8 Hyperion Plugins (optional)
+
+Hyperion includes a flexible plugin architecture to allow further customization.
+Plugins are managed by the `hpm` (hyperion plugin manager) command line tool.

api/config/open_api.ts

@@ -4,6 +4,7 @@ export function generateOpenApiConfig(config: HyperionConfig) {
     const packageData = require('../../package');
     const health_link = `https://${config.api.server_name}/v2/health`;
     const explorer_link = `https://${config.api.server_name}/v2/explore`;
+
     let description = `
 <img height="64" src="https://eosrio.io/hyperion.png">
 ### Scalable Full History API Solution for EOSIO based blockchains
@@ -13,9 +14,15 @@ export function generateOpenApiConfig(config: HyperionConfig) {
 #### Provided by [${config.api.provider_name}](${config.api.provider_url})
 #### Health API: <a target="_blank" href="${health_link}">${health_link}</a>
 `;
-    if(config.api.enable_explorer) {
-        description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
+
+    if (config.plugins) {
+        if (config.plugins.explorer) {
+            if (config.plugins.explorer.enabled) {
+                description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
+            }
+        }
     }
+
     return {
         routePrefix: '/v2/docs',
         exposeRoute: true,

api/helpers/cacheManager.ts

@@ -0,0 +1,86 @@
+import {HyperionConfig} from "../../interfaces/hyperionConfig";
+import {createHash} from "crypto";
+import {FastifyRequest} from "fastify";
+
+export interface CacheConfig {
+    ttl: number;
+}
+
+export interface CachedEntry {
+    data: string;
+    exp: number;
+}
+
+export class CacheManager {
+
+    v1CacheConfigs: Map<string, CacheConfig> = new Map<string, CacheConfig>();
+    v1Caches: Map<string, Map<string, CachedEntry>> = new Map<string, Map<string, CachedEntry>>();
+
+    constructor(conf: HyperionConfig) {
+        if (conf.api.v1_chain_cache) {
+            conf.api.v1_chain_cache.forEach(value => {
+                this.v1CacheConfigs.set(value.path, {
+                    ttl: value.ttl
+                });
+                this.v1Caches.set(value.path, new Map<string, CachedEntry>());
+            });
+        }
+        setInterval(() => {
+            try {
+                // remove expired entries
+                let removeCount = 0;
+                const now = Date.now();
+                this.v1Caches.forEach((pathCacheMap: Map<string, CachedEntry>, pathKey: string) => {
+                    pathCacheMap.forEach((cache: CachedEntry, entryKey: string, map: Map<string, CachedEntry>) => {
+                        if (cache.exp < now) {
+                            map.delete(entryKey);
+                            removeCount++;
+                        }
+                    });
+                });
+                if (removeCount > 0) {
+                    console.log(`${removeCount} expired cache entries removed`);
+                }
+            } catch (e) {
+                console.log(e);
+            }
+        }, 5000);
+    }
+
+    hashPayload(input: string): string {
+        return createHash('sha256').update(input).digest().toString('hex');
+    }
+
+    setCachedData(hash: string, path: string, payload: any): void {
+        if (this.v1CacheConfigs.has(path) && this.v1Caches.has(path)) {
+            this.v1Caches.get(path).set(hash, {
+                data: payload,
+                exp: this.v1CacheConfigs.get(path).ttl + Date.now()
+            });
+        }
+    }
+
+    getCachedData(request: FastifyRequest): [string | null, string, string] {
+        const urlParts = request.url.split("?");
+        const pathComponents = urlParts[0].split('/');
+        const path = pathComponents.at(-1);
+        let payload = '';
+        if (request.method === 'POST') {
+            payload = JSON.stringify(request.body);
+        } else if (request.method === 'GET') {
+            payload = request.url;
+        }
+        const hashedString = this.hashPayload(payload);
+        if (this.v1Caches.has(path)) {
+            const entry = this.v1Caches.get(path).get(hashedString);
+            if (entry && entry.exp > Date.now()) {
+                return [entry.data, hashedString, path];
+            } else {
+                return [null, hashedString, path];
+            }
+        } else {
+            return [null, hashedString, path];
+        }
+    }
+
+}

api/plugins.ts

@@ -3,11 +3,11 @@ import {IncomingMessage, Server, ServerResponse} from "http";
 
 // fastify plugins
 import * as fastifyElasticsearch from 'fastify-elasticsearch';
-import fastifySwagger from 'fastify-swagger';
-import fastifyCors from 'fastify-cors';
-import fastifyFormbody from 'fastify-formbody';
-import fastifyRedis from 'fastify-redis';
-import fastifyRateLimit from 'fastify-rate-limit';
+import fastifySwagger from '@fastify/swagger';
+import fastifyCors from '@fastify/cors';
+import formBodyPlugin from '@fastify/formbody';
+import fastifyRedis from '@fastify/redis';
+import fastifyRateLimit from '@fastify/rate-limit';
 
 // custom plugins
 import fastify_eosjs from "./plugins/fastify-eosjs";
@@ -21,7 +21,7 @@ export function registerPlugins(server: Fastify.FastifyInstance<Server, Incoming
 
     server.register(fastifyCors);
 
-    server.register(fastifyFormbody);
+    server.register(formBodyPlugin);
 
     server.register(fastifyRedis, params.fastify_redis);
 

api/routes.ts

@@ -2,118 +2,120 @@ import {join} from "path";
 import {FastifyError, FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {createReadStream} from "fs";
 import {addSharedSchemas, handleChainApiRedirect} from "./helpers/functions";
-import autoLoad from 'fastify-autoload';
+import autoLoad from '@fastify/autoload';
 import got from "got";
 
 function addRedirect(server: FastifyInstance, url: string, redirectTo: string) {
-	server.route({
-		url,
-		method: 'GET',
-		schema: {
-			hide: true
-		},
-		handler: async (request: FastifyRequest, reply: FastifyReply) => {
-			reply.redirect(redirectTo);
-		}
-	});
+    server.route({
+        url,
+        method: 'GET',
+        schema: {
+            hide: true
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
+            reply.redirect(redirectTo);
+        }
+    });
 }
 
 function addRoute(server: FastifyInstance, handlersPath: string, prefix: string) {
-	server.register(autoLoad, {
-		dir: join(__dirname, 'routes', handlersPath),
-		ignorePattern: /.*(handler|schema).js/,
-		dirNameRoutePrefix: false,
-		options: {prefix}
-	});
+    server.register(autoLoad, {
+        dir: join(__dirname, 'routes', handlersPath),
+        ignorePattern: /.*(handler|schema).js/,
+        dirNameRoutePrefix: false,
+        options: {prefix}
+    });
 }
 
 export function registerRoutes(server: FastifyInstance) {
 
-	// build internal map of routes
-	const routeSet = new Set<string>();
-	server.decorate('routeSet', routeSet);
-	const ignoreList = [
-		'/v2',
-		'/v2/history',
-		'/v2/state',
-		'/v1/chain/*',
-		'/v1/chain'
-	];
-	server.addHook('onRoute', opts => {
-		if (!ignoreList.includes(opts.url)) {
-			if (opts.url.startsWith('/v')) {
-				routeSet.add(opts.url);
-			}
-		}
-	});
+    // build internal map of routes
+    const routeSet = new Set<string>();
+    server.decorate('routeSet', routeSet);
+    const ignoreList = [
+        '/v2',
+        '/v2/history',
+        '/v2/state',
+        '/v1/chain/*',
+        '/v1/chain'
+    ];
+    server.addHook('onRoute', opts => {
+        if (!ignoreList.includes(opts.url)) {
+            if (opts.url.startsWith('/v')) {
+                routeSet.add(opts.url);
+            }
+        }
+    });
 
-	// Register fastify api routes
-	addRoute(server, 'v2', '/v2');
-	addRoute(server, 'v2-history', '/v2/history');
-	addRoute(server, 'v2-state', '/v2/state');
-	addRoute(server, 'v2-stats', '/v2/stats');
+    // Register fastify api routes
+    addRoute(server, 'v2', '/v2');
+    addRoute(server, 'v2-history', '/v2/history');
+    addRoute(server, 'v2-state', '/v2/state');
+    addRoute(server, 'v2-stats', '/v2/stats');
 
-	// legacy routes
-	addRoute(server, 'v1-history', '/v1/history');
-	addRoute(server, 'v1-trace', '/v1/trace_api');
+    // legacy routes
+    addRoute(server, 'v1-history', '/v1/history');
+    addRoute(server, 'v1-trace', '/v1/trace_api');
 
-	addSharedSchemas(server);
+    addSharedSchemas(server);
 
-	// chain api redirects
-	addRoute(server, 'v1-chain', '/v1/chain');
-	server.route({
-		url: '/v1/chain/*',
-		method: ["GET", "POST"],
-		schema: {
-			summary: "Wildcard chain api handler",
-			tags: ["chain"]
-		},
-		handler: async (request: FastifyRequest, reply: FastifyReply) => {
-			await handleChainApiRedirect(request, reply, server);
-		}
-	});
+    // chain api redirects
+    addRoute(server, 'v1-chain', '/v1/chain');
 
-	// /v1/node/get_supported_apis
-	server.route({
-		url: '/v1/node/get_supported_apis',
-		method: ["GET"],
-		schema: {
-			summary: "Get list of supported APIs",
-			tags: ["node"]
-		},
-		handler: async (request: FastifyRequest, reply: FastifyReply) => {
-			const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
-			if (data.apis && data.apis.length > 0) {
-				const apiSet = new Set(server.routeSet);
-				data.apis.forEach((a) => apiSet.add(a));
-				reply.send({apis: [...apiSet]});
-			} else {
-				reply.send({apis: [...server.routeSet], error: 'nodeos did not send any data'});
-			}
-		}
-	});
+	// other v1 requests
+    server.route({
+        url: '/v1/chain/*',
+        method: ["GET", "POST"],
+        schema: {
+            summary: "Wildcard chain api handler",
+            tags: ["chain"]
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
+            await handleChainApiRedirect(request, reply, server);
+        }
+    });
 
-	server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
-		console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
-		done();
-	});
+    // /v1/node/get_supported_apis
+    server.route({
+        url: '/v1/node/get_supported_apis',
+        method: ["GET"],
+        schema: {
+            summary: "Get list of supported APIs",
+            tags: ["node"]
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
+            const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
+            if (data.apis && data.apis.length > 0) {
+                const apiSet = new Set(server.routeSet);
+                data.apis.forEach((a) => apiSet.add(a));
+                reply.send({apis: [...apiSet]});
+            } else {
+                reply.send({apis: [...server.routeSet], error: 'nodeos did not send any data'});
+            }
+        }
+    });
 
-	if (server.manager.config.features.streaming) {
-		// steam client lib
-		server.get(
-			'/stream-client.js',
-			{schema: {tags: ['internal']}},
-			(request: FastifyRequest, reply: FastifyReply) => {
-				const stream = createReadStream('./hyperion-stream-client.js');
-				reply.type('application/javascript').send(stream);
-			});
-	}
+    server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
+        console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
+        done();
+    });
 
-	// Redirect routes to documentation
-	addRedirect(server, '/v2', '/v2/docs');
-	addRedirect(server, '/v2/history', '/v2/docs/index.html#/history');
-	addRedirect(server, '/v2/state', '/v2/docs/index.html#/state');
-	addRedirect(server, '/v1/chain', '/v2/docs/index.html#/chain');
-	addRedirect(server, '/explorer', '/v2/explore');
-	addRedirect(server, '/explore', '/v2/explore');
+    if (server.manager.config.features.streaming) {
+        // steam client lib
+        server.get(
+            '/stream-client.js',
+            {schema: {tags: ['internal']}},
+            (request: FastifyRequest, reply: FastifyReply) => {
+                const stream = createReadStream('./hyperion-stream-client.js');
+                reply.type('application/javascript').send(stream);
+            });
+    }
+
+    // Redirect routes to documentation
+    addRedirect(server, '/v2', '/v2/docs');
+    addRedirect(server, '/v2/history', '/v2/docs/index.html#/history');
+    addRedirect(server, '/v2/state', '/v2/docs/index.html#/state');
+    addRedirect(server, '/v1/chain', '/v2/docs/index.html#/chain');
+    addRedirect(server, '/explorer', '/v2/explore');
+    addRedirect(server, '/explore', '/v2/explore');
 }

api/routes/v1-chain/get_table_rows/index.ts

@@ -36,7 +36,8 @@ export default function (fastify: FastifyInstance, opts: any, next) {
             "lower_bound": {
                 "type": "string"
             }
-        }
+        },
+        ["code", "table"]
     );
     next();
 }

api/routes/v1-history/get_actions/get_actions.ts

@@ -1,6 +1,148 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
-import flatstr from 'flatstr';
+import {Serialize} from "eosjs";
+import {hLog} from "../../../../helpers/common_functions";
+import * as AbiEOS from "@eosrio/node-abieos";
+import {ApiResponse} from "@elastic/elasticsearch";
+import {TextDecoder, TextEncoder} from "util";
+import {JsonRpc} from "eosjs/dist";
+
+const abi_remapping = {
+    "_Bool": "bool"
+};
+
+const txEnc = new TextEncoder();
+const txDec = new TextDecoder();
+
+async function fetchAbiHexAtBlockElastic(client, chain, contract_name, last_block, get_json) {
+    try {
+        const _includes = ["actions", "tables", "block"];
+        if (get_json) {
+            _includes.push("abi");
+        } else {
+            _includes.push("abi_hex");
+        }
+        const query = {
+            bool: {
+                must: [
+                    {term: {account: contract_name}},
+                    {range: {block: {lte: last_block}}}
+                ]
+            }
+        };
+        const queryResult: ApiResponse = await client.search({
+            index: `${chain}-abi-*`,
+            body: {
+                size: 1, query,
+                sort: [{block: {order: "desc"}}],
+                _source: {includes: _includes}
+            }
+        });
+        const results = queryResult.body.hits.hits;
+        if (results.length > 0) {
+            const nextRefResponse: ApiResponse = await client.search({
+                index: `${chain}-abi-*`,
+                body: {
+                    size: 1,
+                    query: {
+                        bool: {
+                            must: [
+                                {term: {account: contract_name}},
+                                {range: {block: {gte: last_block}}}
+                            ]
+                        }
+                    },
+                    sort: [{block: {order: "asc"}}],
+                    _source: {includes: ["block"]}
+                }
+            });
+            const nextRef = nextRefResponse.body.hits.hits;
+            if (nextRef.length > 0) {
+                return {
+                    valid_until: nextRef[0]._source.block,
+                    ...results[0]._source
+                };
+            }
+            return results[0]._source;
+        } else {
+            return null;
+        }
+    } catch (e) {
+        hLog(e);
+        return null;
+    }
+}
+
+async function getAbiFromHeadBlock(rpc: JsonRpc, code) {
+    let _abi;
+    try {
+        _abi = (await rpc.get_abi(code)).abi;
+    } catch (e) {
+        hLog(e);
+    }
+    return {abi: _abi, valid_until: null, valid_from: null};
+}
+
+async function getContractAtBlock(esClient, rpc, chain, accountName: string, block_num: number, check_action?: string) {
+    let savedAbi, abi;
+    savedAbi = await fetchAbiHexAtBlockElastic(esClient, chain, accountName, block_num, true);
+    if (savedAbi === null || (savedAbi.actions && !savedAbi.actions.includes(check_action))) {
+        savedAbi = await getAbiFromHeadBlock(rpc, accountName);
+        if (!savedAbi) return [null, null];
+        abi = savedAbi.abi;
+    } else {
+        try {
+            abi = JSON.parse(savedAbi.abi);
+        } catch (e) {
+            hLog(e);
+            return [null, null];
+        }
+    }
+    if (!abi) return [null, null];
+    const initialTypes = Serialize.createInitialTypes();
+    let types;
+    try {
+        types = Serialize.getTypesFromAbi(initialTypes, abi);
+    } catch (e) {
+        let remapped = false;
+        for (const struct of abi.structs) {
+            for (const field of struct.fields) {
+                if (abi_remapping[field.type]) {
+                    field.type = abi_remapping[field.type];
+                    remapped = true;
+                }
+            }
+        }
+        if (remapped) {
+            try {
+                types = Serialize.getTypesFromAbi(initialTypes, abi);
+            } catch (e) {
+                hLog('failed after remapping abi');
+                hLog(accountName, block_num, check_action);
+                hLog(e);
+            }
+        } else {
+            hLog(accountName, block_num);
+            hLog(e);
+        }
+    }
+    const actions = new Map();
+    for (const {name, type} of abi.actions) {
+        actions.set(name, Serialize.getType(types, type));
+    }
+    const result = {types, actions, tables: abi.tables};
+    if (check_action) {
+        if (actions.has(check_action)) {
+            try {
+                AbiEOS['load_abi'](accountName, JSON.stringify(abi));
+            } catch (e) {
+                hLog(e);
+            }
+        }
+    }
+    return [result, abi];
+}
+
 
 const terms = ["notified", "act.authorization.actor"];
 const extendedActions = new Set(["transfer", "newaccount", "updateauth"]);
@@ -154,11 +296,13 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
             actions = actions.slice(index);
         }
-        actions.forEach((action, index) => {
+
+        for (let i = 0; i < actions.length; i++) {
+            let action = actions[i];
             action = action._source;
             let act: any = {
                 "global_action_seq": action.global_sequence,
-                "account_action_seq": index,
+                "account_action_seq": i,
                 "block_num": action.block_num,
                 "block_time": action['@timestamp'],
                 "action_trace": {
@@ -174,7 +318,31 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             };
             mergeActionMeta(action);
             act.action_trace.act = action.act;
-            act.action_trace.act.hex_data = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+
+            if (reqBody.hex_data) {
+                try {
+                    const [contract, _] = await getContractAtBlock(
+                        fastify.elastic,
+                        fastify.eosjs.rpc,
+                        fastify.manager.chain,
+                        action.act.account,
+                        action.block_num
+                    );
+                    action.act.hex_data = Serialize.serializeActionData(
+                        contract,
+                        action.act.account,
+                        action.act.name,
+                        action.act.data,
+                        txEnc,
+                        txDec
+                    );
+                } catch (e: any) {
+                    console.log(e);
+                }
+            }
+
+            // TODO: Optionally re-encode using the original ABI, will increase query time
+            // act.action_trace.act.hex_data = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
             if (action.act.account_ram_deltas) {
                 act.action_trace.account_ram_deltas = action.account_ram_deltas
             }
@@ -184,7 +352,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             act.action_trace.receiver = receipt.receiver;
             act.account_action_seq = receipt['recv_sequence'];
             response.actions.push(act);
-        });
+        }
     }
     return response;
 }

api/routes/v1-history/get_transaction/get_transaction.ts

@@ -136,7 +136,9 @@ async function getTransaction(fastify: FastifyInstance, request: FastifyRequest)
 		for (let action of actions) {
 			action = action._source;
 			mergeActionMeta(action);
-			action.act['hex_data'] = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+
+			// action.act['hex_data'] = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+
 			if (action.parent === 0) {
 				response.trx.trx.actions.push(action.act);
 			}

api/routes/v2-history/get_actions/definitions.ts

@@ -1,5 +1,6 @@
 export const terms = [
     "notified",
+    "receipts.receiver",
     "act.authorization.actor"
 ];
 
@@ -15,8 +16,8 @@ export const extendedActions = new Set([
 ]);
 
 export const primaryTerms = [
-    "notified",
     "block_num",
+    "block_id",
     "global_sequence",
     "producer",
     "@timestamp",
@@ -24,5 +25,10 @@ export const primaryTerms = [
     "action_ordinal",
     "cpu_usage_us",
     "net_usage_words",
-    "trx_id"
+    "trx_id",
+    "receipts.receiver",
+    "receipts.global_sequence",
+    "receipts.recv_sequence",
+    "receipts.auth_sequence.account",
+    "receipts.auth_sequence.sequence"
 ];

api/routes/v2-history/get_actions/functions.ts

@@ -1,236 +1,257 @@
 import {primaryTerms, terms} from "./definitions";
 
 export function addSortedBy(query, queryBody, sort_direction) {
-	if (query['sortedBy']) {
-		const opts = query['sortedBy'].split(":");
-		const sortedByObj = {};
-		sortedByObj[opts[0]] = opts[1];
-		queryBody['sort'] = sortedByObj;
-	} else {
-		queryBody['sort'] = {
-			"global_sequence": sort_direction
-		};
-	}
+    if (query['sortedBy']) {
+        const opts = query['sortedBy'].split(":");
+        const sortedByObj = {};
+        sortedByObj[opts[0]] = opts[1];
+        queryBody['sort'] = sortedByObj;
+    } else {
+        queryBody['sort'] = {
+            "global_sequence": sort_direction
+        };
+    }
 }
 
 export function processMultiVars(queryStruct, parts, field) {
-	const must = [];
-	const mustNot = [];
+    const must = [];
+    const mustNot = [];
 
-	parts.forEach(part => {
-		if (part.startsWith("!")) {
-			mustNot.push(part.replace("!", ""));
-		} else {
-			must.push(part);
-		}
-	});
+    parts.forEach(part => {
+        if (part.startsWith("!")) {
+            mustNot.push(part.replace("!", ""));
+        } else {
+            must.push(part);
+        }
+    });
 
-	if (must.length > 1) {
-		queryStruct.bool.must.push({
-			bool: {
-				should: must.map(elem => {
-					const _q = {};
-					_q[field] = elem;
-					return {term: _q}
-				})
-			}
-		});
-	} else if (must.length === 1) {
-		const mustQuery = {};
-		mustQuery[field] = must[0];
-		queryStruct.bool.must.push({term: mustQuery});
-	}
+    if (must.length > 1) {
+        queryStruct.bool.must.push({
+            bool: {
+                should: must.map(elem => {
+                    const _q = {};
+                    _q[field] = elem;
+                    return {term: _q}
+                })
+            }
+        });
+    } else if (must.length === 1) {
+        const mustQuery = {};
+        mustQuery[field] = must[0];
+        queryStruct.bool.must.push({term: mustQuery});
+    }
 
-	if (mustNot.length > 1) {
-		queryStruct.bool.must_not.push({
-			bool: {
-				should: mustNot.map(elem => {
-					const _q = {};
-					_q[field] = elem;
-					return {term: _q}
-				})
-			}
-		});
-	} else if (mustNot.length === 1) {
-		const mustNotQuery = {};
-		mustNotQuery[field] = mustNot[0].replace("!", "");
-		queryStruct.bool.must_not.push({term: mustNotQuery});
-	}
+    if (mustNot.length > 1) {
+        queryStruct.bool.must_not.push({
+            bool: {
+                should: mustNot.map(elem => {
+                    const _q = {};
+                    _q[field] = elem;
+                    return {term: _q}
+                })
+            }
+        });
+    } else if (mustNot.length === 1) {
+        const mustNotQuery = {};
+        mustNotQuery[field] = mustNot[0].replace("!", "");
+        queryStruct.bool.must_not.push({term: mustNotQuery});
+    }
 }
 
 function addRangeQuery(queryStruct, prop, pkey, query) {
-	const _termQuery = {};
-	const parts = query[prop].split("-");
-	_termQuery[pkey] = {
-		"gte": parts[0],
-		"lte": parts[1]
-	};
-	queryStruct.bool.must.push({range: _termQuery});
+    const _termQuery = {};
+    const parts = query[prop].split("-");
+    _termQuery[pkey] = {
+        "gte": parts[0],
+        "lte": parts[1]
+    };
+    queryStruct.bool.must.push({range: _termQuery});
 }
 
 export function applyTimeFilter(query, queryStruct) {
-	if (query['after'] || query['before']) {
-		let _lte = "now";
-		let _gte = "0";
-		if (query['before']) {
-			try {
-				_lte = new Date(query['before']).toISOString();
-			} catch (e) {
-				throw new Error(e.message + ' [before]');
-			}
-		}
-		if (query['after']) {
-			try {
-				_gte = new Date(query['after']).toISOString();
-			} catch (e) {
-				throw new Error(e.message + ' [after]');
-			}
-		}
-		if (!queryStruct.bool['filter']) {
-			queryStruct.bool['filter'] = [];
-		}
-		queryStruct.bool['filter'].push({
-			range: {
-				"@timestamp": {
-					"gte": _gte,
-					"lte": _lte
-				}
-			}
-		});
-	}
+    if (query['after'] || query['before']) {
+        if (query['after']?.includes('T') || query['before']?.includes('T')) {
+            let _lte = "now";
+            let _gte = "0";
+            if (query['before']) {
+                try {
+                    _lte = new Date(query['before']).toISOString();
+                } catch (e) {
+                    throw new Error(e.message + ' [before]');
+                }
+            }
+            if (query['after']) {
+                try {
+                    _gte = new Date(query['after']).toISOString();
+                } catch (e) {
+                    throw new Error(e.message + ' [after]');
+                }
+            }
+            if (!queryStruct.bool['filter']) {
+                queryStruct.bool['filter'] = [];
+            }
+            queryStruct.bool['filter'].push({
+                range: {
+                    "@timestamp": {
+                        "gte": _gte,
+                        "lte": _lte
+                    }
+                }
+            });
+        } else {
+            // search by block number
+            const rangeObj = {
+                range: {
+                    block_num: {}
+                }
+            };
+            if (parseInt(query['after']) > 0) {
+                rangeObj.range.block_num['gte'] = query['after'];
+            }
+            if (parseInt(query['before']) > 0) {
+                rangeObj.range.block_num['lte'] = query['before'];
+            }
+            if (Object.keys(rangeObj.range.block_num).length > 0) {
+                if (!queryStruct.bool['filter']) {
+                    queryStruct.bool['filter'] = [];
+                }
+                queryStruct.bool['filter'].push(rangeObj);
+            }
+        }
+    }
 }
 
 export function applyGenericFilters(query, queryStruct, allowedExtraParams: Set<string>) {
-	for (const prop in query) {
-		if (Object.prototype.hasOwnProperty.call(query, prop)) {
-			const pair = prop.split(".");
-			if (pair.length > 1 || primaryTerms.includes(pair[0])) {
-				let pkey;
-				if (pair.length > 1 && allowedExtraParams) {
-					pkey = allowedExtraParams.has(pair[0]) ? "@" + prop : prop;
-				} else {
-					pkey = prop;
-				}
-				if (query[prop].indexOf("-") !== -1) {
-					addRangeQuery(queryStruct, prop, pkey, query);
-				} else {
-					const _qObj = {};
-					const parts = query[prop].split(",");
-					if (parts.length > 1) {
-						processMultiVars(queryStruct, parts, prop);
-					} else if (parts.length === 1) {
+    for (const prop in query) {
+        if (Object.prototype.hasOwnProperty.call(query, prop)) {
+            const pair = prop.split(".");
+            if (pair.length > 1 || primaryTerms.includes(pair[0])) {
+                let pkey;
+                if (pair.length > 1 && allowedExtraParams) {
+                    pkey = allowedExtraParams.has(pair[0]) ? "@" + prop : prop;
+                } else {
+                    pkey = prop;
+                }
+                if (query[prop].indexOf("-") !== -1) {
+                    addRangeQuery(queryStruct, prop, pkey, query);
+                } else {
+                    const _qObj = {};
+                    const parts = query[prop].split(",");
+                    if (parts.length > 1) {
+                        processMultiVars(queryStruct, parts, prop);
+                    } else if (parts.length === 1) {
 
-						// @transfer.memo special case
-						if (pkey === '@transfer.memo') {
-							_qObj[pkey] = {
-								query: parts[0]
-							};
+                        // @transfer.memo special case
+                        if (pkey === '@transfer.memo') {
+                            _qObj[pkey] = {
+                                query: parts[0]
+                            };
 
-							if (query.match_fuzziness) {
-								_qObj[pkey].fuzziness = query.match_fuzziness;
-							}
+                            if (query.match_fuzziness) {
+                                _qObj[pkey].fuzziness = query.match_fuzziness;
+                            }
 
-							if (query.match_operator) {
-								_qObj[pkey].operator = query.match_operator;
-							}
+                            if (query.match_operator) {
+                                _qObj[pkey].operator = query.match_operator;
+                            }
 
-							queryStruct.bool.must.push({
-								match: _qObj
-							});
-						} else {
-							const andParts = parts[0].split(" ");
-							if (andParts.length > 1) {
-								andParts.forEach(value => {
-									const _q = {};
-									_q[pkey] = value;
-									queryStruct.bool.must.push({term: _q});
-								});
-							} else {
-								if (parts[0].startsWith("!")) {
-									_qObj[pkey] = parts[0].replace("!", "");
-									queryStruct.bool.must_not.push({term: _qObj});
-								} else {
-									_qObj[pkey] = parts[0];
-									queryStruct.bool.must.push({term: _qObj});
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-	}
+                            queryStruct.bool.must.push({
+                                match: _qObj
+                            });
+                        } else {
+                            const andParts = parts[0].split(" ");
+                            if (andParts.length > 1) {
+                                andParts.forEach(value => {
+                                    const _q = {};
+                                    _q[pkey] = value;
+                                    queryStruct.bool.must.push({term: _q});
+                                });
+                            } else {
+                                if (parts[0].startsWith("!")) {
+                                    _qObj[pkey] = parts[0].replace("!", "");
+                                    queryStruct.bool.must_not.push({term: _qObj});
+                                } else {
+                                    _qObj[pkey] = parts[0];
+                                    queryStruct.bool.must.push({term: _qObj});
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
 }
 
 export function makeShouldArray(query) {
-	const should_array = [];
-	for (const entry of terms) {
-		const tObj = {term: {}};
-		tObj.term[entry] = query.account;
-		should_array.push(tObj);
-	}
-	return should_array;
+    const should_array = [];
+    for (const entry of terms) {
+        const tObj = {term: {}};
+        tObj.term[entry] = query.account;
+        should_array.push(tObj);
+    }
+    return should_array;
 }
 
 export function applyCodeActionFilters(query, queryStruct) {
-	let filterObj = [];
-	if (query.filter) {
-		for (const filter of query.filter.split(',')) {
-			if (filter !== '*:*') {
-				const _arr = [];
-				const parts = filter.split(':');
-				if (parts.length === 2) {
-					const [code, method] = parts;
-					if (code && code !== "*") {
-						_arr.push({'term': {'act.account': code}});
-					}
-					if (method && method !== "*") {
-						_arr.push({'term': {'act.name': method}});
-					}
-				}
-				if (_arr.length > 0) {
-					filterObj.push({bool: {must: _arr}});
-				}
-			}
-		}
-		if (filterObj.length > 0) {
-			queryStruct.bool['should'] = filterObj;
-			queryStruct.bool['minimum_should_match'] = 1;
-		}
-	}
+    let filterObj = [];
+    if (query.filter) {
+        for (const filter of query.filter.split(',')) {
+            if (filter !== '*:*') {
+                const _arr = [];
+                const parts = filter.split(':');
+                if (parts.length === 2) {
+                    const [code, method] = parts;
+                    if (code && code !== "*") {
+                        _arr.push({'term': {'act.account': code}});
+                    }
+                    if (method && method !== "*") {
+                        _arr.push({'term': {'act.name': method}});
+                    }
+                }
+                if (_arr.length > 0) {
+                    filterObj.push({bool: {must: _arr}});
+                }
+            }
+        }
+        if (filterObj.length > 0) {
+            queryStruct.bool['should'] = filterObj;
+            queryStruct.bool['minimum_should_match'] = 1;
+        }
+    }
 }
 
 export function getSkipLimit(query, max?: number) {
-	let skip, limit;
-	skip = parseInt(query.skip, 10);
-	if (skip < 0) {
-		throw new Error('invalid skip parameter');
-	}
-	limit = parseInt(query.limit, 10);
-	if (limit < 1) {
-		throw new Error('invalid limit parameter');
-	} else if (limit > max) {
-		throw new Error(`limit too big, maximum: ${max}`);
-	}
-	return {skip, limit};
+    let skip, limit;
+    skip = parseInt(query.skip, 10);
+    if (skip < 0) {
+        throw new Error('invalid skip parameter');
+    }
+    limit = parseInt(query.limit, 10);
+    if (limit < 1) {
+        throw new Error('invalid limit parameter');
+    } else if (limit > max) {
+        throw new Error(`limit too big, maximum: ${max}`);
+    }
+    return {skip, limit};
 }
 
 export function getSortDir(query) {
-	let sort_direction = 'desc';
-	if (query.sort) {
-		if (query.sort === 'asc' || query.sort === '1') {
-			sort_direction = 'asc';
-		} else if (query.sort === 'desc' || query.sort === '-1') {
-			sort_direction = 'desc'
-		} else {
-			throw new Error('invalid sort direction');
-		}
-	}
-	return sort_direction;
+    let sort_direction = 'desc';
+    if (query.sort) {
+        if (query.sort === 'asc' || query.sort === '1') {
+            sort_direction = 'asc';
+        } else if (query.sort === 'desc' || query.sort === '-1') {
+            sort_direction = 'desc'
+        } else {
+            throw new Error('invalid sort direction');
+        }
+    }
+    return sort_direction;
 }
 
 export function applyAccountFilters(query, queryStruct) {
-	if (query.account) {
-		queryStruct.bool.must.push({"bool": {should: makeShouldArray(query)}});
-	}
+    if (query.account) {
+        queryStruct.bool.must.push({"bool": {should: makeShouldArray(query)}});
+    }
 }

api/routes/v2-history/get_actions/get_actions.ts

@@ -1,118 +1,134 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {getTrackTotalHits, mergeActionMeta, timedQuery} from "../../../helpers/functions";
 import {
-	addSortedBy,
-	applyAccountFilters,
-	applyCodeActionFilters,
-	applyGenericFilters,
-	applyTimeFilter,
-	getSkipLimit,
-	getSortDir
+    addSortedBy,
+    applyAccountFilters,
+    applyCodeActionFilters,
+    applyGenericFilters,
+    applyTimeFilter,
+    getSkipLimit,
+    getSortDir
 } from "./functions";
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
-	const query: any = request.query;
-	const maxActions = fastify.manager.config.api.limits.get_actions;
-	const queryStruct = {
-		"bool": {
-			must: [],
-			must_not: [],
-			boost: 1.0
-		}
-	};
+    const query: any = request.query;
+    const maxActions = fastify.manager.config.api.limits.get_actions;
+    const queryStruct = {
+        "bool": {
+            must: [],
+            must_not: [],
+            boost: 1.0
+        }
+    };
 
-	const {skip, limit} = getSkipLimit(query, maxActions);
-	const sort_direction = getSortDir(query);
-	applyAccountFilters(query, queryStruct);
-	applyGenericFilters(query, queryStruct, fastify.allowedActionQueryParamSet);
-	applyTimeFilter(query, queryStruct);
-	applyCodeActionFilters(query, queryStruct);
-	// allow precise counting of total hits
-	const trackTotalHits = getTrackTotalHits(query);
+    const {skip, limit} = getSkipLimit(query, maxActions);
 
-	// Prepare query body
-	const query_body = {
-		"track_total_hits": trackTotalHits,
-		"query": queryStruct
-	};
+    const sort_direction = getSortDir(query);
 
-	// Include sorting
-	addSortedBy(query, query_body, sort_direction);
+    applyAccountFilters(query, queryStruct);
 
-	// Perform search
+    applyGenericFilters(query, queryStruct, fastify.allowedActionQueryParamSet);
 
-	let indexPattern = fastify.manager.chain + '-action-*';
-	if (query.hot_only) {
-		indexPattern = fastify.manager.chain + '-action';
-	}
+    applyTimeFilter(query, queryStruct);
 
-	const esResults = await fastify.elastic.search({
-		"index": indexPattern,
-		"from": skip || 0,
-		"size": (limit > maxActions ? maxActions : limit) || 10,
-		"body": query_body
-	});
+    applyCodeActionFilters(query, queryStruct);
 
-	const results = esResults['body']['hits'];
-	const response: any = {
-		cached: false,
-		lib: 0,
-		total: results['total']
-	};
+    // allow precise counting of total hits
+    const trackTotalHits = getTrackTotalHits(query);
 
-	if (query.hot_only) {
-		response.hot_only = true;
-	}
+    // Prepare query body
+    const query_body = {
+        "track_total_hits": trackTotalHits,
+        "query": queryStruct
+    };
 
-	if (query.checkLib) {
-		response.lib = (await fastify.eosjs.rpc.get_info()).last_irreversible_block_num;
-	}
+    // Include sorting
+    addSortedBy(query, query_body, sort_direction);
 
-	if (query.simple) {
-		response['simple_actions'] = [];
-	} else {
-		response['actions'] = [];
-	}
+    // Perform search
 
-	if (results['hits'].length > 0) {
-		const actions = results['hits'];
-		for (let action of actions) {
-			action = action._source;
-			mergeActionMeta(action);
+    let indexPattern = fastify.manager.chain + '-action-*';
+    if (query.hot_only) {
+        indexPattern = fastify.manager.chain + '-action';
+    }
 
-			if (query.noBinary === true) {
-				for (const key in action['act']['data']) {
-					if (action['act']['data'].hasOwnProperty(key)) {
-						if (typeof action['act']['data'][key] === 'string' && action['act']['data'][key].length > 256) {
-							action['act']['data'][key] = action['act']['data'][key].slice(0, 32) + "...";
-						}
-					}
-				}
-			}
+    const esResults = await fastify.elastic.search({
+        "index": indexPattern,
+        "from": skip || 0,
+        "size": (limit > maxActions ? maxActions : limit) || 10,
+        "body": query_body
+    });
 
-			if (query.simple) {
-				response.simple_actions.push({
-					block: action['block_num'],
-					irreversible: response.lib !== 0 ? action['block_num'] < response.lib : undefined,
-					timestamp: action['@timestamp'],
-					transaction_id: action['trx_id'],
-					actors: action['act']['authorization'].map(a => `${a.actor}@${a.permission}`).join(","),
-					notified: action['notified'].join(','),
-					contract: action['act']['account'],
-					action: action['act']['name'],
-					data: action['act']['data']
-				});
+    const results = esResults['body']['hits'];
+    const response: any = {
+        cached: false,
+        lib: 0,
+        total: results['total']
+    };
 
-			} else {
-				response.actions.push(action);
-			}
-		}
-	}
-	return response;
+    if (query.hot_only) {
+        response.hot_only = true;
+    }
+
+    if (query.checkLib) {
+        response.lib = (await fastify.eosjs.rpc.get_info()).last_irreversible_block_num;
+    }
+
+    if (query.simple) {
+        response['simple_actions'] = [];
+    } else {
+        response['actions'] = [];
+    }
+
+    if (results['hits'].length > 0) {
+        const actions = results['hits'];
+        for (let action of actions.map(a => a._source)) {
+
+            try {
+                if (action.act.data) {
+                    if (action.act.data.account && action.act.data.name && action.act.data.authorization) {
+                        action.act.data = action.act.data.data;
+                    }
+                }
+            } catch (e: any) {
+                console.log(e);
+            }
+
+            mergeActionMeta(action);
+
+            if (query.noBinary === true) {
+                for (const key in action['act']['data']) {
+                    if (action['act']['data'].hasOwnProperty(key)) {
+                        if (typeof action['act']['data'][key] === 'string' && action['act']['data'][key].length > 256) {
+                            action['act']['data'][key] = action['act']['data'][key].slice(0, 32) + "...";
+                        }
+                    }
+                }
+            }
+
+            if (query.simple) {
+                response.simple_actions.push({
+                    block: action['block_num'],
+                    irreversible: response.lib !== 0 ? action['block_num'] < response.lib : undefined,
+                    timestamp: action['@timestamp'],
+                    transaction_id: action['trx_id'],
+                    actors: action['act']['authorization'].map(a => `${a.actor}@${a.permission}`).join(","),
+                    notified: action['notified'].join(','),
+                    contract: action['act']['account'],
+                    action: action['act']['name'],
+                    data: action['act']['data']
+                });
+
+            } else {
+                response.actions.push(action);
+            }
+        }
+    }
+    return response;
 }
 
 export function getActionsHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply) => {
-		reply.send(await timedQuery(getActions, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getActions, fastify, request, route));
+    }
 }

api/routes/v2-history/get_actions/index.ts

@@ -3,129 +3,157 @@ import {getActionsHandler} from "./get_actions";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export const getActionResponseSchema = {
-	"@timestamp": {type: "string"},
-	"timestamp": {type: "string"},
-	"block_num": {type: "number"},
-	"trx_id": {type: "string"},
-	"act": {
-		type: 'object',
-		properties: {
-			"account": {type: "string"},
-			"name": {type: "string"}
-		},
-		additionalProperties: true
-	},
-	"notified": {
-		type: "array", items: {type: "string"}
-	},
-	"cpu_usage_us": {type: "number"},
-	"net_usage_words": {type: "number"},
-	"account_ram_deltas": {
-		type: "array",
-		items: {
-			type: "object",
-			properties: {
-				"account": {type: "string"},
-				"delta": {type: "number"}
-			},
-			additionalProperties: true
-		}
-	},
-	"global_sequence": {type: "number"},
-	"receiver": {type: 'string'},
-	"producer": {type: "string"},
-	"parent": {type: "number"},
-	"action_ordinal": {type: 'number'},
-	"creator_action_ordinal": {type: 'number'}
+    "timestamp": {type: "string"},
+    "block_num": {type: "number"},
+    "block_id": {type: "string"},
+    "trx_id": {type: "string"},
+    "act": {
+        type: 'object',
+        properties: {
+            "account": {type: "string"},
+            "name": {type: "string"},
+            "authorization": {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        "actor": {type: "string"},
+                        "permission": {type: "string"},
+                    }
+                }
+            }
+        },
+        additionalProperties: true
+    },
+    "receipts": {
+        type: "array",
+        items: {
+            type: "object",
+            properties: {
+                "receiver": {type: "string"},
+                "global_sequence": {type: "number"},
+                "recv_sequence": {type: "number"},
+                "auth_sequence": {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            "account": {type: "string"},
+                            "sequence": {type: "number"},
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "cpu_usage_us": {type: "number"},
+    "net_usage_words": {type: "number"},
+    "account_ram_deltas": {
+        type: "array",
+        items: {
+            type: "object",
+            properties: {
+                "account": {type: "string"},
+                "delta": {type: "number"}
+            },
+            additionalProperties: true
+        }
+    },
+    "global_sequence": {type: "number"},
+    "producer": {type: "string"},
+    "parent": {type: "number"},
+    "action_ordinal": {type: 'number'},
+    "creator_action_ordinal": {type: 'number'},
+    "signatures": {type: "array", items: {type: 'string'}}
 };
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-	const schema: FastifySchema = {
-		description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
-			' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
-		summary: 'get root actions',
-		tags: ['history'],
-		querystring: extendQueryStringSchema({
-			"account": {
-				description: 'notified account',
-				type: 'string',
-				minLength: 1,
-				maxLength: 12
-			},
-			"track": {
-				description: 'total results to track (count) [number or true]',
-				type: 'string'
-			},
-			"filter": {
-				description: 'code:name filter',
-				type: 'string',
-				minLength: 3
-			},
-			"sort": {
-				description: 'sort direction',
-				enum: ['desc', 'asc', '1', '-1'],
-				type: 'string'
-			},
-			"after": {
-				description: 'filter after specified date (ISO8601)',
-				type: 'string'
-			},
-			"before": {
-				description: 'filter before specified date (ISO8601)',
-				type: 'string'
-			},
-			"simple": {
-				description: 'simplified output mode',
-				type: 'boolean'
-			},
-			"hot_only": {
-				description: 'search only the latest hot index',
-				type: 'boolean'
-			},
-			"noBinary": {
-				description: "exclude large binary data",
-				type: 'boolean'
-			},
-			"checkLib": {
-				description: "perform reversibility check",
-				type: 'boolean'
-			},
-		}),
-		response: extendResponseSchema({
-			"simple_actions": {
-				type: "array",
-				items: {
-					type: "object",
-					properties: {
-						"block": {type: "number"},
-						"timestamp": {type: "string"},
-						"irreversible": {type: "boolean"},
-						"contract": {type: "string"},
-						"action": {type: "string"},
-						"actors": {type: "string"},
-						"notified": {type: "string"},
-						"transaction_id": {type: "string"},
-						"data": {
-							additionalProperties: true
-						}
-					}
-				}
-			},
-			"actions": {
-				type: "array",
-				items: {
-					type: 'object',
-					properties: getActionResponseSchema
-				}
-			}
-		})
-	};
-	addApiRoute(
-		fastify,
-		'GET',
-		getRouteName(__filename),
-		getActionsHandler,
-		schema
-	);
-	next();
+    const schema: FastifySchema = {
+        description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
+            ' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
+        summary: 'get root actions',
+        tags: ['history'],
+        querystring: extendQueryStringSchema({
+            "account": {
+                description: 'notified account',
+                type: 'string',
+                minLength: 1,
+                maxLength: 12
+            },
+            "track": {
+                description: 'total results to track (count) [number or true]',
+                type: 'string'
+            },
+            "filter": {
+                description: 'code:name filter',
+                type: 'string',
+                minLength: 3
+            },
+            "sort": {
+                description: 'sort direction',
+                enum: ['desc', 'asc', '1', '-1'],
+                type: 'string'
+            },
+            "after": {
+                description: 'filter after specified date (ISO8601)',
+                type: 'string'
+            },
+            "before": {
+                description: 'filter before specified date (ISO8601)',
+                type: 'string'
+            },
+            "simple": {
+                description: 'simplified output mode',
+                type: 'boolean'
+            },
+            "hot_only": {
+                description: 'search only the latest hot index',
+                type: 'boolean'
+            },
+            "noBinary": {
+                description: "exclude large binary data",
+                type: 'boolean'
+            },
+            "checkLib": {
+                description: "perform reversibility check",
+                type: 'boolean'
+            },
+        }),
+        response: extendResponseSchema({
+            "simple_actions": {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        "block": {type: "number"},
+                        "timestamp": {type: "string"},
+                        "irreversible": {type: "boolean"},
+                        "contract": {type: "string"},
+                        "action": {type: "string"},
+                        "actors": {type: "string"},
+                        "notified": {type: "string"},
+                        "transaction_id": {type: "string"},
+                        "data": {
+                            additionalProperties: true
+                        }
+                    }
+                }
+            },
+            "actions": {
+                type: "array",
+                items: {
+                    type: 'object',
+                    properties: getActionResponseSchema
+                }
+            }
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getActionsHandler,
+        schema
+    );
+    next();
 }

api/routes/v2-history/get_creator/get_creator.ts

@@ -3,98 +3,104 @@ import {timedQuery} from "../../../helpers/functions";
 
 async function getCreator(fastify: FastifyInstance, request: FastifyRequest) {
 
-	const query: any = request.query;
+    const query: any = request.query;
 
-	const response = {
-		account: query.account,
-		creator: '',
-		timestamp: '',
-		block_num: 0,
-		trx_id: '',
-	};
+    const response = {
+        account: query.account,
+        creator: '',
+        timestamp: '',
+        block_num: 0,
+        trx_id: '',
+    };
 
-	if (query.account === fastify.manager.config.settings.eosio_alias) {
-		const genesisBlock = await fastify.eosjs.rpc.get_block(1);
-		if (genesisBlock) {
-			response.creator = '__self__';
-			response.timestamp = genesisBlock.timestamp;
-			response.block_num = 1;
-			response.trx_id = "";
-			return response;
-		} else {
-			throw new Error("genesis block not found");
-		}
-	}
+    if (query.account === fastify.manager.config.settings.eosio_alias) {
+        try {
+            const genesisBlock = await fastify.eosjs.rpc.get_block(1);
+            if (genesisBlock) {
+                response.timestamp = genesisBlock.timestamp;
+            }
+        } catch (e) {
+            console.log(e.message);
+        }
+        response.creator = '__self__';
+        response.block_num = 1;
+        response.trx_id = "";
+        return response;
+    }
 
-	const results = await fastify.elastic.search({
-		"index": fastify.manager.chain + '-action-*',
-		"body": {
-			size: 1,
-			query: {
-				bool: {
-					must: [{term: {"@newaccount.newact": query.account}}]
-				}
-			}
-		}
-	});
+    const results = await fastify.elastic.search({
+        "index": fastify.manager.chain + '-action-*',
+        "body": {
+            size: 1,
+            query: {
+                bool: {
+                    must: [{term: {"@newaccount.newact": query.account}}]
+                }
+            }
+        }
+    });
 
-	if (results['body']['hits']['hits'].length === 1) {
-		const result = results['body']['hits']['hits'][0]._source;
-		response.trx_id = result.trx_id;
-		response.block_num = result.block_num;
-		response.creator = result.act.data.creator;
-		response.timestamp = result['@timestamp'];
-		return response;
-	} else {
-		let accountInfo;
-		try {
-			accountInfo = await fastify.eosjs.rpc.get_account(query.account);
-		} catch (e) {
-			throw new Error("account not found");
-		}
-		if (accountInfo) {
-			try {
-				response.timestamp = accountInfo.created;
-				const blockHeader = await fastify.elastic.search({
-					"index": fastify.manager.chain + '-block-*',
-					"body": {
-						size: 1,
-						query: {
-							bool: {
-								must: [{term: {"@timestamp": response.timestamp}}]
-							}
-						}
-					}
-				});
-				const blockId = blockHeader['body']['hits']['hits'][0]._source.block_id;
-				const blockData = await fastify.eosjs.rpc.get_block(blockId);
-				response.block_num = blockData.block_num;
-				for (const transaction of blockData["transactions"]) {
-					if (typeof transaction.trx !== 'string') {
-						const actions = transaction.trx.transaction.actions;
-						for (const act of actions) {
-							if (act.name === 'newaccount') {
-								if (act.data.name === query.account) {
-									response.creator = act.data.creator;
-									response.trx_id = transaction.id;
-									return response;
-								}
-							}
-						}
-					}
-				}
-				return response;
-			} catch (e) {
-				throw new Error("account creation not found");
-			}
-		} else {
-			throw new Error("account creation not found");
-		}
-	}
+    if (results['body']['hits']['hits'].length === 1) {
+        const result = results['body']['hits']['hits'][0]._source;
+        response.trx_id = result.trx_id;
+        response.block_num = result.block_num;
+        response.creator = result.act.data.creator;
+        response.timestamp = result['@timestamp'];
+        return response;
+    } else {
+        let accountInfo;
+        try {
+            accountInfo = await fastify.eosjs.rpc.get_account(query.account);
+        } catch (e) {
+            throw new Error("account not found");
+        }
+        if (accountInfo) {
+            try {
+                response.timestamp = accountInfo.created;
+                const blockHeader = await fastify.elastic.search({
+                    "index": fastify.manager.chain + '-block-*',
+                    "body": {
+                        size: 1,
+                        query: {
+                            bool: {
+                                must: [{term: {"@timestamp": response.timestamp}}]
+                            }
+                        }
+                    }
+                });
+                const hits = blockHeader['body']['hits']['hits'];
+                if (hits.length > 0 && hits[0]._source) {
+                    const blockId = blockHeader['body']['hits']['hits'][0]._source.block_id;
+                    const blockData = await fastify.eosjs.rpc.get_block(blockId);
+                    response.block_num = blockData.block_num;
+                    for (const transaction of blockData["transactions"]) {
+                        if (typeof transaction.trx !== 'string') {
+                            const actions = transaction.trx.transaction.actions;
+                            for (const act of actions) {
+                                if (act.name === 'newaccount') {
+                                    if (act.data.name === query.account) {
+                                        response.creator = act.data.creator;
+                                        response.trx_id = transaction.id;
+                                        return response;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                return response;
+            } catch (e) {
+                console.log(e.message);
+                throw new Error("account creation not found");
+            }
+        } else {
+            throw new Error("account not found");
+        }
+    }
 }
 
 export function getCreatorHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply) => {
-		reply.send(await timedQuery(getCreator, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getCreator, fastify, request, route));
+    }
 }

api/routes/v2-history/get_deltas/get_deltas.ts

@@ -3,89 +3,97 @@ import {mergeDeltaMeta, timedQuery} from "../../../helpers/functions";
 import {applyTimeFilter} from "../get_actions/functions";
 
 async function getDeltas(fastify: FastifyInstance, request: FastifyRequest) {
-	let skip, limit;
-	let sort_direction = 'desc';
-	const mustArray = [];
-	const query: any = request.query;
-	for (const param in query) {
-		if (Object.prototype.hasOwnProperty.call(query, param)) {
-			const value = query[param];
-			switch (param) {
-				case 'limit': {
-					limit = parseInt(value, 10);
-					if (limit < 1) {
-						return 'invalid limit parameter';
-					}
-					break;
-				}
-				case 'skip': {
-					skip = parseInt(value, 10);
-					if (skip < 0) {
-						return 'invalid skip parameter';
-					}
-					break;
-				}
-				case 'sort': {
-					if (value === 'asc' || value === '1') {
-						sort_direction = 'asc';
-					} else if (value === 'desc' || value === '-1') {
-						sort_direction = 'desc'
-					} else {
-						return 'invalid sort direction';
-					}
-					break;
-				}
-				case 'before': {
-					break;
-				}
-				case 'after': {
-					break;
-				}
-				default: {
-					const values = query[param].split(",");
-					if (values.length > 1) {
-						const terms = {};
-						terms[param] = values;
-						mustArray.push({terms: terms});
-					} else {
-						const term = {};
-						term[param] = values[0];
-						mustArray.push({term: term});
-					}
-					break;
-				}
-			}
-		}
-	}
+    let skip, limit;
+    let sort_direction = 'desc';
+    const mustArray = [];
+    const query: any = request.query;
+    for (const param in query) {
+        if (Object.prototype.hasOwnProperty.call(query, param)) {
+            const value = query[param];
+            switch (param) {
+                case 'limit': {
+                    limit = parseInt(value, 10);
+                    if (limit < 1) {
+                        return 'invalid limit parameter';
+                    }
+                    break;
+                }
+                case 'skip': {
+                    skip = parseInt(value, 10);
+                    if (skip < 0) {
+                        return 'invalid skip parameter';
+                    }
+                    break;
+                }
+                case 'sort': {
+                    if (value === 'asc' || value === '1') {
+                        sort_direction = 'asc';
+                    } else if (value === 'desc' || value === '-1') {
+                        sort_direction = 'desc'
+                    } else {
+                        return 'invalid sort direction';
+                    }
+                    break;
+                }
+                case 'before': {
+                    break;
+                }
+                case 'after': {
+                    break;
+                }
+                default: {
+                    if (typeof value === 'string') {
+                        const values = query[param].split(",");
+                        if (values.length > 1) {
+                            const terms = {};
+                            terms[param] = values;
+                            mustArray.push({terms: terms});
+                        } else {
+                            const term = {};
+                            term[param] = values[0];
+                            mustArray.push({term: term});
+                        }
+                    } else {
+                        if (typeof value === 'number') {
+                            const term = {};
+                            term[param] = value;
+                            mustArray.push({term: term});
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+    }
 
-	const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
-	const queryStruct = {bool: {must: mustArray}};
+    const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
+    const queryStruct = {bool: {must: mustArray}};
 
-	applyTimeFilter(query, queryStruct);
+    applyTimeFilter(query, queryStruct);
 
-	const results = await fastify.elastic.search({
-		"index": fastify.manager.chain + '-delta-*',
-		"from": skip || 0,
-		"size": (limit > maxDeltas ? maxDeltas : limit) || 10,
-		"body": {
-			query: queryStruct,
-			sort: {
-				"block_num": sort_direction
-			}
-		}
-	});
-	const deltas = results['body']['hits']['hits'].map((d) => {
-		return mergeDeltaMeta(d._source);
-	});
-	return {
-		query_time: null,
-		total: results['body']['hits']['total'],
-		deltas
-	};
+    const results = await fastify.elastic.search({
+        "index": fastify.manager.chain + '-delta-*',
+        "from": skip || 0,
+        "size": (limit > maxDeltas ? maxDeltas : limit) || 10,
+        "body": {
+            query: queryStruct,
+            sort: {
+                "block_num": sort_direction
+            }
+        }
+    });
+    const deltas = results['body']['hits']['hits'].map((d) => {
+        return mergeDeltaMeta(d._source);
+    });
+    return {
+        query_time: null,
+        total: results['body']['hits']['total'],
+        deltas
+    };
 }
 
 export function getDeltasHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply) => {
-		reply.send(await timedQuery(getDeltas, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getDeltas, fastify, request, route));
+    }
 }

api/routes/v2-history/get_transaction_legacy/get_transaction_legacy.ts

@@ -0,0 +1,128 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
+
+async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
+    const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
+    const query: any = request.query;
+
+    let indexPattern = fastify.manager.chain + '-action-*';
+    if (query.hot_only) {
+        indexPattern = fastify.manager.chain + '-action';
+    }
+
+    const pResults = await Promise.all([
+        fastify.eosjs.rpc.get_info(),
+        fastify.elastic.search({
+            index: indexPattern,
+            size: _size,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {term: {trx_id: query.id.toLowerCase()}}
+                        ]
+                    }
+                },
+                sort: {
+                    global_sequence: "asc"
+                }
+            }
+        }),
+        fastify.elastic.search({
+            index: fastify.manager.chain + '-gentrx-*',
+            size: _size,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {term: {trx_id: query.id.toLowerCase()}}
+                        ]
+                    }
+                }
+            }
+        })
+    ]);
+
+    const results = pResults[1];
+    const genTrxRes = pResults[2];
+
+    const response = {
+        "executed": false,
+        "hot_only": false,
+        "trx_id": query.id,
+        "lib": pResults[0].last_irreversible_block_num,
+        "actions": [],
+        "generated": undefined
+    };
+
+    if (query.hot_only) {
+        response.hot_only = true;
+    }
+
+    const hits = results['body']['hits']['hits'];
+
+    if (hits.length > 0) {
+
+        // const producers = {};
+        // for (let hit of hits) {
+        // 	if (hit._source.producer) {
+        // 		if (producers[hit._source.producer]) {
+        // 			producers[hit._source.producer]++;
+        // 		} else {
+        // 			producers[hit._source.producer] = 1;
+        // 		}
+        // 	}
+        // }
+        // let useBlockNumber;
+        // if (Object.keys(producers).length > 1) {
+        // 	// multiple producers of the same tx id, forked actions are present, attempt to clean-up
+        // 	let trueProd = '';
+        // 	let highestActCount = 0;
+        // 	for (const prod in producers) {
+        // 		if (producers.hasOwnProperty(prod)) {
+        // 			if(producers[prod] === highestActCount) {
+        // 				useBlockNumber = true;
+        // 			} else if (producers[prod] > highestActCount) {
+        // 				highestActCount = producers[prod];
+        // 				trueProd = prod;
+        // 			}
+        // 		}
+        // 	}
+        // }
+
+        let highestBlockNum = 0;
+        for (let action of hits) {
+            if (action._source.block_num > highestBlockNum) {
+                highestBlockNum = action._source.block_num;
+            }
+        }
+
+        for (let action of hits) {
+            if (action._source.block_num === highestBlockNum) {
+                action = action._source;
+                mergeActionMeta(action);
+                response.actions.push(action);
+            }
+        }
+
+        response.executed = true;
+    }
+
+    const hits2 = genTrxRes['body']['hits']['hits'];
+
+    if (hits2 && hits2.length > 0) {
+        if (hits2[0]._source['@timestamp']) {
+            hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
+            delete hits2[0]._source['@timestamp'];
+        }
+        response.generated = hits2[0]._source;
+    }
+
+    return response;
+}
+
+export function getTransactionHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getTransaction, fastify, request, route));
+    }
+}

api/routes/v2-history/get_transaction_legacy/index.ts

@@ -0,0 +1,29 @@
+import {FastifyInstance} from "fastify";
+import {getTransactionHandler} from "./get_transaction_legacy";
+import {addApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get all actions belonging to the same transaction',
+        summary: 'get transaction by id',
+        tags: ['history'],
+        querystring: {
+            type: 'object',
+            properties: {
+                "id": {
+                    description: 'transaction id',
+                    type: 'string'
+                }
+            },
+            required: ["id"]
+        }
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getTransactionHandler,
+        schema
+    );
+    next();
+}

api/routes/v2-state/get_tokens/get_tokens.ts

@@ -7,7 +7,10 @@ async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
 
     const query: any = request.query;
 
-    const response = {'account': query.account, 'tokens': []};
+    const response = {
+        account: query.account,
+        tokens: []
+    };
 
     const {skip, limit} = getSkipLimit(request.query);
     const maxDocs = fastify.manager.config.api.limits.get_tokens ?? 100;
@@ -19,7 +22,7 @@ async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
         "body": {
             query: {
                 bool: {
-                    filter: [{term: {"scope": query.account}}]
+                    filter: [{term: {scope: query.account}}]
                 }
             }
         }
@@ -32,9 +35,12 @@ async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
             continue;
         }
         let precision;
+        let token_data;
+        let errorMsg;
+
         const key = `${data.code}_${data.symbol}`;
 
-		if (testSet.has(key)) {
+        if (testSet.has(key)) {
             continue;
         }
 
@@ -45,7 +51,6 @@ async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
         if (fastify.tokenCache.has(key)) {
             precision = fastify.tokenCache.get(key).precision;
         } else {
-            let token_data;
             try {
                 token_data = await fastify.eosjs.rpc.get_currency_balance(data.code, query.account, data.symbol);
                 if (token_data.length > 0) {
@@ -54,20 +59,25 @@ async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
                     if (amount_arr.length === 2) {
                         precision = amount_arr[1].length;
                         fastify.tokenCache.set(key, {precision});
-                        // console.log('Caching token precision -', key, precision);
                     }
                 }
             } catch (e) {
-                console.log(`get_currency_balance error - contract:${data.code} - account:${query.account}`);
+                errorMsg = e.message;
             }
         }
 
-        response.tokens.push({
+        const resp: Record<string, any> = {
             symbol: data.symbol,
             precision: precision,
             amount: parseFloat(data.amount),
-            contract: data.code
-        });
+            contract: data.code,
+        };
+
+        if (errorMsg) {
+            resp.error = errorMsg;
+        }
+
+        response.tokens.push(resp);
     }
 
     return response;

api/routes/v2/health/health.ts

@@ -1,90 +1,137 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {connect} from "amqplib";
 import {timedQuery} from "../../../helpers/functions";
-import {getLastIndexedBlockWithTotalBlocks} from "../../../../helpers/common_functions";
+import {getFirstIndexedBlock, getLastIndexedBlockWithTotalBlocks} from "../../../../helpers/common_functions";
 
-async function checkRabbit(fastify: FastifyInstance) {
-	try {
-		const connection = await connect(fastify.manager.ampqUrl);
-		await connection.close();
-		return createHealth('RabbitMq', 'OK');
-	} catch (e) {
-		console.log(e);
-		return createHealth('RabbitMq', 'Error');
-	}
+
+async function checkRabbit(fastify: FastifyInstance): Promise<ServiceResponse<any>> {
+    try {
+        const connection = await connect(fastify.manager.ampqUrl);
+        await connection.close();
+        return createHealth('RabbitMq', 'OK');
+    } catch (e) {
+        console.log(e);
+        return createHealth('RabbitMq', 'Error');
+    }
 }
 
-async function checkNodeos(fastify: FastifyInstance) {
-	const rpc = fastify.manager.nodeosJsonRPC;
-	try {
-		const results = await rpc.get_info();
-		if (results) {
-			const diff = (new Date().getTime()) - (new Date(results.head_block_time + '+00:00').getTime());
-			return createHealth('NodeosRPC', 'OK', {
-				head_block_num: results.head_block_num,
-				head_block_time: results.head_block_time,
-				time_offset: diff,
-				last_irreversible_block: results.last_irreversible_block_num,
-				chain_id: results.chain_id
-			});
-		} else {
-			return createHealth('NodeosRPC', 'Error');
-		}
-	} catch (e) {
-		return createHealth('NodeosRPC', 'Error');
-	}
+interface NodeosService {
+    head_block_num: number;
+    head_block_time: string;
+    time_offset: number;
+    last_irreversible_block: number;
+    chain_id: string;
 }
 
-async function checkElastic(fastify: FastifyInstance) {
-	try {
-		let esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
-		let indexedBlocks = await getLastIndexedBlockWithTotalBlocks(fastify.elastic, fastify.manager.chain);
-		const data = {
-			last_indexed_block: indexedBlocks[0],
-			total_indexed_blocks: indexedBlocks[1] + 1,
-			active_shards: esStatus.body[0]['active_shards_percent']
-		};
-		let stat = 'OK';
-		esStatus.body.forEach(status => {
-			if (status.status === 'yellow' && stat !== 'Error') {
-				stat = 'Warning'
-			} else if (status.status === 'red') {
-				stat = 'Error'
-			}
-		});
-		return createHealth('Elasticsearch', stat, data);
-	} catch (e) {
-		console.log(e, 'Elasticsearch Error');
-		return createHealth('Elasticsearch', 'Error');
-	}
+async function checkNodeos(fastify: FastifyInstance): Promise<ServiceResponse<NodeosService>> {
+    const rpc = fastify.manager.nodeosJsonRPC;
+    try {
+        const results = await rpc.get_info();
+        if (results) {
+            const diff = (new Date().getTime()) - (new Date(results.head_block_time + '+00:00').getTime());
+            return createHealth('NodeosRPC', 'OK', {
+                head_block_num: results.head_block_num,
+                head_block_time: results.head_block_time,
+                time_offset: diff,
+                last_irreversible_block: results.last_irreversible_block_num,
+                chain_id: results.chain_id
+            });
+        } else {
+            return createHealth('NodeosRPC', 'Error');
+        }
+    } catch (e) {
+        return createHealth('NodeosRPC', 'Error');
+    }
 }
 
-function createHealth(name: string, status, data?: any) {
-	let time = Date.now();
-	return {
-		service: name,
-		status: status,
-		service_data: data,
-		time: time
-	}
+interface ESService {
+    first_indexed_block: number;
+    last_indexed_block: number;
+    total_indexed_blocks: number;
+    active_shards: string;
+    missing_blocks: number;
+    missing_pct: string;
+    head_offset: number;
 }
 
-async function getHealthQuery(fastify: FastifyInstance, request: FastifyRequest) {
-	let response = {
-		version: fastify.manager.current_version,
-		version_hash: fastify.manager.getServerHash(),
-		host: fastify.manager.config.api.server_name,
-		health: [],
-		features: fastify.manager.config.features
-	};
-	response.health.push(await checkRabbit(fastify));
-	response.health.push(await checkNodeos(fastify));
-	response.health.push(await checkElastic(fastify));
-	return response;
+interface ServiceResponse<T> {
+    service: string;
+    time: number;
+    status: any
+    service_data?: T;
+}
+
+async function checkElastic(fastify: FastifyInstance): Promise<ServiceResponse<ESService>> {
+    try {
+        let esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
+        let firstIndexedBlock: number;
+        const fib = await fastify.redis.get(`${fastify.manager.chain}::fib`);
+        if (fib) {
+            firstIndexedBlock = parseInt(fib);
+        } else {
+            firstIndexedBlock = await getFirstIndexedBlock(fastify.elastic, fastify.manager.chain);
+            await fastify.redis.set(`${fastify.manager.chain}::fib`, firstIndexedBlock);
+        }
+        let indexedBlocks = await getLastIndexedBlockWithTotalBlocks(fastify.elastic, fastify.manager.chain);
+        const lastIndexedBlock = indexedBlocks[0];
+        const totalIndexed = indexedBlocks[1] - 1;
+        const missingCounter = (lastIndexedBlock - firstIndexedBlock) - totalIndexed;
+        const missingPct = (missingCounter * 100 / indexedBlocks[1]).toFixed(2) + "%";
+        const data: ESService = {
+            active_shards: esStatus.body[0]['active_shards_percent'],
+            head_offset: null,
+            first_indexed_block: firstIndexedBlock,
+            last_indexed_block: lastIndexedBlock,
+            total_indexed_blocks: totalIndexed,
+            missing_blocks: missingCounter,
+            missing_pct: missingPct
+        };
+        let stat = 'OK';
+        esStatus.body.forEach(status => {
+            if (status.status === 'yellow' && stat !== 'Error') {
+                stat = 'Warning'
+            } else if (status.status === 'red') {
+                stat = 'Error'
+            }
+        });
+        return createHealth('Elasticsearch', stat, data);
+    } catch (e) {
+        console.log(e, 'Elasticsearch Error');
+        return createHealth('Elasticsearch', 'Error');
+    }
+}
+
+function createHealth(name: string, status, data?: any): ServiceResponse<any> {
+    let time = Date.now();
+    return {
+        service: name,
+        status: status,
+        service_data: data,
+        time: time
+    }
+}
+
+async function getHealthQuery(fastify: FastifyInstance) {
+    let response = {
+        version: fastify.manager.current_version,
+        version_hash: fastify.manager.getServerHash(),
+        host: fastify.manager.config.api.server_name,
+        health: [],
+        features: fastify.manager.config.features
+    };
+    response.health = await Promise.all([
+        checkRabbit(fastify),
+        checkNodeos(fastify),
+        checkElastic(fastify)
+    ]);
+    const es = response.health.find(value => value.service === 'Elasticsearch');
+    const nodeos = response.health.find(value => value.service === 'NodeosRPC');
+    es.service_data.head_offset = nodeos.service_data.head_block_num - es.service_data.last_indexed_block;
+    return response;
 }
 
 export function healthHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply) => {
-		reply.send(await timedQuery(getHealthQuery, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getHealthQuery, fastify, request, route));
+    }
 }

api/server.ts

@@ -8,243 +8,256 @@ import {registerPlugins} from "./plugins";
 import {AddressInfo} from "net";
 import {registerRoutes} from "./routes";
 import {generateOpenApiConfig} from "./config/open_api";
-import {createWriteStream, existsSync, mkdirSync} from "fs";
+import {createWriteStream, existsSync, mkdirSync, readFileSync} from "fs";
 import {SocketManager} from "./socketManager";
 import {HyperionModuleLoader} from "../modules/loader";
 import {extendedActions} from "./routes/v2-history/get_actions/definitions";
 import {io, Socket} from "socket.io-client";
+import {CacheManager} from "./helpers/cacheManager";
+
+import {bootstrap} from 'global-agent';
+bootstrap();
 
 class HyperionApiServer {
 
-	private conf: HyperionConfig;
-	private readonly manager: ConnectionManager;
-	mLoader: HyperionModuleLoader;
-	private readonly fastify;
-	private readonly chain: string;
-	socketManager: SocketManager;
+    private hub: Socket;
+    private readonly fastify;
+    private readonly chain: string;
+    private readonly conf: HyperionConfig;
+    private readonly manager: ConnectionManager;
+    private readonly cacheManager: CacheManager;
 
-	private hub: Socket;
+    socketManager: SocketManager;
+    mLoader: HyperionModuleLoader;
 
-	constructor() {
-		const cm = new ConfigurationModule();
-		this.conf = cm.config;
-		this.chain = this.conf.settings.chain;
-		process.title = `hyp-${this.chain}-api`;
-		this.manager = new ConnectionManager(cm);
-		this.manager.calculateServerHash();
-		this.mLoader = new HyperionModuleLoader(cm);
+    constructor() {
 
-		if (!existsSync('./logs/' + this.chain)) {
-			mkdirSync('./logs/' + this.chain, {recursive: true});
-		}
+        const package_json = JSON.parse(readFileSync('./package.json').toString());
+        hLog(`--------- Hyperion API ${package_json.version} ---------`);
 
-		const logStream = createWriteStream('./logs/' + this.chain + '/api.access.log');
+        const cm = new ConfigurationModule();
+        this.conf = cm.config;
+        this.chain = this.conf.settings.chain;
+        process.title = `hyp-${this.chain}-api`;
+        this.manager = new ConnectionManager(cm);
+        this.manager.calculateServerHash();
+        this.mLoader = new HyperionModuleLoader(cm);
+        this.cacheManager = new CacheManager(this.conf);
 
-		const loggerOpts = {
-			stream: logStream,
-			redact: ['req.headers.authorization'],
-			level: 'info',
-			prettyPrint: true,
-			serializers: {
-				res: (reply) => {
-					return {
-						statusCode: reply.statusCode
-					};
-				},
-				req: (request) => {
-					return {
-						method: request.method,
-						url: request.url,
-						ip: request.headers['x-real-ip']
-					}
-				}
-			}
-		};
+        if (!existsSync('./logs/' + this.chain)) {
+            mkdirSync('./logs/' + this.chain, {recursive: true});
+        }
 
-		this.fastify = fastify({
-			ignoreTrailingSlash: false,
-			trustProxy: true,
-			pluginTimeout: 5000,
-			logger: this.conf.api.access_log ? loggerOpts : false
-		});
+        const logStream = createWriteStream('./logs/' + this.chain + '/api.access.log');
 
-		this.fastify.decorate('manager', this.manager);
+        const loggerOpts = {
+            stream: logStream,
+            redact: ['req.headers.authorization'],
+            level: 'info',
+            prettyPrint: true,
+            serializers: {
+                res: (reply) => {
+                    return {
+                        statusCode: reply.statusCode
+                    };
+                },
+                req: (request) => {
+                    return {
+                        method: request.method,
+                        url: request.url,
+                        ip: request.headers['x-real-ip']
+                    }
+                }
+            }
+        };
 
-		// import get_actions query params from custom modules
-		const extendedActionsSet: Set<string> = new Set([...extendedActions]);
-		for (const qPrefix of this.mLoader.extendedActions) {
-			extendedActionsSet.add(qPrefix);
-		}
-		this.fastify.decorate('allowedActionQueryParamSet', extendedActionsSet);
+        this.fastify = fastify({
+            ignoreTrailingSlash: false,
+            trustProxy: true,
+            pluginTimeout: 5000,
+            logger: this.conf.api.access_log ? loggerOpts : false
+        });
 
-		// define chain api url for /v1/chain/ redirects
-		let chainApiUrl: string = this.conf.api.push_api;
-		if (chainApiUrl === null || chainApiUrl === "") {
-			chainApiUrl = this.manager.conn.chains[this.chain].http;
-		}
-		this.fastify.decorate('chain_api', chainApiUrl);
+        this.fastify.decorate('cacheManager', this.cacheManager);
 
-		// define optional push api url for /v1/chain/push_transaction
-		if (this.conf.api.push_api) {
-			this.fastify.decorate('push_api', this.conf.api.push_api);
-		}
+        this.fastify.decorate('manager', this.manager);
 
-		hLog(`Chain API URL: "${this.fastify.chain_api}" | Push API URL: "${this.fastify.push_api}"`);
+        // import get_actions query params from custom modules
+        const extendedActionsSet: Set<string> = new Set([...extendedActions]);
+        for (const qPrefix of this.mLoader.extendedActions) {
+            extendedActionsSet.add(qPrefix);
+        }
+        this.fastify.decorate('allowedActionQueryParamSet', extendedActionsSet);
 
-		const ioRedisClient = new IORedis(this.manager.conn.redis);
+        // define chain api url for /v1/chain/ redirects
+        let chainApiUrl: string = this.conf.api.chain_api;
+        if (chainApiUrl === null || chainApiUrl === "") {
+            chainApiUrl = this.manager.conn.chains[this.chain].http;
+        }
+        this.fastify.decorate('chain_api', chainApiUrl);
 
-		const pluginParams = {
-			fastify_elasticsearch: {
-				client: this.manager.elasticsearchClient
-			},
-			fastify_redis: this.manager.conn.redis,
-			fastify_eosjs: this.manager,
-		} as any;
+        // define optional push api url for /v1/chain/push_transaction
+        if (this.conf.api.push_api) {
+            this.fastify.decorate('push_api', this.conf.api.push_api);
+        }
 
-		if(!this.conf.api.disable_rate_limit) {
-			let rateLimiterWhitelist = ['127.0.0.1'];
-			if (this.conf.api.rate_limit_allow && this.conf.api.rate_limit_allow.length > 0) {
-				const tempSet = new Set<string>(['127.0.0.1', ...this.conf.api.rate_limit_allow]);
-				rateLimiterWhitelist = [...tempSet];
-			}
-			let rateLimiterRPM = 1000;
-			if (this.conf.api.rate_limit_rpm) {
-				rateLimiterRPM = this.conf.api.rate_limit_rpm;
-			}
-			pluginParams.fastify_rate_limit = {
-				max: rateLimiterRPM,
-				allowList: rateLimiterWhitelist,
-				timeWindow: '1 minute',
-				redis: ioRedisClient
-			}
-		}
+        hLog(`Chain API URL: "${this.fastify.chain_api}" | Push API URL: "${this.fastify.push_api}"`);
 
-		if (this.conf.features.streaming.enable) {
-			this.activateStreaming();
-		}
+        const ioRedisClient = new IORedis(this.manager.conn.redis);
 
-		const docsConfig = generateOpenApiConfig(this.manager.config);
-		if(docsConfig) {
-			pluginParams.fastify_swagger = docsConfig;
-		}
+        const pluginParams = {
+            fastify_elasticsearch: {
+                client: this.manager.elasticsearchClient
+            },
+            fastify_redis: this.manager.conn.redis,
+            fastify_eosjs: this.manager,
+        } as any;
 
-		registerPlugins(this.fastify, pluginParams);
+        if (!this.conf.api.disable_rate_limit) {
+            let rateLimiterWhitelist = ['127.0.0.1'];
+            if (this.conf.api.rate_limit_allow && this.conf.api.rate_limit_allow.length > 0) {
+                const tempSet = new Set<string>(['127.0.0.1', ...this.conf.api.rate_limit_allow]);
+                rateLimiterWhitelist = [...tempSet];
+            }
+            let rateLimiterRPM = 1000;
+            if (this.conf.api.rate_limit_rpm) {
+                rateLimiterRPM = this.conf.api.rate_limit_rpm;
+            }
+            pluginParams.fastify_rate_limit = {
+                max: rateLimiterRPM,
+                allowList: rateLimiterWhitelist,
+                timeWindow: '1 minute',
+                redis: ioRedisClient
+            }
+        }
 
-		this.addGenericTypeParsing();
-	}
+        if (this.conf.features.streaming.enable) {
+            this.activateStreaming();
+        }
 
-	activateStreaming() {
-		console.log('Importing stream module');
-		import('./socketManager').then((mod) => {
-			const connOpts = this.manager.conn.chains[this.chain];
+        const docsConfig = generateOpenApiConfig(this.manager.config);
+        if (docsConfig) {
+            pluginParams.fastify_swagger = docsConfig;
+        }
 
-			let _port = 57200;
-			if (connOpts.WS_ROUTER_PORT) {
-				_port = connOpts.WS_ROUTER_PORT;
-			}
+        registerPlugins(this.fastify, pluginParams);
 
-			let _host = "127.0.0.1";
-			if (connOpts.WS_ROUTER_HOST) {
-				_host = connOpts.WS_ROUTER_HOST;
-			}
+        this.addGenericTypeParsing();
+    }
 
-			this.socketManager = new mod.SocketManager(
-				this.fastify,
-				`http://${_host}:${_port}`,
-				this.manager.conn.redis
-			);
-			this.socketManager.startRelay();
-		});
-	}
+    activateStreaming() {
+        console.log('Importing stream module');
+        import('./socketManager').then((mod) => {
+            const connOpts = this.manager.conn.chains[this.chain];
 
-	private addGenericTypeParsing() {
-		this.fastify.addContentTypeParser('*', (request, payload, done) => {
-			let data = '';
-			payload.on('data', chunk => {
-				data += chunk;
-			});
-			payload.on('end', () => {
-				done(null, data);
-			});
-			payload.on('error', (err) => {
-				console.log('---- Content Parsing Error -----');
-				console.log(err);
-			});
-		});
-	}
+            let _port = 57200;
+            if (connOpts.WS_ROUTER_PORT) {
+                _port = connOpts.WS_ROUTER_PORT;
+            }
 
-	async init() {
+            let _host = "127.0.0.1";
+            if (connOpts.WS_ROUTER_HOST) {
+                _host = connOpts.WS_ROUTER_HOST;
+            }
 
-		await this.mLoader.init();
+            this.socketManager = new mod.SocketManager(
+                this.fastify,
+                `http://${_host}:${_port}`,
+                this.manager.conn.redis
+            );
+            this.socketManager.startRelay();
+        });
+    }
 
-		// add custom plugin routes
-		for (const plugin of this.mLoader.plugins) {
-			if (plugin.hasApiRoutes) {
-				plugin.addRoutes(this.fastify);
-				plugin.chainName = this.chain;
-			}
-		}
+    private addGenericTypeParsing() {
+        this.fastify.addContentTypeParser('*', (request, payload, done) => {
+            let data = '';
+            payload.on('data', chunk => {
+                data += chunk;
+            });
+            payload.on('end', () => {
+                done(null, data);
+            });
+            payload.on('error', (err) => {
+                console.log('---- Content Parsing Error -----');
+                console.log(err);
+            });
+        });
+    }
 
-		registerRoutes(this.fastify);
+    async init() {
 
-		// register documentation when ready
-		this.fastify.ready().then(async () => {
-			await this.fastify.swagger();
-		}, (err) => {
-			hLog('an error happened', err)
-		});
+        await this.mLoader.init();
 
-		try {
-			await this.fastify.listen({
-				host: this.conf.api.server_addr,
-				port: this.conf.api.server_port
-			});
-			hLog(`${this.chain} hyperion api ready and listening on port ${(this.fastify.server.address() as AddressInfo).port}`);
-			this.startHyperionHub();
-		} catch (err) {
-			hLog(err);
-			process.exit(1)
-		}
-	}
+        // add custom plugin routes
+        for (const plugin of this.mLoader.plugins) {
+            if (plugin.hasApiRoutes) {
+                hLog(`Adding routes for plugin: ${plugin.internalPluginName}`);
+                plugin.addRoutes(this.fastify);
+                plugin.chainName = this.chain;
+            }
+        }
 
-	startHyperionHub() {
-		if (this.conf.hub) {
-			const url = this.conf.hub.inform_url;
-			hLog(`Connecting API to Hyperion Hub`);
-			this.hub = io(url, {
-				query: {
-					key: this.conf.hub.publisher_key,
-					client_mode: 'false'
-				}
-			});
-			this.hub.on('connect', () => {
-				hLog(`Hyperion Hub connected!`);
-				this.emitHubApiUpdate();
-			});
-		}
-	}
+        registerRoutes(this.fastify);
 
-	private emitHubApiUpdate() {
-		this.hub.emit('hyp_info', {
-			type: 'api',
-			production: this.conf.hub.production,
-			location: this.conf.hub.location,
-			chainId: this.manager.conn.chains[this.chain].chain_id,
-			providerName: this.conf.api.provider_name,
-			explorerEnabled: this.conf.api.enable_explorer,
-			providerUrl: this.conf.api.provider_url,
-			providerLogo: this.conf.api.provider_logo,
-			chainLogo: this.conf.api.chain_logo_url,
-			chainCodename: this.chain,
-			chainName: this.conf.api.chain_name,
-			endpoint: this.conf.api.server_name,
-			features: this.conf.features,
-			filters: {
-				blacklists: this.conf.blacklists,
-				whitelists: this.conf.whitelists
-			}
-		});
-	}
+        // register documentation when ready
+        this.fastify.ready().then(async () => {
+            await this.fastify.swagger();
+        }, (err) => {
+            hLog('an error happened', err)
+        });
+
+        try {
+            await this.fastify.listen({
+                host: this.conf.api.server_addr,
+                port: this.conf.api.server_port
+            });
+            hLog(`${this.chain} hyperion api ready and listening on port ${(this.fastify.server.address() as AddressInfo).port}`);
+            this.startHyperionHub();
+        } catch (err) {
+            hLog(err);
+            process.exit(1)
+        }
+    }
+
+    startHyperionHub() {
+        if (this.conf.hub) {
+            const url = this.conf.hub.inform_url;
+            hLog(`Connecting API to Hyperion Hub`);
+            this.hub = io(url, {
+                query: {
+                    key: this.conf.hub.publisher_key,
+                    client_mode: 'false'
+                }
+            });
+            this.hub.on('connect', () => {
+                hLog(`Hyperion Hub connected!`);
+                this.emitHubApiUpdate();
+            });
+        }
+    }
+
+    private emitHubApiUpdate() {
+        this.hub.emit('hyp_info', {
+            type: 'api',
+            production: this.conf.hub.production,
+            location: this.conf.hub.location,
+            chainId: this.manager.conn.chains[this.chain].chain_id,
+            providerName: this.conf.api.provider_name,
+            explorerEnabled: this.conf.plugins.explorer?.enabled,
+            providerUrl: this.conf.api.provider_url,
+            providerLogo: this.conf.api.provider_logo,
+            chainLogo: this.conf.api.chain_logo_url,
+            chainCodename: this.chain,
+            chainName: this.conf.api.chain_name,
+            endpoint: this.conf.api.server_name,
+            features: this.conf.features,
+            filters: {
+                blacklists: this.conf.blacklists,
+                whitelists: this.conf.whitelists
+            }
+        });
+    }
 }
 
 const server = new HyperionApiServer();

api/socketManager.ts

@@ -1,384 +1,202 @@
-import {checkFilter, hLog} from '../helpers/common_functions';
+import {hLog} from '../helpers/common_functions';
 import {Server, Socket} from 'socket.io';
 import {createAdapter} from 'socket.io-redis';
 import {io} from 'socket.io-client';
 import {FastifyInstance} from "fastify";
 import IORedis from "ioredis";
+import {App, TemplatedApp} from 'uWebSockets.js';
+import {streamPastActions, streamPastDeltas} from "./helpers/functions";
 
 export interface StreamDeltasRequest {
-	code: string;
-	table: string;
-	scope: string;
-	payer: string;
-	start_from: number | string;
-	read_until: number | string;
+    code: string;
+    table: string;
+    scope: string;
+    payer: string;
+    start_from: number | string;
+    read_until: number | string;
 }
 
 export interface RequestFilter {
-	field: string;
-	value: string;
+    field: string;
+    value: string;
 }
 
 export interface StreamActionsRequest {
-	contract: string;
-	account: string;
-	action: string;
-	filters: RequestFilter[];
-	start_from: number | string;
-	read_until: number | string;
+    contract: string;
+    account: string;
+    action: string;
+    filters: RequestFilter[];
+    start_from: number | string;
+    read_until: number | string;
 }
 
-async function addBlockRangeOpts(data, search_body, fastify: FastifyInstance) {
-
-	let timeRange;
-	let blockRange;
-	let head;
-
-	if (typeof data['start_from'] === 'string' && data['start_from'] !== '') {
-		if (!timeRange) {
-			timeRange = {"@timestamp": {}};
-		}
-		timeRange["@timestamp"]['gte'] = data['start_from'];
-	}
-
-	if (typeof data['read_until'] === 'string' && data['read_until'] !== '') {
-		if (!timeRange) {
-			timeRange = {"@timestamp": {}};
-		}
-		timeRange["@timestamp"]['lte'] = data['read_until'];
-	}
-
-	if (typeof data['start_from'] === 'number' && data['start_from'] !== 0) {
-		if (!blockRange) {
-			blockRange = {"block_num": {}};
-		}
-		if (data['start_from'] < 0) {
-			if (!head) {
-				head = (await fastify.eosjs.rpc.get_info()).head_block_num;
-			}
-			blockRange["block_num"]['gte'] = head + data['start_from'];
-		} else {
-			blockRange["block_num"]['gte'] = data['start_from'];
-		}
-	}
-
-	if (typeof data['read_until'] === 'number' && data['read_until'] !== 0) {
-		if (!blockRange) {
-			blockRange = {"block_num": {}};
-		}
-		if (data['read_until'] < 0) {
-			if (!head) {
-				head = (await fastify.eosjs.rpc.get_info()).head_block_num;
-			}
-			blockRange["block_num"]['lte'] = head + data['read_until'];
-		} else {
-			blockRange["block_num"]['lte'] = data['read_until'];
-		}
-	}
-
-	if (timeRange) {
-		search_body.query.bool.must.push({
-			range: timeRange,
-		});
-	}
-	if (blockRange) {
-		search_body.query.bool.must.push({
-			range: blockRange,
-		});
-	}
-}
-
-function addTermMatch(data, search_body, field) {
-	if (data[field] !== '*' && data[field] !== '') {
-		const termQuery = {};
-		termQuery[field] = data[field];
-		search_body.query.bool.must.push({'term': termQuery});
-	}
-}
-
-const deltaQueryFields = ['code', 'table', 'scope', 'payer'];
-
-async function streamPastDeltas(fastify: FastifyInstance, socket, data) {
-	const search_body = {
-		query: {bool: {must: []}},
-		sort: {block_num: 'asc'},
-	};
-	await addBlockRangeOpts(data, search_body, fastify);
-	deltaQueryFields.forEach(f => {
-		addTermMatch(data, search_body, f);
-	});
-	const responseQueue = [];
-	let counter = 0;
-	const init_response = await fastify.elastic.search({
-		index: fastify.manager.chain + '-delta-*',
-		scroll: '30s',
-		size: 20,
-		body: search_body,
-	});
-	responseQueue.push(init_response);
-	while (responseQueue.length) {
-		const {body} = responseQueue.shift();
-		counter += body['hits']['hits'].length;
-		if (socket.connected) {
-			socket.emit('message', {
-				type: 'delta_trace',
-				mode: 'history',
-				messages: body['hits']['hits'].map(doc => doc._source),
-			});
-		} else {
-			hLog('LOST CLIENT');
-			break;
-		}
-		if (body['hits'].total.value === counter) {
-			hLog(`${counter} past deltas streamed to ${socket.id}`);
-			break;
-		}
-
-		const next_response = await fastify.elastic.scroll({
-			body: {
-				scroll_id: body['_scroll_id'],
-				scroll: '30s'
-			}
-		});
-		responseQueue.push(next_response);
-	}
-}
-
-async function streamPastActions(fastify: FastifyInstance, socket, data) {
-	const search_body = {
-		query: {bool: {must: []}},
-		sort: {global_sequence: 'asc'},
-	};
-
-	await addBlockRangeOpts(data, search_body, fastify);
-
-	if (data.account !== '') {
-		search_body.query.bool.must.push({
-			bool: {
-				should: [
-					{term: {'notified': data.account}},
-					{term: {'act.authorization.actor': data.account}},
-				],
-			},
-		});
-	}
-
-	if (data.contract !== '*' && data.contract !== '') {
-		search_body.query.bool.must.push({'term': {'act.account': data.contract}});
-	}
-
-	if (data.action !== '*' && data.action !== '') {
-		search_body.query.bool.must.push({'term': {'act.name': data.action}});
-	}
-
-	const onDemandFilters = [];
-	if (data.filters.length > 0) {
-		data.filters.forEach(f => {
-			if (f.field && f.value) {
-				if (f.field.startsWith('@') && !f.field.startsWith('act.data')) {
-					const _q = {};
-					_q[f.field] = f.value;
-					search_body.query.bool.must.push({'term': _q});
-				} else {
-					onDemandFilters.push(f);
-				}
-			}
-		});
-	}
-
-	const responseQueue = [];
-	let counter = 0;
-
-	const init_response = await fastify.elastic.search({
-		index: fastify.manager.chain + '-action-*',
-		scroll: '30s',
-		size: 20,
-		body: search_body,
-	});
-	responseQueue.push(init_response);
-	while (responseQueue.length) {
-		const {body} = responseQueue.shift();
-		const enqueuedMessages = [];
-		counter += body['hits']['hits'].length;
-		for (const doc of body['hits']['hits']) {
-			let allow = false;
-			if (onDemandFilters.length > 0) {
-				allow = onDemandFilters.every(filter => {
-					return checkFilter(filter, doc._source);
-				});
-			} else {
-				allow = true;
-			}
-			if (allow) {
-				enqueuedMessages.push(doc._source);
-			}
-		}
-		if (socket.connected) {
-			socket.emit('message', {type: 'action_trace', mode: 'history', messages: enqueuedMessages});
-		} else {
-			hLog('LOST CLIENT');
-			break;
-		}
-		if (body['hits'].total.value === counter) {
-			hLog(`${counter} past actions streamed to ${socket.id}`);
-			break;
-		}
-		if (init_response.body.hits.total.value < 1000) {
-			const next_response = await fastify.elastic.scroll({
-				body: {scroll_id: body['_scroll_id'], scroll: '30s'}
-			});
-			responseQueue.push(next_response);
-		} else {
-			hLog('Request too large!');
-			socket.emit('message', {type: 'action_trace', mode: 'history', messages: []});
-		}
-	}
-}
 
 export class SocketManager {
 
-	private io: Server;
-	private relay;
-	relay_restored = true;
-	relay_down = false;
-	private readonly url;
-	private readonly server: FastifyInstance;
+    private io: Server;
+    private relay;
+    relay_restored = true;
+    relay_down = false;
+    private readonly url;
+    private readonly server: FastifyInstance;
+    private readonly uwsApp: TemplatedApp;
 
-	constructor(fastify: FastifyInstance, url, redisOptions) {
-		this.server = fastify;
-		this.url = url;
+    constructor(fastify: FastifyInstance, url, redisOptions) {
+        this.server = fastify;
+        this.url = url;
+        this.uwsApp = App({});
 
-		this.io = new Server(fastify.server, {
-			allowEIO3: true,
-			transports: ['websocket', 'polling'],
-		});
+        // this.io = new Server(fastify.server, {
+        // 	allowEIO3: true,
+        // 	transports: ['websocket', 'polling'],
+        // });
 
-		const pubClient = new IORedis(redisOptions);
-		const subClient = pubClient.duplicate();
-		this.io.adapter(createAdapter({pubClient, subClient}));
+        // WS Server for public access
+        this.io = new Server({
+            transports: ['websocket'],
+            path: '/stream'
+        });
 
-		this.io.on('connection', (socket: Socket) => {
+        this.io.attachApp(this.uwsApp);
 
-			if (socket.handshake.headers['x-forwarded-for']) {
-				hLog(`[socket] ${socket.id} connected via ${socket.handshake.headers['x-forwarded-for']}`);
-			}
+        const pubClient = new IORedis(redisOptions);
+        const subClient = pubClient.duplicate();
+        this.io.adapter(createAdapter({pubClient, subClient}));
 
-			socket.emit('message', {
-				event: 'handshake',
-				chain: fastify.manager.chain,
-			});
+        this.io.on('connection', (socket: Socket) => {
 
-			if (this.relay) {
-				this.relay.emit('event', {
-					type: 'client_count',
-					counter: this.io.sockets.sockets.size,
-				});
-			}
+            if (socket.handshake.headers['x-forwarded-for']) {
+                hLog(`[socket] ${socket.id} connected via ${socket.handshake.headers['x-forwarded-for']}`);
+            }
 
-			socket.on('delta_stream_request', async (data: StreamDeltasRequest, callback) => {
-				if (typeof callback === 'function' && data) {
-					try {
-						if (data.start_from) {
-							await streamPastDeltas(this.server, socket, data);
-						}
-						this.emitToRelay(data, 'delta_request', socket, callback);
-					} catch (e) {
-						console.log(e);
-					}
-				}
-			});
+            socket.emit('message', {
+                event: 'handshake',
+                chain: fastify.manager.chain,
+            });
 
-			socket.on('action_stream_request', async (data: StreamActionsRequest, callback) => {
-				if (typeof callback === 'function' && data) {
-					try {
-						if (data.start_from) {
-							await streamPastActions(this.server, socket, data);
-						}
-						this.emitToRelay(data, 'action_request', socket, callback);
-					} catch (e) {
-						console.log(e);
-					}
-				}
-			});
+            if (this.relay) {
+                this.relay.emit('event', {
+                    type: 'client_count',
+                    counter: this.io.sockets.sockets.size,
+                });
+            }
 
-			socket.on('disconnect', (reason) => {
-				hLog(`[socket] ${socket.id} disconnected - ${reason}`);
-				this.relay.emit('event', {
-					type: 'client_disconnected',
-					id: socket.id,
-					reason,
-				});
-			});
-		});
-		hLog('Websocket manager loaded!');
-	}
+            socket.on('delta_stream_request', async (data: StreamDeltasRequest, callback) => {
+                if (typeof callback === 'function' && data) {
+                    try {
+                        if (data.start_from) {
+                            await streamPastDeltas(this.server, socket, data);
+                        }
+                        this.emitToRelay(data, 'delta_request', socket, callback);
+                    } catch (e) {
+                        console.log(e);
+                    }
+                }
+            });
 
-	startRelay() {
-		hLog(`starting relay - ${this.url}`);
+            socket.on('action_stream_request', async (data: StreamActionsRequest, callback) => {
+                if (typeof callback === 'function' && data) {
+                    try {
+                        if (data.start_from) {
+                            await streamPastActions(this.server, socket, data);
+                        }
+                        this.emitToRelay(data, 'action_request', socket, callback);
+                    } catch (e) {
+                        console.log(e);
+                    }
+                }
+            });
 
-		this.relay = io(this.url, {path: '/router'});
+            socket.on('disconnect', (reason) => {
+                hLog(`[socket] ${socket.id} disconnected - ${reason}`);
+                this.relay.emit('event', {
+                    type: 'client_disconnected',
+                    id: socket.id,
+                    reason,
+                });
+            });
+        });
 
-		this.relay.on('connect', () => {
-			hLog('Relay Connected!');
-			if (this.relay_down) {
-				this.relay_restored = true;
-				this.relay_down = false;
-				this.io.emit('status', 'relay_restored');
-			}
-		});
+        try {
+            const port = this.server.manager.config.api.stream_port || 1234;
+            this.uwsApp.listen(port, () => {
+                hLog(`Socket.IO via uWS started on port ${port}`);
+            });
+        } catch (e) {
+            hLog(e.message);
+        }
 
-		this.relay.on('disconnect', () => {
-			hLog('Relay disconnected!');
-			this.io.emit('status', 'relay_down');
-			this.relay_down = true;
-			this.relay_restored = false;
-		});
+        hLog('Websocket manager loaded!');
+    }
 
-		this.relay.on('delta', (traceData) => {
-			this.emitToClient(traceData, 'delta_trace');
-		});
+    /*
+    WS Relay will connect to the indexer
+     */
+    startRelay() {
+        hLog(`starting relay - ${this.url}`);
+        this.relay = io(this.url, {path: '/router'});
 
-		this.relay.on('trace', (traceData) => {
-			this.emitToClient(traceData, 'action_trace');
-		});
+        this.relay.on('connect', () => {
+            hLog('Relay Connected!');
+            if (this.relay_down) {
+                this.relay_restored = true;
+                this.relay_down = false;
+                this.io.emit('status', 'relay_restored');
+            }
+        });
 
-		// Relay LIB info to clients;
-		this.relay.on('lib_update', (data) => {
-			if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-				this.io.emit('lib_update', data);
-			}
-		});
+        this.relay.on('disconnect', () => {
+            hLog('Relay disconnected!');
+            this.io.emit('status', 'relay_down');
+            this.relay_down = true;
+            this.relay_restored = false;
+        });
 
-		// Relay LIB info to clients;
-		this.relay.on('fork_event', (data) => {
-			hLog(data);
-			if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-				this.io.emit('fork_event', data);
-			}
-		});
-	}
+        this.relay.on('delta', (traceData) => {
+            this.emitToClient(traceData, 'delta_trace');
+        });
 
-	emitToClient(traceData, type) {
-		if (this.io.sockets.sockets.has(traceData.client)) {
-			this.io.sockets.sockets.get(traceData.client).emit('message', {
-				type: type,
-				mode: 'live',
-				message: traceData.message,
-			});
-		}
-	}
+        this.relay.on('trace', (traceData) => {
+            this.emitToClient(traceData, 'action_trace');
+        });
 
-	emitToRelay(data, type, socket, callback) {
-		if (this.relay.connected) {
-			this.relay.emit('event', {
-				type: type,
-				client_socket: socket.id,
-				request: data,
-			}, (response) => {
-				callback(response);
-			});
-		} else {
-			callback('STREAMING_OFFLINE');
-		}
-	}
+        // Relay LIB info to clients;
+        this.relay.on('lib_update', (data) => {
+            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+                this.io.emit('lib_update', data);
+            }
+        });
+
+        // Relay LIB info to clients;
+        this.relay.on('fork_event', (data) => {
+            hLog(data);
+            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+                this.io.emit('fork_event', data);
+            }
+        });
+    }
+
+    emitToClient(traceData, type) {
+        if (this.io.sockets.sockets.has(traceData.client)) {
+            this.io.sockets.sockets.get(traceData.client).emit('message', {
+                type: type,
+                mode: 'live',
+                message: traceData.message,
+            });
+        }
+    }
+
+    emitToRelay(data, type, socket, callback) {
+        if (this.relay.connected) {
+            this.relay.emit('event', {
+                type: type,
+                client_socket: socket.id,
+                request: data,
+            }, (response) => {
+                callback(response);
+            });
+        } else {
+            callback('STREAMING_OFFLINE');
+        }
+    }
 }

auto_setup.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-git clone https://github.com/eosrio/hyperion-history-api.git  || exit
-cd hyperion-history-api || exit
-bash ./install_env.sh

chains/example.config.json

@@ -2,9 +2,11 @@
   "api": {
     "enabled": true,
     "pm2_scaling": 1,
+    "node_max_old_space_size": 1024,
     "chain_name": "EXAMPLE Chain",
     "server_addr": "127.0.0.1",
     "server_port": 7000,
+    "stream_port": 1234,
     "server_name": "127.0.0.1:7000",
     "provider_name": "Example Provider",
     "provider_url": "https://example.com",
@@ -21,7 +23,6 @@
       "get_trx_actions": 200
     },
     "access_log": false,
-    "enable_explorer": false,
     "chain_api_error_log": false,
     "custom_core_token": "",
     "enable_export_action": false,
@@ -29,10 +30,21 @@
     "rate_limit_rpm": 1000,
     "rate_limit_allow": [],
     "disable_tx_cache": false,
-    "tx_cache_expiration_sec": 3600
+    "tx_cache_expiration_sec": 3600,
+    "v1_chain_cache": [
+      {
+        "path": "get_block",
+        "ttl": 3000
+      },
+      {
+        "path": "get_info",
+        "ttl": 500
+      }
+    ]
   },
   "indexer": {
     "enabled": true,
+    "node_max_old_space_size": 4096,
     "start_on": 0,
     "stop_on": 0,
     "rewrite": false,
@@ -51,7 +63,7 @@
     "preview": false,
     "chain": "eos",
     "eosio_alias": "eosio",
-    "parser": "2.1",
+    "parser": "3.2",
     "auto_stop": 0,
     "index_version": "v1",
     "debug": false,
@@ -66,7 +78,8 @@
     "hot_warm_policy": false,
     "custom_policy": "",
     "bypass_index_map": false,
-    "index_partition_size": 10000000
+    "index_partition_size": 10000000,
+    "es_replicas": 0
   },
   "blacklists": {
     "actions": [],
@@ -79,21 +92,21 @@
     "root_only": false
   },
   "scaling": {
-    "readers":            1,
-    "ds_queues":          1,
-    "ds_threads":         1,
-    "ds_pool_size":       1,
-    "indexing_queues":    1,
-    "ad_idx_queues":      1,
-    "dyn_idx_queues":     1,
-    "max_autoscale":      4,
-    "batch_size":         5000,
-    "resume_trigger":     5000,
+    "readers": 1,
+    "ds_queues": 1,
+    "ds_threads": 1,
+    "ds_pool_size": 1,
+    "indexing_queues": 1,
+    "ad_idx_queues": 1,
+    "dyn_idx_queues": 1,
+    "max_autoscale": 4,
+    "batch_size": 5000,
+    "resume_trigger": 5000,
     "auto_scale_trigger": 20000,
-    "block_queue_limit":  10000,
-    "max_queue_limit":    100000,
-    "routing_mode":       "round_robin",
-    "polling_interval":   10000
+    "block_queue_limit": 10000,
+    "max_queue_limit": 100000,
+    "routing_mode": "round_robin",
+    "polling_interval": 10000
   },
   "features": {
     "streaming": {
@@ -107,7 +120,7 @@
       "voters": true
     },
     "index_deltas": true,
-    "index_transfer_memo": false,
+    "index_transfer_memo": true,
     "index_all_deltas": true,
     "deferred_trx": false,
     "failed_trx": false,

connections-demo.json

@@ -1,55 +0,0 @@
-{
-  "amqp": {
-    "host": "127.0.0.1:5672",
-    "api": "127.0.0.1:15672",
-    "user": "username",
-    "pass": "password",
-    "vhost": "hyperion"
-  },
-  "elasticsearch": {
-    "host": "127.0.0.1:9200",
-    "ingest_nodes": [
-      "127.0.0.1:9200"
-    ],
-    "user": "elastic",
-    "pass": "password"
-  },
-  "redis": {
-    "host": "127.0.0.1",
-    "port": "6379"
-  },
-  "chains": {
-    "eos": {
-      "name": "EOS Mainnet",
-      "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
-      "http": "http://127.0.0.1:8888",
-      "ship": "ws://127.0.0.1:8080",
-      "WS_ROUTER_PORT": 7001,
-      "WS_ROUTER_HOST": "127.0.0.1"
-    },
-    "eos2": {
-      "name": "EOS Mainnet",
-      "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
-      "http": "http://127.0.0.1:8888",
-      "ship": "ws://127.0.0.1:8080",
-      "WS_ROUTER_PORT": 7001,
-      "WS_ROUTER_HOST": "127.0.0.1"
-    },
-    "demo": {
-      "name": "Demo Chain",
-      "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
-      "http": "http://api.eosrio.io",
-      "ship": "ws://200.201.191.228:58013",
-      "WS_ROUTER_PORT": 57201,
-      "WS_ROUTER_HOST": "192.168.0.1"
-    },
-    "demo2": {
-      "name": "Demo Chain",
-      "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
-      "http": "http://api.eosrio.io",
-      "ship": "ws://200.201.191.228:58013",
-      "WS_ROUTER_PORT": 57201,
-      "WS_ROUTER_HOST": "192.168.0.1"
-    }
-  }
-}

connections/manager.class.ts

@@ -9,170 +9,167 @@ import {StateHistorySocket} from "./state-history";
 import fetch from 'cross-fetch';
 import {exec} from "child_process";
 import {hLog} from "../helpers/common_functions";
+import {readFileSync} from "fs";
 
 export class ConnectionManager {
 
-	config: HyperionConfig;
-	conn: HyperionConnections;
+    config: HyperionConfig;
+    conn: HyperionConnections;
 
-	chain: string;
-	last_commit_hash: string;
-	current_version: string;
+    chain: string;
+    last_commit_hash: string;
+    current_version: string;
 
-	private readonly esIngestClients: Client[];
-	private esIngestClient: Client;
+    private readonly esIngestClients: Client[];
+    private esIngestClient: Client;
 
-	constructor(private cm: ConfigurationModule) {
-		this.config = cm.config;
-		this.conn = cm.connections;
-		if (!this.conn.amqp.protocol) {
-			this.conn.amqp.protocol = 'http';
-		}
-		this.chain = this.config.settings.chain;
-		this.esIngestClients = [];
-		this.prepareESClient();
-		this.prepareIngestClients();
-	}
+    constructor(private cm: ConfigurationModule) {
+        this.config = cm.config;
+        this.conn = cm.connections;
+        if (!this.conn.amqp.protocol) {
+            this.conn.amqp.protocol = 'http';
+        }
+        this.chain = this.config.settings.chain;
+        this.esIngestClients = [];
+        this.prepareESClient();
+        this.prepareIngestClients();
+    }
 
-	get nodeosJsonRPC() {
-		// @ts-ignore
-		return new JsonRpc(this.conn.chains[this.chain].http, {fetch});
-	}
+    get nodeosJsonRPC() {
+        // @ts-ignore
+        return new JsonRpc(this.conn.chains[this.chain].http, {fetch});
+    }
 
-	async purgeQueues() {
-		hLog(`Purging all ${this.chain} queues!`);
-		const apiUrl = `http://${this.conn.amqp.api}`;
-		const vHost = encodeURIComponent(this.conn.amqp.vhost);
-		const getAllQueuesFromVHost = apiUrl + `/api/queues/${vHost}`;
-		const opts = {
-			username: this.conn.amqp.user,
-			password: this.conn.amqp.pass
-		};
-		let result;
-		try {
-			const data = await got(getAllQueuesFromVHost, opts);
-			if (data) {
-				result = JSON.parse(data.body);
-			}
-		} catch (e) {
-			console.log(e.message);
-			console.error('failed to connect to rabbitmq http api');
-			process.exit(1);
-		}
-		if (result) {
-			for (const queue of result) {
-				if (queue.name.startsWith(this.chain + ":")) {
-					const msg_count = parseInt(queue.messages);
-					if (msg_count > 0) {
-						try {
-							await got.delete(apiUrl + `/api/queues/${vHost}/${queue.name}/contents`, opts);
-							hLog(`${queue.messages} messages deleted on queue ${queue.name}`);
-						} catch (e) {
-							console.log(e.message);
-							console.error('failed to connect to rabbitmq http api');
-							process.exit(1);
-						}
-					}
-				}
-			}
-		}
-	}
+    async purgeQueues() {
+        hLog(`Purging all ${this.chain} queues!`);
+        const apiUrl = `http://${this.conn.amqp.api}`;
+        const vHost = encodeURIComponent(this.conn.amqp.vhost);
+        const getAllQueuesFromVHost = apiUrl + `/api/queues/${vHost}`;
+        const opts = {
+            username: this.conn.amqp.user,
+            password: this.conn.amqp.pass
+        };
+        let result;
+        try {
+            const data = await got(getAllQueuesFromVHost, opts);
+            if (data) {
+                result = JSON.parse(data.body);
+            }
+        } catch (e) {
+            console.log(e.message);
+            console.error('failed to connect to rabbitmq http api');
+            process.exit(1);
+        }
+        if (result) {
+            for (const queue of result) {
+                if (queue.name.startsWith(this.chain + ":")) {
+                    const msg_count = parseInt(queue.messages);
+                    if (msg_count > 0) {
+                        try {
+                            await got.delete(apiUrl + `/api/queues/${vHost}/${queue.name}/contents`, opts);
+                            hLog(`${queue.messages} messages deleted on queue ${queue.name}`);
+                        } catch (e) {
+                            console.log(e.message);
+                            console.error('failed to connect to rabbitmq http api');
+                            process.exit(1);
+                        }
+                    }
+                }
+            }
+        }
+    }
 
-	prepareESClient() {
-		let es_url;
-		const _es = this.conn.elasticsearch;
-		if (!_es.protocol) {
-			_es.protocol = 'http';
-		}
-		if (_es.user !== '') {
-			es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${_es.host}`;
-		} else {
-			es_url = `${_es.protocol}://${_es.host}`
-		}
-		this.esIngestClient = new Client({
-			node: es_url,
-			ssl: {
-				rejectUnauthorized: false
-			}
-		});
-	}
+    prepareESClient() {
+        const _es = this.conn.elasticsearch;
+        if (!_es.protocol) {
+            _es.protocol = 'http';
+        }
+        this.esIngestClient = new Client({
+            node: `${_es.protocol}://${_es.host}`,
+            auth: {
+                username: _es.user,
+                password: _es.pass
+            },
+            ssl: _es.protocol === 'https' ? {
+                rejectUnauthorized: false
+            } : undefined
+        });
+    }
 
-	get elasticsearchClient() {
-		return this.esIngestClient;
-	}
+    get elasticsearchClient() {
+        return this.esIngestClient;
+    }
 
-	prepareIngestClients() {
-		const _es = this.conn.elasticsearch;
-		if (!_es.protocol) {
-			_es.protocol = 'http';
-		}
-		if (_es.ingest_nodes) {
-			if (_es.ingest_nodes.length > 0) {
-				for (const node of _es.ingest_nodes) {
-					let es_url;
-					if (_es.user !== '') {
-						es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${node}`;
-					} else {
-						es_url = `${_es.protocol}://${node}`
-					}
-					this.esIngestClients.push(new Client({
-						node: es_url,
-						pingTimeout: 100,
-						ssl: {
-							rejectUnauthorized: false
-						}
-					}));
-				}
-			}
-		}
-	}
+    prepareIngestClients() {
+        const _es = this.conn.elasticsearch;
+        if (!_es.protocol) {
+            _es.protocol = 'http';
+        }
+        if (_es.ingest_nodes) {
+            if (_es.ingest_nodes.length > 0) {
+                for (const node of _es.ingest_nodes) {
+                    this.esIngestClient = new Client({
+                        node: `${_es.protocol}://${_es.host}`,
+                        auth: {
+                            username: _es.user,
+                            password: _es.pass
+                        },
+                        pingTimeout: 100,
+                        ssl: _es.protocol === 'https' ? {
+                            rejectUnauthorized: false
+                        } : undefined
+                    });
+                }
+            }
+        }
+    }
 
-	get ingestClients() {
-		if (this.esIngestClients.length > 0) {
-			return this.esIngestClients;
-		} else {
-			return [this.esIngestClient];
-		}
-	}
+    get ingestClients() {
+        if (this.esIngestClients.length > 0) {
+            return this.esIngestClients;
+        } else {
+            return [this.esIngestClient];
+        }
+    }
 
-	async createAMQPChannels(onReconnect, onClose) {
-		return await amqpConnect(onReconnect, this.conn.amqp, onClose);
-	}
+    async createAMQPChannels(onReconnect, onClose) {
+        return await amqpConnect(onReconnect, this.conn.amqp, onClose);
+    }
 
-	async checkQueueSize(queue) {
-		return await checkQueueSize(queue, this.conn.amqp);
-	}
+    async checkQueueSize(queue) {
+        return await checkQueueSize(queue, this.conn.amqp);
+    }
 
-	get shipClient(): StateHistorySocket {
-		return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_mb);
-	}
+    get shipClient(): StateHistorySocket {
+        return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_mb);
+    }
 
-	get ampqUrl() {
-		return getAmpqUrl(this.conn.amqp);
-	}
+    get ampqUrl() {
+        return getAmpqUrl(this.conn.amqp);
+    }
 
-	calculateServerHash() {
-		exec('git rev-parse HEAD', (err, stdout) => {
-			if (err) {
-				// hLog(`\n ${err.message}\n >>> Failed to check last commit hash. Version hash will be "custom"`);
-				hLog(`Failed to check last commit hash. Version hash will be "custom"`);
-				this.last_commit_hash = 'custom';
-			} else {
-				hLog('Last commit hash on this branch is:', stdout);
-				this.last_commit_hash = stdout.trim();
-			}
-			this.getHyperionVersion();
-		});
-	}
+    calculateServerHash() {
+        exec('git rev-parse HEAD', (err, stdout) => {
+            if (err) {
+                // hLog(`\n ${err.message}\n >>> Failed to check last commit hash. Version hash will be "custom"`);
+                hLog(`Failed to check last commit hash. Version hash will be "custom"`);
+                this.last_commit_hash = 'custom';
+            } else {
+                hLog('Last commit hash on this branch is:', stdout);
+                this.last_commit_hash = stdout.trim();
+            }
+            this.getHyperionVersion();
+        });
+    }
 
-	getServerHash() {
-		return this.last_commit_hash;
-	}
+    getServerHash() {
+        return this.last_commit_hash;
+    }
 
-	getHyperionVersion() {
-		this.current_version = require('../package.json').version;
-		if (this.last_commit_hash === 'custom') {
-			this.current_version = this.current_version + '-dirty';
-		}
-	}
+    getHyperionVersion() {
+        this.current_version = require('../package.json').version;
+        if (this.last_commit_hash === 'custom') {
+            this.current_version = this.current_version + '-dirty';
+        }
+    }
 }

connections/state-history.ts

@@ -6,6 +6,8 @@ export class StateHistorySocket {
 	private ws;
 	private readonly shipUrl;
 	private readonly max_payload_mb;
+	retryOnDisconnect = true;
+	connected = false;
 
 	constructor(ship_url, max_payload_mb) {
 		this.shipUrl = ship_url;
@@ -17,13 +19,13 @@ export class StateHistorySocket {
 	}
 
 	connect(onMessage, onDisconnect, onError, onConnected) {
-
 		debugLog(`Connecting to ${this.shipUrl}...`);
 		this.ws = new WebSocket(this.shipUrl, {
 			perMessageDeflate: false,
 			maxPayload: this.max_payload_mb * 1024 * 1024,
 		});
 		this.ws.on('open', () => {
+			this.connected = true;
 			hLog('Websocket connected!');
 			if (onConnected) {
 				onConnected();
@@ -31,15 +33,21 @@ export class StateHistorySocket {
 		});
 		this.ws.on('message', (data) => onMessage(data));
 		this.ws.on('close', () => {
+			this.connected = false;
 			hLog('Websocket disconnected!');
-			onDisconnect();
+			if(this.retryOnDisconnect) {
+				onDisconnect();
+			}
 		});
 		this.ws.on('error', (err) => {
 			hLog(`${this.shipUrl} :: ${err.message}`);
 		});
 	}
 
-	close() {
+	close(graceful: boolean) {
+		if(graceful) {
+			this.retryOnDisconnect = false;
+		}
 		this.ws.close();
 	}
 

definitions/ecosystem_settings.js

@@ -1,52 +1,51 @@
-// max js heap in MB for each indexer subprocess
-const idx_js_heap = 4096;
-
-// max js heap in MB for each api process
-const api_js_heap = 1024;
-
 function interpreterArgs(heap) {
-  const arr = ['--max-old-space-size=' + heap, '--trace-deprecation', '--trace-warnings'];
-  if (process.env.INSPECT) {
-    arr.push('--inspect');
-  }
-  return arr;
+    const arr = ['--trace-deprecation', '--trace-warnings'];
+    if (heap) {
+        arr.push('--max-old-space-size=' + heap);
+    } else {
+        arr.push('--max-old-space-size=2048');
+    }
+    if (process.env.INSPECT) {
+        arr.push('--inspect');
+    }
+    return arr;
 }
 
-function addIndexer(chainName) {
-  return {
-    script: './launcher.js',
-    name: chainName + '-indexer',
-    namespace: chainName,
-    interpreter: 'node',
-    interpreter_args: interpreterArgs(idx_js_heap),
-    autorestart: false,
-    kill_timeout: 3600,
-    watch: false,
-    time: true,
-    env: {
-      CONFIG_JSON: 'chains/' + chainName + '.config.json',
-      TRACE_LOGS: 'false',
-    },
-  };
+function addIndexer(chainName, heap) {
+    return {
+        script: './launcher.js',
+        name: chainName + '-indexer',
+        namespace: chainName,
+        interpreter: 'node',
+        interpreter_args: interpreterArgs(heap),
+        autorestart: false,
+        kill_timeout: 3600,
+        watch: false,
+        time: true,
+        env: {
+            CONFIG_JSON: 'chains/' + chainName + '.config.json',
+            TRACE_LOGS: 'false',
+        },
+    };
 }
 
-function addApiServer(chainName, threads) {
-  return {
-    script: './api/server.js',
-    name: chainName + '-api',
-    namespace: chainName,
-    node_args: interpreterArgs(api_js_heap),
-    exec_mode: 'cluster',
-    merge_logs: true,
-    instances: threads,
-    autorestart: true,
-    exp_backoff_restart_delay: 100,
-    watch: false,
-    time: true,
-    env: {
-      CONFIG_JSON: 'chains/' + chainName + '.config.json',
-    },
-  };
+function addApiServer(chainName, threads, heap) {
+    return {
+        script: './api/server.js',
+        name: chainName + '-api',
+        namespace: chainName,
+        node_args: interpreterArgs(heap),
+        exec_mode: 'cluster',
+        merge_logs: true,
+        instances: threads,
+        autorestart: true,
+        exp_backoff_restart_delay: 100,
+        watch: false,
+        time: true,
+        env: {
+            CONFIG_JSON: 'chains/' + chainName + '.config.json',
+        },
+    };
 }
 
 module.exports = {addIndexer, addApiServer};

definitions/index-templates.ts

@@ -1,9 +1,7 @@
 import {ConfigurationModule} from "../modules/config";
 
 const shards = 2;
-const replicas = 0;
 const refresh = "1s";
-let defaultLifecyclePolicy = "200G";
 
 export * from './index-lifecycle-policies';
 
@@ -15,12 +13,10 @@ const compression = "best_compression";
 const cm = new ConfigurationModule();
 const chain = cm.config.settings.chain;
 
-if (cm.config.settings.hot_warm_policy) {
-	defaultLifecyclePolicy = "hyperion-rollover";
-}
-
-if (cm.config.settings.custom_policy) {
-	defaultLifecyclePolicy = cm.config.settings.custom_policy;
+// update number of replicas if set to larger than 0
+let replicas = 0;
+if (cm.config.settings.es_replicas) {
+	replicas = cm.config.settings.es_replicas;
 }
 
 const defaultIndexSettings = {
@@ -54,6 +50,16 @@ if (cm.config.settings.hot_warm_policy) {
 	actionSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
 }
 
+const transferProps = {
+	"properties": {
+		"from": {"type": "keyword"},
+		"to": {"type": "keyword"},
+		"amount": {"type": "float"},
+		"symbol": {"type": "keyword"},
+		"memo": {"type": "text"}
+	}
+};
+
 export const action = {
 	order: 0,
 	index_patterns: [
@@ -82,7 +88,6 @@ export const action = {
 			"abi_sequence": {"type": "integer"},
 			"trx_id": {"type": "keyword"},
 			"producer": {"type": "keyword"},
-			"notified": {"type": "keyword"},
 			"signatures": {"enabled": false},
 			"inline_count": {"type": "short"},
 			"max_inline": {"type": "short"},
@@ -120,15 +125,7 @@ export const action = {
 			},
 
 			// *::transfer
-			"@transfer": {
-				"properties": {
-					"from": {"type": "keyword"},
-					"to": {"type": "keyword"},
-					"amount": {"type": "float"},
-					"symbol": {"type": "keyword"},
-					"memo": {"type": "text"}
-				}
-			},
+			"@transfer": transferProps,
 
 			// eosio::unstaketorex
 			"@unstaketorex": {

ecosystem.config.js

@@ -0,0 +1,23 @@
+const {addApiServer, addIndexer} = require('./definitions/ecosystem_settings');
+const {readdirSync, readFileSync} = require("fs");
+const path = require('path');
+
+const apps = [];
+const chainsRoot = path.join(path.resolve(), 'chains');
+readdirSync(chainsRoot)
+    .filter(f => f.endsWith('.config.json'))
+    .forEach(value => {
+        const configFile = readFileSync(path.join(chainsRoot, value))
+        const config = JSON.parse(configFile.toString());
+        const chainName = config.settings.chain;
+        if (config.api.enabled) {
+            const apiHeap = config.api.node_max_old_space_size;
+            apps.push(addApiServer(chainName, config.api.pm2_scaling, apiHeap));
+        }
+        if (config.indexer.enabled) {
+            const indexerHeap = config.indexer.node_max_old_space_size;
+            apps.push(addIndexer(chainName, indexerHeap));
+        }
+    });
+
+module.exports = {apps};

example-ecosystem.config.js

@@ -1,14 +0,0 @@
-const {addApiServer, addIndexer} = require("./definitions/ecosystem_settings");
-
-module.exports = {
-    apps: [
-        // launch indexer
-        addIndexer('eos'),
-
-        // launch a single threaded api
-        addApiServer('eos', 1)
-
-        // example: launching a 4-node cluster of apis using pm2 cluster
-        // addApiServer('eos', 4)
-    ]
-};

helpers/common_functions.ts

@@ -66,6 +66,18 @@ export async function getLastIndexedBlockWithTotalBlocks(es_client: Client, chai
 	return [lastBlock, totalBlocks];
 }
 
+export async function getFirstIndexedBlock(es_client: Client, chain: string): Promise<number> {
+	const results: ApiResponse = await es_client.search({
+		index: chain + '-block-*',
+		size: 1,
+		body: {
+			query: {bool: {filter: {match_all: {}}}},
+			sort: [{block_num: {order: "asc"}}]
+		}
+	});
+	return getLastResult(results);
+}
+
 
 export async function getLastIndexedABI(es_client: Client, chain: string) {
 	const results: ApiResponse = await es_client.search({

hpm

@@ -0,0 +1,3 @@
+#!/bin/bash
+node ./scripts/hpm.js "$@"
+# HPM_DEBUG=1 node ./scripts/hpm.js "$@"

hyp-config

@@ -0,0 +1,2 @@
+#!/bin/bash
+node ./scripts/hyp-config.js "$@"

install.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash -
-sudo apt-get install -y nodejs
-sudo npm install pm2 -g
-npm install

install_env.sh

@@ -1,303 +0,0 @@
-#!/usr/bin/env bash
-set -eo pipefail
-
-#terminal colors
-COLOR_NC=$(tput sgr0)
-export COLOR_NC
-COLOR_RED=$(tput setaf 1)
-export COLOR_RED
-COLOR_GREEN=$(tput setaf 2)
-export COLOR_GREEN
-COLOR_YELLOW=$(tput setaf 3)
-export COLOR_YELLOW
-COLOR_BLUE=$(tput setaf 4)
-export COLOR_BLUE
-
-#variables
-INITIAL_PARAMS="$*"
-GLOBAL="y"
-RABBIT_USER="hyperion"
-RABBIT_PASSWORD="123456"
-NODE=false
-ELASTIC=false
-REDIS=false
-RABBIT=false
-RAM=0
-
-help_function() {
-    echo -e "\nAutomated shell script that installs all dependencies and then configure Hyperion."
-    echo -e "\nUsage: ./install.env <option>\n"
-    echo -e "Options:"
-    echo -e "  --help ==> Show this menu"
-    echo -e "  --version ==> Show Hyperion current version\n"
-    exit 0
-}
-
-version (){
-  echo -e "\nv3.0.0\n"
-  exit 0
-
-}
-
-arg_checker() {
-    if [ "${INITIAL_PARAMS}" == "--help" ]; then
-        help_function
-
-    elif [ "${INITIAL_PARAMS}" == "--version" ]; then
-        version
-    fi
-}
-
-check_ram() {
-  RAM=$(free --giga | awk '/Mem/ {print $2}')
-}
-
-# first check if is installed, after check version
-check_dependencies() {
-  echo -e "\n\n${COLOR_BLUE}Checking dependencies...${COLOR_NC}\n\n"
-  CMD=$(if (dpkg --compare-versions $(dpkg -s nodejs 2>/dev/null | awk '/Version/ {print $2}') ge "5" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    NODE=true
-    echo -e "\n\n${COLOR_BLUE}Nodejs compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}Nodejs installed version is < 13. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n\n${COLOR_BLUE}Nodejs not installed ${COLOR_NC}"
-  fi
-  CMD=$(if (dpkg --compare-versions $(dpkg -s elasticsearch 2>/dev/null | awk '/Version/ {print $2}') ge "7.6" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    ELASTIC=true
-    echo -e "\n${COLOR_BLUE}Elasticsearch compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}Elasticsearch installed version is < 7.6. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n${COLOR_BLUE}Elasticsearch not installed ${COLOR_NC}"
-  fi
-  CMD=$(if (dpkg --compare-versions $(dpkg -s redis-server 2>/dev/null | awk '/Version/ {print $2}') ge "5" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    REDIS=true
-    echo -e "\n${COLOR_BLUE}Redis compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}Redis installed version is < 5. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n${COLOR_BLUE}Redis not installed ${COLOR_NC}"
-  fi
-  CMD=$(if (dpkg --compare-versions $(dpkg -s rabbitmq-server 2>/dev/null | awk '/Version/ {print $2}') ge "3.8" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    RABBIT=true
-    echo -e "\n${COLOR_BLUE}RabbitMQ compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}RabbitMQ installed version is < 3.8. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n${COLOR_BLUE}RabbitMQ not installed ${COLOR_NC}"
-  fi
-  CMD=$(if (dpkg --compare-versions $(dpkg -s kibana 2>/dev/null | awk '/Version/ {print $2}') ge "7.6" 2>/dev/null); then echo true; else echo false; fi)
-  if ("$CMD" = true); then
-    RABBIT=true
-    echo -e "\n${COLOR_BLUE}Kibana compatible version already installed ${COLOR_NC}"
-  elif ("$CMD" = false); then
-    echo -e "\n\n${COLOR_RED}Kibana installed version is < 7.6. Please, update and try again. ${COLOR_NC}\n\n"
-    exit 1
-  else
-    echo -e "\n${COLOR_BLUE}Kibana not installed ${COLOR_NC}\n\n"
-  fi
-}
-
-# Make a directory for global installations
-configure_npm() {
-  mkdir ~/.npm-global
-  npm config set prefix "$HOME/.npm-global"
-  # export path for the current session
-  export PATH=$HOME/.npm-global/bin:$PATH
-  # make PATH persistent
-  echo "export PATH=~/.npm-global/bin:$PATH" >>"$HOME/.profile"
-  # shellcheck source=/dev/null
-  source ~/.profile
-}
-
-install_keys_sources() {
-  echo -e "\n\n${COLOR_BLUE}Configuring keys and sources...${COLOR_NC}\n\n"
-
-  PPA="https://artifacts.elastic.co/packages/7.x/apt stable main"
-  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
-    wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
-    echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
-  fi
-
-  PPA="http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang"
-  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
-    curl -fsSL https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc | sudo apt-key add -
-    echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" | sudo tee /etc/apt/sources.list.d/bintray.rabbitmq.list
-    curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | sudo bash
-  fi
-
-  PPA="https://deb.nodesource.com/node_13.x bionic main"
-  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
-    curl -sL "https://deb.nodesource.com/setup_13.x" | sudo -E bash -
-  fi
-
-  sudo apt update -y
-}
-
-install_dep() {
- echo -e "\n\n${COLOR_BLUE}Installing dependencies...${COLOR_NC}\n\n"
- # sudo apt install -y build-essential curl
- sudo apt install -y curl
-}
-
-install_node() {
-  echo -e "\n\n${COLOR_BLUE}Installing nodejs...${COLOR_NC}\n\n"
-  sudo apt install -y nodejs
-  read -p "Do you want to create a directory for npm global installations [Y/n] : " GLOBAL
-  GLOBAL=${GLOBAL:-y}
-  if [ "$GLOBAL" = "y" ]; then
-    configure_npm
-  fi
-}
-
-install_build_hyperion() {
-  echo -e "\n\n${COLOR_BLUE}Installing packages and building hyperion...${COLOR_NC}\n\n"
-  npm install
-}
-
-install_pm2() {
-  echo -e "\n\n${COLOR_BLUE}Installing pm2...${COLOR_NC}\n\n"
-  if [ "$GLOBAL" = "y" ]; then
-    npm install pm2@latest -g
-  else
-    sudo npm install pm2@latest -g
-  fi
-}
-
-install_redis() {
-  echo -e "\n\n${COLOR_BLUE}Installing redis...${COLOR_NC}\n\n"
-  sudo apt install -y redis-server
-  sudo sed -ie 's/supervised no/supervised systemd/' /etc/redis/redis.conf
-  sudo systemctl restart redis.service
-}
-
-install_erlang() {
-  echo -e "\n\n${COLOR_BLUE}Installing erlang...${COLOR_NC}\n\n"
-  sudo apt -y install erlang
-}
-
-#ask user for rabbit credentials
-rabbit_credentials() {
-  read -p "Enter rabbitmq user [hyperion]: " RABBIT_USER
-  RABBIT_USER=${RABBIT_USER:-hyperion}
-
-  read -p "Enter rabbitmq password [123456]: " RABBIT_PASSWORD
-  RABBIT_PASSWORD=${RABBIT_PASSWORD:-123456}
-
-}
-
-install_rabittmq() {
-  echo -e "\n\n${COLOR_BLUE}Installing rabbit-mq...${COLOR_NC}\n\n"
-  sudo apt install -y rabbitmq-server
-  #enable web gui
-  sudo rabbitmq-plugins enable rabbitmq_management
-  sudo rabbitmqctl add_vhost /hyperion
-  sudo rabbitmqctl add_user ${RABBIT_USER} ${RABBIT_PASSWORD}
-  sudo rabbitmqctl set_user_tags ${RABBIT_USER} administrator
-  sudo rabbitmqctl set_permissions -p /hyperion ${RABBIT_USER} ".*" ".*" ".*"
-}
-
-install_elastic() {
-  echo -e "\n\n${COLOR_BLUE}Installing elastic...${COLOR_NC}\n\n"
-  sudo apt install -y elasticsearch
-
-  echo -e "\n\n${COLOR_BLUE}Configuring elastic...${COLOR_NC}\n\n"
-
-  # edit configs
-  sudo sed -ie 's/#cluster.name: my-application/cluster.name: myCluster/; s/#bootstrap.memory_lock: true/bootstrap.memory_lock: true/' /etc/elasticsearch/elasticsearch.yml
-  # set jvm options based on system RAM
-  check_ram
-  if [ "$RAM" -lt 32 ]; then
-    (( RAM=RAM/2 ))
-    sudo sed -ie 's/-Xms1g/-Xms'"$RAM"'g/; s/-Xmx1g/-Xmx'"$RAM"'g/' /etc/elasticsearch/jvm.options
-  else
-    sudo sed -ie 's/-Xms1g/-Xms16g/; s/-Xmx1g/-Xmx16g/' /etc/elasticsearch/jvm.options
-  fi
-
-  sudo bash -c 'echo "xpack.security.enabled: true" >> /etc/elasticsearch/elasticsearch.yml'
-
-  sudo mkdir -p /etc/systemd/system/elasticsearch.service.d/
-  echo -e "[Service]\nLimitMEMLOCK=infinity" | sudo tee /etc/systemd/system/elasticsearch.service.d/override.conf
-  sudo systemctl daemon-reload
-  sudo service elasticsearch start
-  sudo systemctl enable elasticsearch
-
-  echo -e "\n\n${COLOR_BLUE}Generating Elasticsearch cluster passwords...${COLOR_NC}\n\n"
-
-  echo "y" | sudo /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto >elastic_pass.txt
-
-}
-
-install_kibana() {
-  echo -e "\n\n${COLOR_BLUE}Installing and configuring Kibana...${COLOR_NC}\n\n"
-  sudo apt install -y kibana
-
-  KIBANA_PASSWORD=$(awk <elastic_pass.txt '/PASSWORD kibana =/ {print $4}')
-
-  sudo sed -ie 's/#elasticsearch.password: "pass"/elasticsearch.password: '"$KIBANA_PASSWORD"'/; s/#elasticsearch.username: "kibana"/elasticsearch.username: "kibana"/' /etc/kibana/kibana.yml
-
-  sudo systemctl enable kibana
-  sudo systemctl start kibana
-}
-
-#******************
-# End of functions
-#******************
-arg_checker
-
-echo -e "\n\n${COLOR_BLUE}*** STARTING HYPERION HISTORY API CONFIGURATION ***${COLOR_NC}\n\n"
-
-check_dependencies
-
-if [ "$RABBIT" = false ]; then
-  rabbit_credentials
-fi
-
-install_dep
-
-install_keys_sources
-if [ "$NODE" = false ]; then
-  install_node
-fi
-
-install_pm2
-
-install_build_hyperion
-
-if [ "$RABBIT" = false ]; then
-  install_rabittmq
-fi
-
-if [ "$REDIS" = false ]; then
-  install_redis
-fi
-
-if [ "$ELASTIC" = false ]; then
-  install_elastic
-  install_kibana
-fi
-
-printf "
- _   ___   ______  _____ ____  ___ ___  _   _
-| | | \ \ / /  _ \| ____|  _ \|_ _/ _ \| \ | |
-| |_| |\ V /| |_) |  _| | |_) || | | | |  \| |
-|  _  | | | |  __/| |___|  _ < | | |_| | |\  |
-|_| |_| |_| |_|   |_____|_| \_\___\___/|_| \_|
-Made with ♥ by EOS Rio
-"
-echo -e "\n\n${COLOR_GREEN}Hyperion History API successfully installed!!${COLOR_NC}"
-echo -e "\n${COLOR_YELLOW}Please, check your installation: ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- Elastic: http://localhost:9200 ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- Kibana: http://localhost:5601 ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- RabbitMQ: http://localhost:15672 ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- Refer to the elastic_pas.txt file for the elasticsearch passwords. ${COLOR_NC}"
-echo -e "${COLOR_YELLOW}- Make pm2 auto-boot at server restart: \$ pm2 startup ${COLOR_NC}\n"

interfaces/hyperionConfig.ts

@@ -1,174 +1,190 @@
 import {AlertManagerOptions} from "../modules/alertsManager";
 
 export interface ScalingConfigs {
-	polling_interval: number;
-	resume_trigger: number;
-	max_queue_limit: number;
-	block_queue_limit: number;
-	routing_mode: string;
-	batch_size: number;
-	queue_limit: number;
-	readers: number;
-	ds_queues: number;
-	ds_threads: number;
-	ds_pool_size: number;
-	indexing_queues: number;
-	ad_idx_queues: number;
-	dyn_idx_queues: number;
-	max_autoscale: number;
-	auto_scale_trigger: number;
+    polling_interval: number;
+    resume_trigger: number;
+    max_queue_limit: number;
+    block_queue_limit: number;
+    routing_mode: string;
+    batch_size: number;
+    queue_limit: number;
+    readers: number;
+    ds_queues: number;
+    ds_threads: number;
+    ds_pool_size: number;
+    indexing_queues: number;
+    ad_idx_queues: number;
+    dyn_idx_queues: number;
+    max_autoscale: number;
+    auto_scale_trigger: number;
 }
 
 export interface MainSettings {
-	process_prefix?: string;
-	index_partition_size: number;
-	ignore_snapshot?: boolean;
-	ship_request_rev: string;
-	custom_policy: string;
-	bypass_index_map: boolean;
-	hot_warm_policy: boolean;
-	auto_mode_switch: boolean;
-	ds_profiling: boolean;
-	max_ws_payload_mb: number;
-	ipc_debug_rate?: number;
-	bp_monitoring?: boolean;
-	preview: boolean;
-	chain: string;
-	eosio_alias: string;
-	parser: string;
-	auto_stop: number;
-	index_version: string;
-	debug: boolean;
-	rate_monitoring: boolean;
-	bp_logs: boolean;
-	dsp_parser: boolean;
-	allow_custom_abi: boolean;
+    process_prefix?: string;
+    ignore_snapshot?: boolean;
+    ship_request_rev: string;
+    custom_policy: string;
+    bypass_index_map: boolean;
+    hot_warm_policy: boolean;
+    auto_mode_switch: boolean;
+    ds_profiling: boolean;
+    max_ws_payload_mb: number;
+    ipc_debug_rate?: number;
+    bp_monitoring?: boolean;
+    preview: boolean;
+    chain: string;
+    eosio_alias: string;
+    parser: string;
+    auto_stop: number;
+    index_version: string;
+    debug: boolean;
+    rate_monitoring: boolean;
+    bp_logs: boolean;
+    dsp_parser: boolean;
+    allow_custom_abi: boolean;
+    index_partition_size: number;
+    es_replicas: number;
 }
 
 export interface IndexerConfigs {
-	fill_state: boolean;
-	start_on: number;
-	stop_on: number;
-	rewrite: boolean;
-	purge_queues: boolean;
-	live_reader: boolean;
-	live_only_mode: boolean;
-	abi_scan_mode: boolean;
-	fetch_block: boolean;
-	fetch_traces: boolean;
-	fetch_deltas: boolean;
-	disable_reading: boolean;
-	disable_indexing: boolean;
-	process_deltas: boolean;
-	repair_mode: boolean;
-	max_inline: number;
-	disable_delta_rm?: boolean;
+    enabled?: boolean;
+    node_max_old_space_size?: number;
+    fill_state: boolean;
+    start_on: number;
+    stop_on: number;
+    rewrite: boolean;
+    purge_queues: boolean;
+    live_reader: boolean;
+    live_only_mode: boolean;
+    abi_scan_mode: boolean;
+    fetch_block: boolean;
+    fetch_traces: boolean;
+    fetch_deltas: boolean;
+    disable_reading: boolean;
+    disable_indexing: boolean;
+    process_deltas: boolean;
+    repair_mode: boolean;
+    max_inline: number;
+    disable_delta_rm?: boolean;
 }
 
 interface ApiLimits {
-	get_links?: number;
-	get_actions?: number;
-	get_blocks?: number;
-	get_created_accounts?: number;
-	get_deltas?: number;
-	get_key_accounts?: number;
-	get_proposals?: number;
-	get_tokens?: number;
-	get_transfers?: number;
-	get_voters?: number;
-	get_trx_actions?: number;
+    get_links?: number;
+    get_actions?: number;
+    get_blocks?: number;
+    get_created_accounts?: number;
+    get_deltas?: number;
+    get_key_accounts?: number;
+    get_proposals?: number;
+    get_tokens?: number;
+    get_transfers?: number;
+    get_voters?: number;
+    get_trx_actions?: number;
+}
+
+interface CachedRouteConfig {
+    path: string;
+    ttl: number
 }
 
 interface ApiConfigs {
+    enabled?: boolean;
+    pm2_scaling?: number;
+    node_max_old_space_size?: number;
     disable_rate_limit?: boolean;
-	disable_tx_cache?: boolean;
-	tx_cache_expiration_sec?: number | string;
-	rate_limit_rpm?: number;
-	rate_limit_allow?: string[];
-	custom_core_token?: string;
-	chain_api_error_log?: boolean;
-	chain_api?: string;
-	push_api?: string;
-	enable_explorer?: boolean;
-	access_log: boolean;
-	chain_name: string;
-	server_port: number;
-	server_addr: string;
-	server_name: string;
-	provider_name: string;
-	provider_url: string;
-	provider_logo: string;
-	chain_logo_url: string;
-	enable_caching: boolean,
-	cache_life: number;
-	limits: ApiLimits
+    disable_tx_cache?: boolean;
+    tx_cache_expiration_sec?: number | string;
+    rate_limit_rpm?: number;
+    rate_limit_allow?: string[];
+    custom_core_token?: string;
+    chain_api_error_log?: boolean;
+    chain_api?: string;
+    push_api?: string;
+    access_log: boolean;
+    chain_name: string;
+    server_port: number;
+    stream_port: number;
+    server_addr: string;
+    server_name: string;
+    provider_name: string;
+    provider_url: string;
+    provider_logo: string;
+    chain_logo_url: string;
+    enable_caching: boolean,
+    cache_life: number;
+    limits: ApiLimits,
+    v1_chain_cache?: CachedRouteConfig[]
 }
 
 interface HubLocation {
-	city: string,
-	country: string,
-	lat: number,
-	lon: number
+    city: string,
+    country: string,
+    lat: number,
+    lon: number
 }
 
 interface HyperionHubConfigs {
-	location: HubLocation;
-	production: boolean;
-	publisher_key: string;
-	inform_url: string;
+    location: HubLocation;
+    production: boolean;
+    publisher_key: string;
+    inform_url: string;
 }
 
 export interface HyperionConfig {
-	hub: HyperionHubConfigs;
-	settings: MainSettings;
-	scaling: ScalingConfigs;
-	indexer: IndexerConfigs;
+    api: ApiConfigs;
 
-	api: ApiConfigs;
+    settings: MainSettings;
 
-	blacklists: {
-		actions: string[],
-		deltas: string[]
-	};
+    hub: HyperionHubConfigs;
+    scaling: ScalingConfigs;
 
-	whitelists: {
-		max_depth: number;
-		root_only: boolean,
-		actions: string[],
-		deltas: string[]
-	};
+    indexer: IndexerConfigs;
 
-	features: {
-		streaming: {
-			enable: boolean,
-			traces: boolean,
-			deltas: boolean
-		},
-		tables: {
-			proposals: boolean,
-			accounts: boolean,
-			voters: boolean,
-			userres: boolean,
-			delband: boolean
-		},
-		index_deltas: boolean,
-		index_transfer_memo: boolean,
-		index_all_deltas: boolean,
-		deferred_trx: boolean,
-		failed_trx: boolean,
-		resource_usage: boolean,
-		resource_limits: boolean,
-	};
+    blacklists: {
+        actions: string[],
+        deltas: string[]
+    };
 
-	prefetch: {
-		read: number,
-		block: number,
-		index: number
-	};
+    whitelists: {
+        max_depth: number;
+        root_only: boolean,
+        actions: string[],
+        deltas: string[]
+    };
 
-	experimental: any;
+    features: {
+        streaming: {
+            enable: boolean,
+            traces: boolean,
+            deltas: boolean
+        },
+        tables: {
+            proposals: boolean,
+            accounts: boolean,
+            voters: boolean,
+            userres: boolean,
+            delband: boolean
+        },
+        index_deltas: boolean,
+        index_transfer_memo: boolean,
+        index_all_deltas: boolean,
+        deferred_trx: boolean,
+        failed_trx: boolean,
+        resource_usage: boolean,
+        resource_limits: boolean,
+    };
 
-	plugins: any;
+    prefetch: {
+        read: number,
+        block: number,
+        index: number
+    };
 
-	alerts: AlertManagerOptions;
+    experimental: any;
+
+    plugins: {
+        [key: string]: any;
+    };
+
+    alerts: AlertManagerOptions;
 }

interfaces/hyperionConnections.ts

@@ -20,6 +20,7 @@ interface HyperionChainData {
     name: string;
     chain_id: string;
     http: string;
+    ship: string;
     WS_ROUTER_PORT: number;
     WS_ROUTER_HOST: string;
 }

modules/action_data/daobet-eosio-delegatebw.js

@@ -1,45 +0,0 @@
-const hyperionModule = {
-    chain: "74d023a9293d9b68c3c52e2f738ee681c1671cc3dc0f263cf2c533cd5523ff95",
-    contract: 'eosio',
-    action: 'delegatebw',
-    parser_version: ['1.7'],
-    defineQueryPrefix: 'delegatebw',
-    mappings: {
-        action: {
-            "@delegatebw": {
-                "properties": {
-                    "from": {"type": "keyword"},
-                    "receiver": {"type": "keyword"},
-                    "stake_cpu_quantity": {"type": "float"},
-                    "stake_net_quantity": {"type": "float"},
-                    "stake_vote_quantity": {"type": "float"},
-                    "transfer": {"type": "boolean"},
-                    "amount": {"type": "float"}
-                }
-            }
-        }
-    },
-    handler: (action) => {
-        const data = action['act']['data'];
-        let cpu_qtd = null;
-        let net_qtd = null;
-        let vote_qtd = null;
-        if (data['stake_net_quantity'] && data['stake_cpu_quantity'] && data['stake_vote_quantity']) {
-            cpu_qtd = parseFloat(data['stake_cpu_quantity'].split(' ')[0]);
-            net_qtd = parseFloat(data['stake_net_quantity'].split(' ')[0]);
-            vote_qtd = parseFloat(data['stake_vote_quantity'].split(' ')[0]);
-        }
-        action['@delegatebw'] = {
-            amount: cpu_qtd + net_qtd + vote_qtd,
-            stake_cpu_quantity: cpu_qtd,
-            stake_net_quantity: net_qtd,
-            stake_vote_quantity: vote_qtd,
-            from: data['from'],
-            receiver: data['receiver'],
-            transfer: data['transfer']
-        };
-        delete action['act']['data'];
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/daobet-eosio-undelegatebw.js

@@ -1,43 +0,0 @@
-const hyperionModule = {
-    chain: "74d023a9293d9b68c3c52e2f738ee681c1671cc3dc0f263cf2c533cd5523ff95",
-    contract: 'eosio',
-    action: 'undelegatebw',
-    parser_version: ['1.7'],
-    defineQueryPrefix: 'undelegatebw',
-    mappings: {
-        action: {
-            "@undelegatebw": {
-                "properties": {
-                    "from": {"type": "keyword"},
-                    "receiver": {"type": "keyword"},
-                    "unstake_cpu_quantity": {"type": "float"},
-                    "unstake_net_quantity": {"type": "float"},
-                    "unstake_vote_quantity": {"type": "float"},
-                    "amount": {"type": "float"}
-                }
-            }
-        }
-    },
-    handler: (action) => {
-        const data = action['act']['data'];
-        let cpu_qtd = null;
-        let net_qtd = null;
-        let vote_qtd = null;
-        if (data['unstake_net_quantity'] && data['unstake_cpu_quantity'] && data['unstake_vote_quantity']) {
-            cpu_qtd = parseFloat(data['unstake_cpu_quantity'].split(' ')[0]);
-            net_qtd = parseFloat(data['unstake_net_quantity'].split(' ')[0]);
-            vote_qtd = parseFloat(data['unstake_vote_quantity'].split(' ')[0]);
-        }
-        action['@undelegatebw'] = {
-            amount: cpu_qtd + net_qtd + vote_qtd,
-            unstake_cpu_quantity: cpu_qtd,
-            unstake_net_quantity: net_qtd,
-            unstake_vote_quantity: vote_qtd,
-            from: data['from'],
-            receiver: data['receiver']
-        };
-        delete action['act']['data'];
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/eosio-buyram.js

@@ -1,20 +1,20 @@
 const hyperionModule = {
-  chain: '*',
-  contract: 'eosio',
-  action: 'buyram',
-  defineQueryPrefix: 'buyram',
-  parser_version: ['2.1', '1.8', '1.7'],
-  handler: (action) => {
-    const data = action['act']['data'];
-    action['@buyram'] = {
-      payer: data['payer'],
-      receiver: data['receiver'],
-    };
-    if (data['quant']) {
-      action['@buyram']['quant'] = parseFloat(data['quant'].split(' ')[0]);
-    }
-    delete action['act']['data'];
-  },
+    chain: '*',
+    contract: 'eosio',
+    action: 'buyram',
+    defineQueryPrefix: 'buyram',
+    parser_version: ['3.2', '2.1', '1.8', '1.7'],
+    handler: (action) => {
+        const data = action['act']['data'];
+        action['@buyram'] = {
+            payer: data['payer'],
+            receiver: data['receiver'],
+        };
+        if (data['quant']) {
+            action['@buyram']['quant'] = parseFloat(data['quant'].split(' ')[0]);
+        }
+        delete action['act']['data'];
+    },
 };
 
 module.exports = {hyperionModule};

modules/action_data/eosio-buyrambytes.js

@@ -3,7 +3,7 @@ const hyperionModule = {
     contract: 'eosio',
     action: 'buyrambytes',
     defineQueryPrefix: 'buyrambytes',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     handler: (action) => {
         const data = action['act']['data'];
         action['@buyrambytes'] = {

modules/action_data/eosio-buyrex.js

@@ -3,7 +3,7 @@ const hyperionModule = {
     contract: 'eosio',
     action: 'buyrex',
     defineQueryPrefix: 'buyrex',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     handler: (action) => {
         const data = action['act']['data'];
         let qtd = null;

modules/action_data/eosio-delegatebw.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'delegatebw',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'delegatebw',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-newaccount.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'newaccount',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'newaccount',
     handler: (action) => {
         let name = null;

modules/action_data/eosio-undelegatebw.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'undelegatebw',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'undelegatebw',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-unstaketorex.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'unstaketorex',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'unstaketorex',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-updateauth.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'updateauth',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'updateauth',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-voteproducer.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'voteproducer',
-    parser_version: ['2.1','1.8', '1.7'],
+    parser_version: ['3.2', '2.1','1.8', '1.7'],
     defineQueryPrefix: 'voteproducer',
     mappings: {
         action: {

modules/action_data/fio-bprewdupdate.js

@@ -2,7 +2,7 @@ const hyperionModule = {
   chain: '21dcae42c0182200e93f954a074011f9048a7624c6fe81d3c9541a614a88bd1c',
   contract: 'fio.treasury',
   action: 'bprewdupdate',
-  parser_version: ['1.8'],
+  parser_version: ['3.2', '1.8'],
   defineQueryPrefix: 'bprewdupdate',
   mappings: {
     action: {

modules/action_data/fio-fdtnrwdupdat.js

@@ -2,7 +2,7 @@ const hyperionModule = {
   chain: '21dcae42c0182200e93f954a074011f9048a7624c6fe81d3c9541a614a88bd1c',
   contract: 'fio.treasury',
   action: 'fdtnrwdupdat',
-  parser_version: ['1.8'],
+  parser_version: ['3.2', '1.8'],
   defineQueryPrefix: 'fdtnrwdupdat',
   mappings: {
     action: {

modules/action_data/transfer.js

@@ -4,7 +4,7 @@ const hyperionModule = {
   chain: '*',
   contract: '*',
   action: 'transfer',
-  parser_version: ['2.1','1.8', '1.7'],
+  parser_version: ['3.2', '2.1', '1.8', '1.7'],
   defineQueryPrefix: 'transfer',
   handler: (action) => {
     let qtd = null;

modules/loader.ts

@@ -5,189 +5,217 @@ import {HyperionConnections} from "../interfaces/hyperionConnections";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
 import {BaseParser} from "./parsers/base-parser";
 import {hLog} from "../helpers/common_functions";
-import {HyperionPlugin} from "../plugins/hyperion-plugin";
+import {HyperionPlugin, HyperionStreamHandler} from "../plugins/hyperion-plugin";
 
 export class HyperionModuleLoader {
 
-	private handledActions = new Map();
-	private handledDeltas = new Map();
-	chainMappings = new Map();
-	extendedActions: Set<string> = new Set();
-	extraMappings = [];
-	chainID;
-	private conn: HyperionConnections;
-	private config: HyperionConfig;
-	public parser: BaseParser;
-	public plugins: HyperionPlugin[];
+    private handledActions = new Map();
+    private handledDeltas = new Map();
+    private streamHandlers: HyperionStreamHandler[] = [];
 
-	constructor(private cm: ConfigurationModule) {
-		this.plugins = [];
-		this.conn = cm.connections;
-		this.config = cm.config;
-		if (!this.conn.chains[this.config.settings.chain]) {
-			console.log('Chain  ' + this.config.settings.chain + ' not defined on connections.json!');
-			process.exit(0);
-		}
-		this.chainID = this.conn.chains[this.config.settings.chain].chain_id;
-		this.loadActionHandlers();
-	}
+    chainMappings = new Map();
+    extendedActions: Set<string> = new Set();
+    extraMappings = [];
+    chainID;
+    private conn: HyperionConnections;
+    private config: HyperionConfig;
+    public parser: BaseParser;
+    public plugins: HyperionPlugin[];
 
-	async loadParser() {
-		const path = join(__dirname, 'parsers', this.config.settings.parser + "-parser");
-		const mod = (await import(path)).default;
-		this.parser = new mod(this.cm) as BaseParser;
-	}
+    constructor(private cm: ConfigurationModule) {
+        this.plugins = [];
+        this.conn = cm.connections;
+        this.config = cm.config;
+        if (!this.conn.chains[this.config.settings.chain]) {
+            console.log('Chain  ' + this.config.settings.chain + ' not defined on connections.json!');
+            process.exit(0);
+        }
+        this.chainID = this.conn.chains[this.config.settings.chain].chain_id;
+        this.loadActionHandlers();
+    }
 
-	processActionData(action) {
-		const wildcard = this.handledActions.get('*');
-		if (wildcard && wildcard.has(action.act.name)) {
-			wildcard.get(action.act.name)(action);
-		}
-		if (this.handledActions && this.handledActions.has(action.act.account)) {
-			const _c = this.handledActions.get(action.act.account);
-			if (_c.has(action.act.name)) {
-				_c.get(action.act.name)(action);
-			}
-		}
-	}
+    async loadParser() {
+        const path = join(__dirname, 'parsers', this.config.settings.parser + "-parser");
+        const mod = (await import(path)).default;
+        this.parser = new mod(this.cm) as BaseParser;
+    }
 
-	async processDeltaData(delta): Promise<void> {
-		if (this.handledDeltas.has(delta.code)) {
-			const _c = this.handledDeltas.get(delta.code);
-			if (_c.has(delta.table)) {
-				await _c.get(delta.table)(delta);
-			}
-		}
-	}
+    processActionData(action) {
+        const wildcard = this.handledActions.get('*');
+        if (wildcard && wildcard.has(action.act.name)) {
+            wildcard.get(action.act.name)(action);
+        }
+        if (this.handledActions && this.handledActions.has(action.act.account)) {
+            const _c = this.handledActions.get(action.act.account);
+            if (_c.has(action.act.name)) {
+                _c.get(action.act.name)(action);
+            }
+        }
+    }
 
-	includeActionModule(_module) {
-		if (this.handledActions.has(_module.contract)) {
-			const existing = this.handledActions.get(_module.contract);
-			existing.set(_module.action, _module.handler);
-		} else {
-			const _map = new Map();
-			_map.set(_module.action, _module.handler);
-			this.handledActions.set(_module.contract, _map);
-		}
-		if (_module.mappings) {
-			this.extraMappings.push(_module.mappings);
-		}
-		if (_module.defineQueryPrefix) {
-			this.extendedActions.add(_module.defineQueryPrefix);
-		}
-	}
+    async processDeltaData(delta): Promise<void> {
+        if (this.handledDeltas.has(delta.code)) {
+            const _c = this.handledDeltas.get(delta.code);
+            if (_c.has(delta.table)) {
+                await _c.get(delta.table)(delta);
+            }
+        }
+    }
 
-	includeDeltaModule(deltaModule) {
-		if (this.handledDeltas.has(deltaModule.contract)) {
-			const existing = this.handledDeltas.get(deltaModule.contract);
-			existing.set(deltaModule.table, deltaModule.handler);
-		} else {
-			const _map = new Map();
-			_map.set(deltaModule.table, deltaModule.handler);
-			this.handledDeltas.set(deltaModule.contract, _map);
-		}
-		if (deltaModule.mappings) {
-			this.extraMappings.push(deltaModule.mappings);
-		}
-	}
 
-	loadActionHandlers() {
-		const files = readdirSync('modules/action_data/');
-		for (const plugin of files) {
-			const _module = require(join(__dirname, 'action_data', plugin)).hyperionModule;
-			if (_module.parser_version.includes(this.config.settings.parser)) {
-				if (_module.chain === this.chainID || _module.chain === '*') {
-					const key = `${_module.contract}::${_module.action}`;
-					if (this.chainMappings.has(key)) {
-						if (this.chainMappings.get(key) === '*' && _module.chain === this.chainID) {
-							this.includeActionModule(_module);
-							this.chainMappings.set(key, _module.chain);
-						}
-					} else {
-						this.includeActionModule(_module);
-						this.chainMappings.set(key, _module.chain);
-					}
-				}
-			}
-		}
-	}
+    includeActionModule(_module) {
+        if (this.handledActions.has(_module.contract)) {
+            const existing = this.handledActions.get(_module.contract);
+            existing.set(_module.action, _module.handler);
+        } else {
+            const _map = new Map();
+            _map.set(_module.action, _module.handler);
+            this.handledActions.set(_module.contract, _map);
+        }
+        if (_module.mappings) {
+            this.extraMappings.push(_module.mappings);
+        }
+        if (_module.defineQueryPrefix) {
+            this.extendedActions.add(_module.defineQueryPrefix);
+        }
+    }
 
-	async init() {
-		try {
-			await this.loadParser();
-			await this.loadPlugins();
-		} catch (e) {
-			console.log(e);
-			process.exit(1);
-		}
-	}
+    includeDeltaModule(deltaModule) {
+        if (this.handledDeltas.has(deltaModule.contract)) {
+            const existing = this.handledDeltas.get(deltaModule.contract);
+            existing.set(deltaModule.table, deltaModule.handler);
+        } else {
+            const _map = new Map();
+            _map.set(deltaModule.table, deltaModule.handler);
+            this.handledDeltas.set(deltaModule.contract, _map);
+        }
+        if (deltaModule.mappings) {
+            this.extraMappings.push(deltaModule.mappings);
+        }
+    }
 
-	// main loader function for plugin modules
-	private async loadPlugins() {
-		const base = join(__dirname, '..', 'plugins');
-		if (!existsSync(base)) {
-			return;
-		}
-		const repos = join(base, 'repos');
-		if (!existsSync(repos)) {
-			return;
-		}
-		const state = join(base, '.state.json');
-		if (!existsSync(state)) {
-			return;
-		}
+    loadActionHandlers() {
+        const files = readdirSync('modules/action_data/');
+        for (const plugin of files) {
+            const _module = require(join(__dirname, 'action_data', plugin)).hyperionModule;
+            if (_module.parser_version.includes(this.config.settings.parser)) {
+                if (_module.chain === this.chainID || _module.chain === '*') {
+                    const key = `${_module.contract}::${_module.action}`;
+                    if (this.chainMappings.has(key)) {
+                        if (this.chainMappings.get(key) === '*' && _module.chain === this.chainID) {
+                            this.includeActionModule(_module);
+                            this.chainMappings.set(key, _module.chain);
+                        }
+                    } else {
+                        this.includeActionModule(_module);
+                        this.chainMappings.set(key, _module.chain);
+                    }
+                }
+            }
+        }
+    }
 
-		let pluginState;
-		try {
-			pluginState = JSON.parse(readFileSync(state).toString());
-		} catch (e) {
-			hLog('Failed to read plugin state');
-			return;
-		}
+    async init() {
+        try {
+            await this.loadParser();
+            await this.loadPlugins();
+        } catch (e) {
+            console.log(e);
+            process.exit(1);
+        }
+    }
 
-		for (const pluginName of Object.keys(pluginState)) {
-			if (pluginState[pluginName] && this.config.plugins[pluginName]) {
-				if (pluginState[pluginName].enabled && this.config.plugins[pluginName].enabled) {
-					try {
-						const pluginModule = (await import(join(repos, pluginName))).default;
-						const pl = new pluginModule(this.config.plugins[pluginName]);
-						if (pl.actionHandlers) {
-							this.loadPluginActionHandlers(pl.actionHandlers);
-						}
-						if (pl.deltaHandlers) {
-							this.loadPluginDeltaHandlers(pl.deltaHandlers);
-						}
-						this.plugins.push(pl);
-					} catch (e) {
-						hLog(`Plugin "${pluginName}" failed to load: ${e.message}`);
-					}
-				}
-			}
-		}
-	}
+    // main loader function for plugin modules
+    private async loadPlugins() {
+        const base = join(__dirname, '..', 'plugins');
+        if (!existsSync(base)) {
+            return;
+        }
+        const repos = join(base, 'repos');
+        if (!existsSync(repos)) {
+            return;
+        }
+        const state = join(base, '.state.json');
+        if (!existsSync(state)) {
+            return;
+        }
 
-	appendDynamicContracts(allowedDynamicContracts: Set<string>) {
-		for (const plugin of this.plugins) {
-			if (plugin.dynamicContracts) {
-				if (plugin.dynamicContracts.length > 0) {
-					plugin.dynamicContracts.forEach(value => {
-						allowedDynamicContracts.add(value);
-					});
-				}
-			}
-		}
-	}
 
-	private loadPluginActionHandlers(actionHandlers: any) {
-		for (const handler of actionHandlers) {
-			this.includeActionModule(handler);
-		}
-	}
+        let pState;
+        try {
+            const stateFile = JSON.parse(readFileSync(state).toString());
+            pState = stateFile.plugins;
+        } catch (e) {
+            hLog('Failed to read plugin state');
+            return;
+        }
 
-	private loadPluginDeltaHandlers(deltaHandlers: any) {
-		for (const handler of deltaHandlers) {
-			this.includeDeltaModule(handler);
-		}
-	}
+        for (const key in this.config.plugins) {
+            if (this.config.plugins.hasOwnProperty(key)) {
+                if (pState[key] && pState[key].enabled && this.config.plugins[key].enabled) {
+                    try {
+                        const pMod = (await import(join(repos, key))).default;
+                        const pl = new pMod(this.config.plugins[key]);
+                        if (pl.actionHandlers) {
+                            this.loadPluginActionHandlers(pl.actionHandlers);
+                        }
+                        if (pl.deltaHandlers) {
+                            this.loadPluginDeltaHandlers(pl.deltaHandlers);
+                        }
+                        if (pl.streamHandlers) {
+                            this.loadPluginStreamHandlers(pl.streamHandlers);
+                        }
+                        this.plugins.push(pl);
+                    } catch (e) {
+                        hLog(`Plugin "${key}" failed to load: ${e.message}`);
+                    }
+                }
+            }
+        }
+    }
 
+    appendDynamicContracts(allowedDynamicContracts: Set<string>) {
+        for (const plugin of this.plugins) {
+            if (plugin.dynamicContracts) {
+                if (plugin.dynamicContracts.length > 0) {
+                    plugin.dynamicContracts.forEach(value => {
+                        allowedDynamicContracts.add(value);
+                    });
+                }
+            }
+        }
+    }
+
+
+    private loadPluginActionHandlers(actionHandlers: any) {
+        for (const handler of actionHandlers) {
+            this.includeActionModule(handler);
+        }
+    }
+
+    private loadPluginDeltaHandlers(deltaHandlers: any) {
+        for (const handler of deltaHandlers) {
+            this.includeDeltaModule(handler);
+        }
+    }
+
+    private loadPluginStreamHandlers(streamHandlers: any) {
+        for (const handler of streamHandlers) {
+            this.includeStreamModule(handler);
+        }
+    }
+
+    includeStreamModule(_module) {
+        this.streamHandlers.push(_module);
+    }
+
+    processStreamEvent(msg) {
+        if (this.streamHandlers.length > 0) {
+            this.streamHandlers.forEach(sth => {
+                sth.handler(msg).catch(reason => {
+                    hLog(`Stream processing failed:`, reason);
+                });
+            });
+        }
+    }
 }

modules/parsers/3.2-parser.ts

@@ -1,31 +1,27 @@
+// noinspection JSUnusedGlobalSymbols
+
 import {BaseParser} from "./base-parser";
 import MainDSWorker from "../../workers/deserializer";
 import {Message} from "amqplib";
 import DSPoolWorker from "../../workers/ds-pool";
 import {TrxMetadata} from "../../interfaces/trx-metadata";
 import {ActionTrace} from "../../interfaces/action-trace";
-import {hLog} from "../../helpers/common_functions";
-import {unzip} from "zlib";
-import {omit} from "lodash";
-
-async function unzipAsync(data) {
-    return new Promise((resolve, reject) => {
-        const buf = Buffer.from(data, 'hex');
-        unzip(buf, (err, result) => {
-            if (err) {
-                reject(err);
-            } else {
-                resolve(result);
-            }
-        })
-    });
-}
+import {deserialize, hLog} from "../../helpers/common_functions";
 
 export default class HyperionParser extends BaseParser {
 
-    flatten = true;
+    flatten = false;
 
-    public async parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded): Promise<boolean> {
+    public async parseAction(
+        worker: DSPoolWorker,
+        ts,
+        action: ActionTrace,
+        trx_data: TrxMetadata,
+        _actDataArray,
+        _processedTraces: ActionTrace[],
+        full_trace,
+        usageIncluded
+    ): Promise<boolean> {
 
         // check filters
         if (this.checkBlacklist(action.act)) return false;
@@ -37,35 +33,32 @@ export default class HyperionParser extends BaseParser {
 
         action["@timestamp"] = ts;
         action.block_num = trx_data.block_num;
+        action.block_id = trx_data.block_id;
         action.producer = trx_data.producer;
         action.trx_id = trx_data.trx_id;
 
+        action.account_ram_deltas = action.account_ram_deltas.filter(value => value.delta !== '0')
+
         if (action.account_ram_deltas.length === 0) {
             delete action.account_ram_deltas;
         }
 
         if (action.except === null) {
-
             if (!action.receipt) {
                 console.log(full_trace.status);
                 console.log(action);
             }
-
             action.receipt = action.receipt[1];
             action.global_sequence = parseInt(action.receipt.global_sequence, 10);
-
             delete action.except;
-            delete action.error_code;
-            delete action.elapsed;
-            delete action.context_free;
-            delete action.level;
-
+            if (action.error_code === null) {
+                delete action.error_code;
+            }
+            // add usage data to the first action on the transaction
             if (!usageIncluded.status) {
                 this.extendFirstAction(worker, action, trx_data, full_trace, usageIncluded);
             }
-
             _processedTraces.push(action);
-
         } else {
             hLog(action);
         }
@@ -75,7 +68,8 @@ export default class HyperionParser extends BaseParser {
 
     public async parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void> {
         for (const message of messages) {
-            const ds_msg = worker.deserializeNative('result', message.content);
+            let allowProcessing = true;
+            const ds_msg = deserialize('result', message.content, this.txEnc, this.txDec, worker.types);
             if (!ds_msg) {
                 if (worker.ch_ready) {
                     worker.ch.nack(message);
@@ -83,34 +77,58 @@ export default class HyperionParser extends BaseParser {
                 }
             }
             const res = ds_msg[1];
-            let block, traces = [], deltas = [];
+            let block = null;
+            let traces = [];
+            let deltas = [];
+
             if (res.block && res.block.length) {
+
                 block = worker.deserializeNative('signed_block', res.block);
-                if (!block) {
-                    hLog('null block', res);
+
+                if (block === null) {
+                    hLog('incompatible block');
                     process.exit(1);
                 }
-            }
-            if (res['traces'] && res['traces'].length) {
-                try {
-                    traces = worker.deserializeNative(
-                        'transaction_trace[]',
-                        await unzipAsync(res['traces'])
-                    );
-                } catch (e) {
-                    hLog(e);
+
+                // verify for whitelisted contracts (root actions only)
+                if (worker.conf.whitelists.root_only) {
+                    try {
+                        if (worker.conf.whitelists && (worker.conf.whitelists.actions.length > 0 || worker.conf.whitelists.deltas.length > 0)) {
+                            allowProcessing = false;
+                            for (const transaction of block.transactions) {
+                                if (transaction.status === 0 && transaction.trx[1] && transaction.trx[1].packed_trx) {
+                                    const unpacked_trx = worker.api.deserializeTransaction(Buffer.from(transaction.trx[1].packed_trx, 'hex'));
+                                    for (const act of unpacked_trx.actions) {
+                                        if (this.checkWhitelist(act)) {
+                                            allowProcessing = true;
+                                            break;
+                                        }
+                                    }
+                                    if (allowProcessing) break;
+                                }
+                            }
+                        }
+                    } catch (e) {
+                        console.log(e);
+                        allowProcessing = true;
+                    }
                 }
             }
-            if (res['deltas'] && res['deltas'].length) {
-                try {
-                    deltas = worker.deserializeNative(
-                        'table_delta[]',
-                        await unzipAsync(res['deltas'])
-                    );
-                } catch (e) {
-                    hLog(e);
+
+            if (allowProcessing && res.traces && res.traces.length) {
+                traces = worker.deserializeNative('transaction_trace[]', res.traces);
+                if (!traces) {
+                    hLog(`[WARNING] transaction_trace[] deserialization failed on block ${res['this_block']['block_num']}`);
                 }
             }
+
+            if (allowProcessing && res.deltas && res.deltas.length) {
+                deltas = deserialize('table_delta[]', res.deltas, this.txEnc, this.txDec, worker.types);
+                if (!deltas) {
+                    hLog(`[WARNING] table_delta[] deserialization failed on block ${res['this_block']['block_num']}`);
+                }
+            }
+
             let result;
             try {
                 result = await worker.processBlock(res, block, traces, deltas);
@@ -144,28 +162,9 @@ export default class HyperionParser extends BaseParser {
         }
     }
 
-    async flattenInlineActions(action_traces: any[], level: number, trace_counter: any, parent_index: number): Promise<any[]> {
-        const arr = [];
-        const nextLevel = [];
-        for (const action_trace of action_traces) {
-            const trace = action_trace[1];
-            trace['creator_action_ordinal'] = parent_index;
-            trace_counter.trace_index++;
-            trace['level'] = level;
-            trace['action_ordinal'] = trace_counter.trace_index;
-            arr.push(["action_trace_v0", omit(trace, "inline_traces")]);
-            if (trace.inline_traces && trace.inline_traces.length > 0) {
-                nextLevel.push({
-                    traces: trace.inline_traces,
-                    parent: trace_counter.trace_index
-                });
-            }
-        }
-        for (const data of nextLevel) {
-            const appendedArray = await this.flattenInlineActions(data.traces, level + 1, trace_counter, data.parent);
-            arr.push(...appendedArray);
-        }
-        return Promise.resolve(arr);
+    async flattenInlineActions(action_traces: any[]): Promise<any[]> {
+        hLog(`Calling undefined flatten operation!`);
+        return Promise.resolve(undefined);
     }
 
 }

modules/parsers/base-parser.ts

@@ -10,211 +10,223 @@ import {HyperionActionAct} from "../../interfaces/hyperion-action";
 
 export abstract class BaseParser {
 
-	txDec = new TextDecoder();
-	txEnc = new TextEncoder();
-	configModule: ConfigurationModule;
-	filters: Filters;
-	private readonly chain: string;
-	flatten: boolean = false;
-	private actionReinterpretMap: Map<string, (act: HyperionActionAct) => any>;
+    txDec = new TextDecoder();
+    txEnc = new TextEncoder();
+    configModule: ConfigurationModule;
+    filters: Filters;
+    private readonly chain: string;
+    flatten: boolean = false;
+    private actionReinterpretMap: Map<string, (act: HyperionActionAct) => any>;
 
-	protected constructor(cm: ConfigurationModule) {
-		this.configModule = cm;
-		this.filters = this.configModule.filters;
-		this.chain = this.configModule.config.settings.chain;
-		this.actionReinterpretMap = new Map();
-		this.addCustomHandlers();
-	}
+    protected constructor(cm: ConfigurationModule) {
+        this.configModule = cm;
+        this.filters = this.configModule.filters;
+        this.chain = this.configModule.config.settings.chain;
+        this.actionReinterpretMap = new Map();
+        this.addCustomHandlers();
+    }
 
-	private anyFromCode(act) {
-		return this.chain + '::' + act['account'] + '::*';
-	}
+    private anyFromCode(act) {
+        return this.chain + '::' + act['account'] + '::*';
+    }
 
-	private anyFromName(act) {
-		return this.chain + '::*::' + act['name'];
-	}
+    private anyFromName(act) {
+        return this.chain + '::*::' + act['name'];
+    }
 
-	private codeActionPair(act) {
-		return this.chain + '::' + act['account'] + '::' + act['name'];
-	}
+    private codeActionPair(act) {
+        return this.chain + '::' + act['account'] + '::' + act['name'];
+    }
 
-	protected checkBlacklist(act) {
+    protected checkBlacklist(act) {
 
-		// test action blacklist for chain::code::*
-		if (this.filters.action_blacklist.has(this.anyFromCode(act))) {
-			return true;
-		}
+        // test action blacklist for chain::code::*
+        if (this.filters.action_blacklist.has(this.anyFromCode(act))) {
+            return true;
+        }
 
-		// test action blacklist for chain::*::name
-		if (this.filters.action_blacklist.has(this.anyFromName(act))) {
-			return true;
-		}
+        // test action blacklist for chain::*::name
+        if (this.filters.action_blacklist.has(this.anyFromName(act))) {
+            return true;
+        }
 
-		// test action blacklist for chain::code::name
-		return this.filters.action_blacklist.has(this.codeActionPair(act));
-	}
+        // test action blacklist for chain::code::name
+        return this.filters.action_blacklist.has(this.codeActionPair(act));
+    }
 
-	protected checkWhitelist(act) {
+    protected checkWhitelist(act) {
 
-		// test action whitelist for chain::code::*
-		if (this.filters.action_whitelist.has(this.anyFromCode(act))) {
-			return true;
-		}
+        // test action whitelist for chain::code::*
+        if (this.filters.action_whitelist.has(this.anyFromCode(act))) {
+            return true;
+        }
 
-		// test action whitelist for chain::*::name
-		if (this.filters.action_whitelist.has(this.anyFromName(act))) {
-			return true;
-		}
+        // test action whitelist for chain::*::name
+        if (this.filters.action_whitelist.has(this.anyFromName(act))) {
+            return true;
+        }
 
-		// test action whitelist for chain::code::name
-		return this.filters.action_whitelist.has(this.codeActionPair(act));
-	}
+        // test action whitelist for chain::code::name
+        return this.filters.action_whitelist.has(this.codeActionPair(act));
+    }
 
-	protected extendFirstAction(worker: DSPoolWorker, action: ActionTrace, trx_data: TrxMetadata, full_trace: any, usageIncluded) {
-		action.cpu_usage_us = trx_data.cpu_usage_us;
-		action.net_usage_words = trx_data.net_usage_words;
-		action.signatures = trx_data.signatures;
-		if (full_trace.action_traces.length > 1) {
-			action.inline_count = trx_data.inline_count - 1;
-			action.inline_filtered = trx_data.filtered;
-			if (action.inline_filtered) {
-				action.max_inline = worker.conf.indexer.max_inline;
-			}
-		} else {
-			action.inline_count = 0;
-		}
-		usageIncluded.status = true;
-	}
+    protected extendFirstAction(
+        worker: DSPoolWorker,
+        action: ActionTrace,
+        trx_data: TrxMetadata,
+        full_trace: any,
+        usageIncluded: { status: boolean }
+    ) {
+        action.cpu_usage_us = trx_data.cpu_usage_us;
+        action.net_usage_words = trx_data.net_usage_words;
+        action.signatures = trx_data.signatures;
+        if (full_trace.action_traces.length > 1) {
+            action.inline_count = trx_data.inline_count - 1;
+            action.inline_filtered = trx_data.filtered;
+            if (action.inline_filtered) {
+                action.max_inline = worker.conf.indexer.max_inline;
+            }
+        } else {
+            action.inline_count = 0;
+        }
+        usageIncluded.status = true;
+    }
 
-	protected createSerialBuffer(data: string) {
-		return new SerialBuffer({
-			textDecoder: this.txDec,
-			textEncoder: this.txEnc,
-			array: Buffer.from(data, 'hex')
-		});
-	}
+    protected createSerialBuffer(data: string) {
+        return new SerialBuffer({
+            textDecoder: this.txDec,
+            textEncoder: this.txEnc,
+            array: Buffer.from(data, 'hex')
+        });
+    }
 
-	protected addCustomHandlers() {
-		// simple assets
-		this.actionReinterpretMap.set('*::saecreate', (act) => {
-			const _sb = this.createSerialBuffer(act.data);
-			const result = {owner: "", assetid: null};
-			result.owner = _sb.getName();
-			result.assetid = _sb.getUint64AsNumber()
-			return result;
-		});
+    protected addCustomHandlers() {
+        // simple assets
+        this.actionReinterpretMap.set('*::saecreate', (act) => {
+            const _sb = this.createSerialBuffer(act.data);
+            const result = {owner: "", assetid: null};
+            result.owner = _sb.getName();
+            result.assetid = _sb.getUint64AsNumber()
+            return result;
+        });
 
-		this.actionReinterpretMap.set('*::saetransfer', (act) => {
-			const _sb = this.createSerialBuffer(act.data);
-			const result = {from: "", to: "", assetids: [], memo: ""};
-			result.from = _sb.getName();
-			result.to = _sb.getName();
-			const len = _sb.getVaruint32();
-			for (let i = 0; i < len; i++) {
-				result.assetids.push(_sb.getUint64AsNumber());
-			}
-			result.memo = _sb.getString();
-			return result;
-		});
+        this.actionReinterpretMap.set('*::saetransfer', (act) => {
+            const _sb = this.createSerialBuffer(act.data);
+            const result = {from: "", to: "", assetids: [], memo: ""};
+            result.from = _sb.getName();
+            result.to = _sb.getName();
+            const len = _sb.getVaruint32();
+            for (let i = 0; i < len; i++) {
+                result.assetids.push(_sb.getUint64AsNumber());
+            }
+            result.memo = _sb.getString();
+            return result;
+        });
 
-		this.actionReinterpretMap.set('*::saeclaim', (act) => {
-			const _sb = this.createSerialBuffer(act.data);
-			const result = {who: "", assetids: {}};
-			result.who = _sb.getName();
-			const len = _sb.getVaruint32();
-			for (let i = 0; i < len; i++) {
-				result.assetids[_sb.getUint64AsNumber()] = _sb.getName();
-			}
-			return result;
-		});
+        this.actionReinterpretMap.set('*::saeclaim', (act) => {
+            const _sb = this.createSerialBuffer(act.data);
+            const result = {who: "", assetids: {}};
+            result.who = _sb.getName();
+            const len = _sb.getVaruint32();
+            for (let i = 0; i < len; i++) {
+                result.assetids[_sb.getUint64AsNumber()] = _sb.getName();
+            }
+            return result;
+        });
 
-		this.actionReinterpretMap.set('*::saeburn', (act) => {
-			const _sb = this.createSerialBuffer(act.data);
-			const result = {who: "", assetids: [], memo: ""};
-			result.who = _sb.getName();
-			const len = _sb.getVaruint32();
-			for (let i = 0; i < len; i++) {
-				result.assetids.push(_sb.getUint64AsNumber());
-			}
-			result.memo = _sb.getString();
-			return result;
-		});
-	}
+        this.actionReinterpretMap.set('*::saeburn', (act) => {
+            const _sb = this.createSerialBuffer(act.data);
+            const result = {who: "", assetids: [], memo: ""};
+            result.who = _sb.getName();
+            const len = _sb.getVaruint32();
+            for (let i = 0; i < len; i++) {
+                result.assetids.push(_sb.getUint64AsNumber());
+            }
+            result.memo = _sb.getString();
+            return result;
+        });
+    }
 
-	async reinterpretActionData(act: HyperionActionAct) {
-		if (this.actionReinterpretMap.has(`${act.account}::${act.name}`)) {
-			// code and action
-			return await this.actionReinterpretMap.get(`${act.account}::${act.name}`)(act);
+    async reinterpretActionData(act: HyperionActionAct) {
+        if (this.actionReinterpretMap.has(`${act.account}::${act.name}`)) {
+            // code and action
+            return await this.actionReinterpretMap.get(`${act.account}::${act.name}`)(act);
 
-		} else if (this.actionReinterpretMap.has(`*::${act.name}`)) {
-			// wildcard action
-			return await this.actionReinterpretMap.get(`*::${act.name}`)(act);
+        } else if (this.actionReinterpretMap.has(`*::${act.name}`)) {
+            // wildcard action
+            return await this.actionReinterpretMap.get(`*::${act.name}`)(act);
 
-		} else {
-			return;
-		}
-	}
+        } else {
+            return;
+        }
+    }
 
-	async deserializeActionData(worker: DSPoolWorker, action: ActionTrace, trx_data) {
-		let act = action.act;
-		const original_act = Object.assign({}, act);
-		let ds_act, error_message;
-		try {
-			ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num);
-		} catch (e) {
-			console.log(e);
-			error_message = e.message;
-		}
+    async deserializeActionData(worker: DSPoolWorker, action: ActionTrace, trx_data) {
+        let act = action.act;
+        const original_act = Object.assign({}, act);
+        let ds_act, error_message;
+        try {
+            ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num);
+        } catch (e) {
+            console.log(e);
+            error_message = e.message;
+        }
 
-		// retry failed deserialization for custom ABI maps
-		if (!ds_act) {
-			try {
-				ds_act = await this.reinterpretActionData(act);
-			} catch (e) {
-				hLog(`Failed to reinterpret action: ${act.account}::${act.name}`);
-				hLog(act.data);
-			}
-		}
+        // retry failed deserialization for custom ABI maps
+        if (!ds_act) {
+            try {
+                ds_act = await this.reinterpretActionData(act);
+            } catch (e) {
+                hLog(`Failed to reinterpret action: ${act.account}::${act.name}`);
+                hLog(act.data);
+            }
+        }
 
-		// retry with last abi before the given block_num
-		if (!ds_act) {
-			debugLog('DS Failed ->>', original_act);
-			ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num - 1);
-			debugLog('Retry with previous ABI ->>', ds_act);
-		}
+        // retry with last abi before the given block_num
+        if (!ds_act) {
+            debugLog('DS Failed ->>', original_act);
+            ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num - 1);
+            debugLog('Retry with previous ABI ->>', ds_act);
+        }
 
-		if (ds_act) {
-			// save serialized data
-			action.act.data = ds_act;
-			try {
-				worker.common.attachActionExtras(worker, action);
-			} catch (e) {
-				hLog('Failed to call attachActionExtras:', e.message);
-				hLog(action?.act?.data);
-			}
-		} else {
-			action['act'] = original_act;
-			if (typeof action.act.data !== 'string') {
-				action.act.data = Buffer.from(action.act.data).toString('hex');
-			}
-			action['ds_error'] = true;
-			process.send({
-				event: 'ds_error',
-				data: {
-					type: 'action_ds_error',
-					block: trx_data.block_num,
-					account: act.account,
-					action: act.name,
-					gs: parseInt(action.receipt[1].global_sequence, 10),
-					message: error_message
-				}
-			});
-		}
-	}
+        if (ds_act) {
 
-	abstract parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded: { status: boolean }): Promise<boolean>
+            if (ds_act.account && ds_act.name && ds_act.authorization) {
+                console.log(ds_act);
+                action.act.data = ds_act.data;
+            }
 
-	abstract parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void>
+            // save serialized data
+            action.act.data = ds_act;
+            try {
+                worker.common.attachActionExtras(worker, action);
+            } catch (e) {
+                hLog('Failed to call attachActionExtras:', e.message);
+                hLog(action?.act?.account, action?.act?.name, action?.act?.data);
+            }
+        } else {
+            action['act'] = original_act;
+            if (typeof action.act.data !== 'string') {
+                action.act.data = Buffer.from(action.act.data).toString('hex');
+            }
+            action['ds_error'] = true;
+            process.send({
+                event: 'ds_error',
+                data: {
+                    type: 'action_ds_error',
+                    block: trx_data.block_num,
+                    account: act.account,
+                    action: act.name,
+                    gs: parseInt(action.receipt[1].global_sequence, 10),
+                    message: error_message
+                }
+            });
+        }
+    }
 
-	abstract flattenInlineActions(action_traces: any[], level?: number, trace_counter?: any, parent_index?: number): Promise<any[]>
+    abstract parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded: { status: boolean }): Promise<boolean>
+
+    abstract parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void>
+
+    abstract flattenInlineActions(action_traces: any[], level?: number, trace_counter?: any, parent_index?: number): Promise<any[]>
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperion-history",
-  "version": "3.3.4-rc7",
+  "version": "3.3.9",
   "description": "Scalable Full History API Solution for EOSIO based blockchains",
   "main": "launcher.js",
   "scripts": {
@@ -9,12 +9,12 @@
     "start:indexer": "pm2 start --only Indexer --update-env",
     "tsc": "tsc",
     "build": "tsc",
+    "build:watch": "tsc --watch",
     "postinstall": "npm run build && npm run fix-permissions",
-    "fix-permissions": "node scripts/fix-permissions.js",
-    "plugin-manager": "node plugins/hyperion-plugin-manager.js"
+    "fix-permissions": "node scripts/fix-permissions.js"
   },
   "engines": {
-    "node": "^16"
+    "node": "^18"
   },
   "author": {
     "name": "EOS Rio",
@@ -26,52 +26,54 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@elastic/elasticsearch": "7.14.1",
+    "@elastic/elasticsearch": "^7.17.0",
     "@eosrio/node-abieos": "^2.1.1",
+    "@fastify/autoload": "4.0.1",
+    "@fastify/cors": "7.0.0",
+    "@fastify/formbody": "^6.0.1",
+    "@fastify/rate-limit": "^6.0.1",
+    "@fastify/redis": "^5.0.0",
+    "@fastify/swagger": "6.1.0",
     "@pm2/io": "^5.0.0",
-    "amqplib": "^0.8.0",
-    "async": "^3.2.1",
-    "base-x": "^3.0.8",
-    "cross-fetch": "^3.1.4",
+    "amqplib": "^0.10.3",
+    "async": "^3.2.4",
+    "base-x": "^4.0.0",
+    "commander": "^8.3.0",
+    "cross-fetch": "^3.1.5",
     "eosjs": "^22.1.0",
-    "fast-json-stringify": "^2.7.9",
-    "fastify": "^3.20.2",
-    "fastify-autoload": "^3.8.1",
-    "fastify-compress": "^3.6.0",
-    "fastify-cors": "^6.0.2",
+    "fast-json-stringify": "2.7.13",
+    "fastify": "3.29.4",
     "fastify-elasticsearch": "^2.0.0",
-    "fastify-formbody": "^5.1.0",
-    "fastify-plugin": "^3.0.0",
-    "fastify-rate-limit": "^5.6.2",
-    "fastify-redis": "^4.3.1",
-    "fastify-static": "^4.2.3",
-    "fastify-swagger": "^4.9.1",
+    "fastify-plugin": "^3.0.1",
     "flatstr": "^1.0.12",
-    "got": "11.8.2",
-    "ioredis": "^4.27.9",
+    "global-agent": "^3.0.0",
+    "got": "11.8.5",
+    "ioredis": "^4.28.5",
     "lodash": "^4.17.21",
-    "moment": "^2.29.1",
-    "nodemailer": "^6.6.3",
-    "portfinder": "^1.0.28",
-    "socket.io": "^4.2.0",
-    "socket.io-client": "^4.2.0",
+    "moment": "^2.29.4",
+    "nodemailer": "^6.9.0",
+    "portfinder": "^1.0.32",
+    "pino-pretty": "^9.1.1",
+    "socket.io": "4.5.4",
+    "socket.io-client": "4.5.4",
     "socket.io-redis": "^6.1.1",
-    "telegraf": "^4.4.1",
-    "typescript": "^4.3.5",
-    "ws": "^8.2.1",
-    "yargs": "^17.1.1"
+    "telegraf": "^4.11.2",
+    "typescript": "^4.5.2",
+    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.19.0",
+    "ws": "^8.12.0"
   },
   "devDependencies": {
     "@types/amqplib": "^0.8.2",
-    "@types/async": "^3.2.7",
-    "@types/ioredis": "^4.27.2",
-    "@types/lodash": "^4.14.172",
-    "@types/node": "^16.7.10",
-    "@types/nodemailer": "^6.4.4",
-    "@types/ws": "^7.4.7"
+    "@types/async": "^3.2.16",
+    "@types/global-agent": "^2.1.1",
+    "@types/ioredis": "^4.28.10",
+    "@types/lodash": "^4.14.191",
+    "@types/node": "^18.11.18",
+    "@types/nodemailer": "^6.4.7",
+    "@types/ws": "^8.5.4"
   },
   "optionalDependencies": {
-    "bufferutil": "^4.0.3",
-    "utf-8-validate": "^5.0.5"
+    "bufferutil": "^4.0.7",
+    "utf-8-validate": "^5.0.10"
   }
 }

plugins/hyperion-plugin-manager.js

@@ -72,7 +72,6 @@ async function verifyInstallation(data) {
       process.exit(0);
     }
   }
-
 }
 
 function startInstallation(data) {

plugins/hyperion-plugin.js

@@ -3,8 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.HyperionPlugin = void 0;
 class HyperionPlugin {
     constructor(config) {
+        this.internalPluginName = '';
         this.actionHandlers = [];
         this.deltaHandlers = [];
+        this.streamHandlers = [];
         this.dynamicContracts = [];
         this.hasApiRoutes = false;
         this.chainName = '';
@@ -12,6 +14,13 @@ class HyperionPlugin {
             this.baseConfig = config;
         }
     }
+    // abstract processActionData(input: any): Promise<any>;
+    initOnce() {
+        // called only once
+    }
+    initHandlerMap() {
+        return {};
+    }
 }
 exports.HyperionPlugin = HyperionPlugin;
 //# sourceMappingURL=hyperion-plugin.js.map

plugins/hyperion-plugin.js.map

@@ -1 +1 @@
-{"version":3,"file":"hyperion-plugin.js","sourceRoot":"","sources":["hyperion-plugin.ts"],"names":[],"mappings":";;;AAmBA,MAAsB,cAAc;IAQnC,YAAsB,MAAY;QAPlC,mBAAc,GAA4B,EAAE,CAAC;QAC7C,kBAAa,GAA2B,EAAE,CAAC;QAC3C,qBAAgB,GAAa,EAAE,CAAC;QAChC,iBAAY,GAAY,KAAK,CAAC;QAE9B,cAAS,GAAW,EAAE,CAAC;QAGtB,IAAI,MAAM,EAAE;YACX,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;SACzB;IACF,CAAC;CAKD;AAjBD,wCAiBC"}
+{"version":3,"file":"hyperion-plugin.js","sourceRoot":"","sources":["hyperion-plugin.ts"],"names":[],"mappings":";;;AAgCA,MAAsB,cAAc;IAYhC,YAAsB,MAAY;QAXlC,uBAAkB,GAAW,EAAE,CAAC;QAGhC,mBAAc,GAA4B,EAAE,CAAC;QAC7C,kBAAa,GAA2B,EAAE,CAAC;QAC3C,mBAAc,GAA4B,EAAE,CAAC;QAC7C,qBAAgB,GAAa,EAAE,CAAC;QAChC,iBAAY,GAAY,KAAK,CAAC;QAE9B,cAAS,GAAW,EAAE,CAAC;QAGnB,IAAI,MAAM,EAAE;YACR,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;SAC5B;IACL,CAAC;IAID,wDAAwD;IACxD,QAAQ;QACJ,mBAAmB;IACvB,CAAC;IAED,cAAc;QACV,OAAO,EAAE,CAAC;IACd,CAAC;CACJ;AA5BD,wCA4BC"}

plugins/hyperion-plugin.ts

@@ -4,34 +4,58 @@ import {HyperionDelta} from "../interfaces/hyperion-delta";
 
 
 interface HyperionActionHandler {
-	action: string;
-	contract: string;
-	mappings?: any;
-	handler: (action: HyperionAction) => Promise<void>;
+    action: string;
+    contract: string;
+    mappings?: any;
+    handler: (action: HyperionAction) => Promise<void>;
 }
 
 interface HyperionDeltaHandler {
-	table: string;
-	contract: string;
-	mappings?: any;
-	handler: (delta: HyperionDelta) => Promise<void>;
+    table: string;
+    contract: string;
+    mappings?: any;
+    handler: (delta: HyperionDelta) => Promise<void>;
+}
+
+interface HyperionStreamEvent {
+
+}
+
+export interface HyperionStreamHandler {
+    event: string;
+    code?: string;
+    account?: string;
+    name?: string;
+    table?: string;
+    handler: (streamEvent: any) => Promise<void>;
 }
 
 export abstract class HyperionPlugin {
-	actionHandlers: HyperionActionHandler[] = [];
-	deltaHandlers: HyperionDeltaHandler[] = [];
-	dynamicContracts: string[] = [];
-	hasApiRoutes: boolean = false;
-	baseConfig: any;
-	chainName: string = '';
+    internalPluginName: string = '';
+    indexerPlugin: boolean;
+    apiPlugin: boolean;
+    actionHandlers: HyperionActionHandler[] = [];
+    deltaHandlers: HyperionDeltaHandler[] = [];
+    streamHandlers: HyperionStreamHandler[] = [];
+    dynamicContracts: string[] = [];
+    hasApiRoutes: boolean = false;
+    baseConfig: any;
+    chainName: string = '';
 
-	protected constructor(config?: any) {
-		if (config) {
-			this.baseConfig = config;
-		}
-	}
+    protected constructor(config?: any) {
+        if (config) {
+            this.baseConfig = config;
+        }
+    }
 
-	abstract addRoutes(server: FastifyInstance): void;
+    abstract addRoutes(server: FastifyInstance): void;
 
-	// abstract processActionData(input: any): Promise<any>;
+    // abstract processActionData(input: any): Promise<any>;
+    initOnce() {
+        // called only once
+    }
+
+    initHandlerMap(): any {
+        return {};
+    }
 }

scripts/fix-permissions.js

@@ -5,5 +5,5 @@ if (os.type() === 'Windows_NT') {
   console.log('Skipping permission update on windows');
 } else {
   console.log('Updating permissions...');
-  execSync('chmod u+x run.sh stop.sh');
+  execSync('chmod u+x run.sh stop.sh hpm hyp-config');
 }

scripts/fix_missing_blocks/README.MD

@@ -0,0 +1,36 @@
+# Hyperion History API - Script
+
+## Fix Missing Blocks
+
+- Python script that connects to elasticsearch and identifies any missing blocks within indexes, and performs a re-indexing of those ranges to include any missing blocks.
+- Once the script completes, it will automatically re-start the indexer on live mode with re-write disabled.
+
+### Install
+
+```
+pip3 install sh
+pip3 install elasticsearch==7.15.2
+```
+
+### Update the following 3 config lines in the python code.
+
+```
+hyperionfolder="/home/charles/hyperion-history-api"
+indexer_log_file = "/home/charles/.pm2/logs/wax-indexer-out.log"
+# Please set the version of Hyperion to either 3.1 or 3.3 to take into account changes to no blocks processed text in log file. 
+hyperion_version = '3.3'  #'3.1' or '3.3'
+```
+
+### RUN
+
+```
+python3 fix-missing-blocks.py http://elasticusername:elasticpasswordD@127.0.0.1
+```
+
+
+### TODO:
+- Add progress 
+
+
+
+

scripts/hpm.js

@@ -0,0 +1,405 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const path_1 = __importDefault(require("path"));
+const promises_1 = require("fs/promises");
+const fs_1 = __importStar(require("fs"));
+const child_process_1 = require("child_process");
+const crypto_1 = __importDefault(require("crypto"));
+const package_json = JSON.parse((0, fs_1.readFileSync)('./package.json').toString());
+const debug = process.env.HPM_DEBUG;
+const program = new commander_1.Command();
+const pluginDirAbsolutePath = path_1.default.join(path_1.default.resolve(), 'plugins', 'repos');
+const pluginStatePath = path_1.default.join(path_1.default.resolve(), 'plugins', '.state.json');
+let stateJson;
+const ignoredDirsForHash = ['.git', 'node_modules'];
+program.version(package_json.version);
+function checkUrl(value) {
+    if (value.startsWith('https://') || value.startsWith('git@')) {
+        return value;
+    }
+    else {
+        throw new commander_1.InvalidArgumentError('Invalid url.');
+    }
+}
+function throwAndQuit(message) {
+    console.error('⚠ >>', message);
+    process.exit(1);
+}
+function getCurrentBranch(dir) {
+    const gitCmd = `(cd ${dir} && git rev-parse --abbrev-ref HEAD)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return (0, child_process_1.execSync)(gitCmd).toString().trim();
+}
+function checkoutBranch(dir, branchName) {
+    const gitCmd = `(cd ${dir} && git checkout ${branchName})`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return (0, child_process_1.execSync)(gitCmd).toString();
+}
+function gitPull(dir) {
+    const gitCmd = `(cd ${dir} && git pull)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return (0, child_process_1.execSync)(gitCmd).toString();
+}
+function gitReset(dir) {
+    const gitCmd = `(cd ${dir} && git reset --hard)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return (0, child_process_1.execSync)(gitCmd).toString();
+}
+async function buildPlugin(name, flags) {
+    const pluginDir = path_1.default.join(pluginDirAbsolutePath, name);
+    if (!(0, fs_1.existsSync)(pluginDir)) {
+        throwAndQuit(`plugin ${name} not found!`);
+    }
+    const currentBranch = getCurrentBranch(pluginDir);
+    console.log(`-------- ⚒ ${name} @ ${currentBranch} -----------`);
+    if (!flags.skipInstall) {
+        // run npm install
+        await new Promise(resolve => {
+            const npmInstall = (0, child_process_1.spawn)('npm', ['install'], {
+                cwd: pluginDir,
+                stdio: 'inherit',
+            });
+            npmInstall.on('close', (code) => {
+                if (code !== 0) {
+                    console.log(`process exited with code ${code}`);
+                    process.exit(code);
+                }
+                resolve(true);
+            });
+        });
+    }
+    // typescript build
+    await new Promise(resolve => {
+        const npmInstall = (0, child_process_1.spawn)('npm', ['run', 'build'], {
+            cwd: pluginDir,
+            stdio: 'inherit',
+        });
+        npmInstall.on('close', (code) => {
+            if (code !== 0) {
+                console.log(`process exited with code ${code}`);
+                process.exit(code);
+            }
+            resolve(true);
+        });
+    });
+    console.log('-------------------------------');
+    // save build date
+    if (stateJson) {
+        if (!stateJson.plugins[name]) {
+            stateJson.plugins[name] = {};
+        }
+        stateJson.plugins[name].last_build_date = new Date().toISOString();
+        const hash = await hashItem(pluginDir);
+        console.log(`SHA1 Hash: ${hash}`);
+        stateJson.plugins[name].hash = hash;
+        stateJson.plugins[name].branch = currentBranch;
+        saveState();
+    }
+}
+async function verifyInstalledPlugin(pluginName, options) {
+    const dir = path_1.default.join(pluginDirAbsolutePath, pluginName);
+    const dirExists = (0, fs_1.existsSync)(dir);
+    if (dirExists) {
+        if (options.branch && options.branch !== '') {
+            const currentBranch = getCurrentBranch(dir);
+            if (currentBranch === options.branch) {
+                console.log('✅ Plugin already installed, branch check OK');
+                process.exit(0);
+            }
+            else {
+                // checkout new branch
+                const checkoutResults = checkoutBranch(dir, options.branch);
+                console.log(checkoutResults);
+                console.log('done!');
+                process.exit(0);
+            }
+        }
+        throwAndQuit('plugin already installed');
+    }
+}
+async function hashItem(item) {
+    if (fs_1.default.lstatSync(item).isDirectory()) {
+        const dir = fs_1.default.readdirSync(item);
+        const sha = crypto_1.default.createHash('sha1');
+        for (const file of dir) {
+            if (ignoredDirsForHash.includes(file) && fs_1.default.lstatSync(path_1.default.join(item, file)).isDirectory()) {
+                sha.update('');
+            }
+            else {
+                const hash = await hashItem(path_1.default.join(item, file));
+                sha.update(hash.toString());
+            }
+        }
+        return sha.digest('hex');
+    }
+    else {
+        const sha = crypto_1.default.createHash('sha1');
+        const reader = fs_1.default.createReadStream(item);
+        return new Promise(resolve => {
+            reader.on('data', chunk => {
+                sha.update(chunk);
+            });
+            reader.on('close', () => {
+                resolve(sha.digest('hex'));
+            });
+        });
+    }
+}
+async function verifyInstallation(installPath, pluginName, opts) {
+    console.log('Verifying integrity...');
+    const hash = await hashItem(installPath);
+    console.log(`SHA1 Hash: ${hash}`);
+    if (stateJson && stateJson.plugins) {
+        stateJson.plugins[pluginName].hash = hash;
+        saveState();
+    }
+    if (!opts.skipValidation) {
+        // if (hash !== data['integrityHash']) {
+        //     console.log('WARNING! Integrity hash dos not match!');
+        //     console.log(`Expected:\t${data['integrityHash']}`);
+        //     console.log(`Current:\t${hash}`);
+        //     process.exit(0);
+        // }
+        const installedFiles = fs_1.default.readdirSync(installPath);
+        if (!installedFiles.includes('package.json')) {
+            console.log('package.json not found!');
+            process.exit(0);
+        }
+    }
+}
+async function clonePluginRepo(pluginName, options) {
+    const dir = path_1.default.join(pluginDirAbsolutePath, pluginName);
+    await new Promise((resolve, reject) => {
+        const gitClone = (0, child_process_1.spawn)('git', ['clone', options.repository, dir]);
+        gitClone.stdout.on('data', chunk => {
+            console.log(`->> ${chunk}`);
+        });
+        gitClone.stderr.on('data', chunk => {
+            console.log(`->> ${chunk}`);
+        });
+        gitClone.on('close', (code) => {
+            if (code !== 0) {
+                console.log(`process exited with code ${code}`);
+            }
+            verifyInstallation(dir, pluginName, options)
+                .then(() => {
+                resolve();
+            })
+                .catch((err) => {
+                throwAndQuit(err.message);
+                reject();
+            });
+        });
+    });
+}
+function init() {
+    const check = (0, fs_1.existsSync)(pluginStatePath);
+    if (check) {
+        try {
+            stateJson = JSON.parse((0, fs_1.readFileSync)(pluginStatePath).toString());
+        }
+        catch (e) {
+            console.log(e);
+        }
+    }
+    else {
+        console.log('Creating new state file...');
+        stateJson = {
+            update_ts: Date.now(),
+            hyperion_version: package_json.version,
+            plugins: {}
+        };
+        (0, fs_1.writeFileSync)(pluginStatePath, JSON.stringify(stateJson, null, 2));
+    }
+}
+async function installPlugin(plugin, options) {
+    console.log(`Plugin Name: ${plugin}`);
+    console.log(`Repository Url: ${options.repository}`);
+    if (options.branch) {
+        console.log(`Branch Name: ${options.branch}`);
+    }
+    // check installation
+    await verifyInstalledPlugin(plugin, options);
+    if (stateJson && stateJson.plugins) {
+        stateJson.plugins[plugin] = {};
+    }
+    else {
+        throwAndQuit('fatal error on state');
+    }
+    // clone repository
+    await clonePluginRepo(plugin, options);
+    // checkout branch
+    if (options.branch) {
+        const dir = path_1.default.join(pluginDirAbsolutePath, plugin);
+        const results = checkoutBranch(dir, options.branch);
+        stateJson.plugins[plugin].branch = options.branch;
+        console.log(results);
+    }
+    // build plugin
+    await buildPlugin(plugin, {
+        skipInstall: false
+    });
+}
+async function uninstall(plugin) {
+    const pluginDir = path_1.default.join(pluginDirAbsolutePath, plugin);
+    try {
+        if (!(0, fs_1.existsSync)(pluginDir)) {
+            throwAndQuit(`plugin ${plugin} not found!`);
+        }
+        console.log(`Removing ${pluginDir}...`);
+        await (0, promises_1.rm)(pluginDir, {
+            recursive: true
+        });
+        if (stateJson && stateJson.plugins) {
+            delete stateJson.plugins[plugin];
+            saveState();
+        }
+        console.log(`${plugin} removed!`);
+    }
+    catch (e) {
+        throwAndQuit(e.message);
+    }
+}
+async function listPlugins() {
+    const dirs = await (0, promises_1.readdir)(pluginDirAbsolutePath);
+    const results = [];
+    for (const dir of dirs) {
+        try {
+            const packageJsonPath = path_1.default.join(pluginDirAbsolutePath, dir, 'package.json');
+            const package_json = JSON.parse((0, fs_1.readFileSync)(packageJsonPath).toString());
+            const { name, version, description } = package_json;
+            // console.table([dir, name, version, description].join('\t'));
+            let pluginState = false;
+            let branch = '';
+            if (stateJson.plugins && stateJson.plugins[dir]) {
+                if (stateJson.plugins[dir].enabled) {
+                    pluginState = true;
+                }
+                branch = stateJson.plugins[dir].branch;
+            }
+            results.push({
+                alias: dir,
+                enabled: pluginState,
+                plugin_name: name,
+                version,
+                branch,
+                description
+            });
+        }
+        catch (e) {
+            console.log(e.message);
+        }
+    }
+    results.sort((a, b) => b.ative - a.ative);
+    console.table(results);
+}
+function saveState() {
+    stateJson.update_ts = Date.now();
+    (0, fs_1.writeFileSync)(pluginStatePath, JSON.stringify(stateJson, null, 2));
+}
+async function setPluginState(pluginName, newState) {
+    if (stateJson) {
+        if (stateJson.plugins[pluginName]) {
+            stateJson.plugins[pluginName].enabled = newState;
+        }
+        else {
+            stateJson.plugins[pluginName] = {
+                enabled: newState
+            };
+        }
+    }
+    saveState();
+    console.log(`[${pluginName}] plugin ${newState ? 'enabled' : 'disabled'} successfully!`);
+}
+function enablePlugin(pluginName) {
+    return setPluginState(pluginName, true);
+}
+function disablePlugin(pluginName) {
+    return setPluginState(pluginName, false);
+}
+function printState() {
+    console.log(JSON.stringify(stateJson, null, 2));
+}
+async function buildAllPlugins(flags) {
+    const names = (0, fs_1.readdirSync)(pluginDirAbsolutePath);
+    for (const name of names) {
+        await buildPlugin(name, flags);
+    }
+}
+async function updatePlugin(name) {
+    const pluginDir = path_1.default.join(pluginDirAbsolutePath, name);
+    const resetStatus = gitReset(pluginDir);
+    console.log(resetStatus);
+    const pullStatus = gitPull(pluginDir);
+    console.log(pullStatus);
+    await buildPlugin(name, {});
+}
+(() => {
+    init();
+    program.description('Command-line utility to manage Hyperion plugins\nMade with ♥  by EOS Rio');
+    program.command('install <plugin>').alias('i')
+        .description('install new plugin')
+        .option('-r, --repository <url>', 'custom repository url', checkUrl)
+        .option('-b, --branch <branch>', 'checkout specific branch')
+        .option('-s, --skip-validation', 'skip plugin integrity validation')
+        .action(installPlugin);
+    program.command('enable <plugin>')
+        .description('enable plugin')
+        .action(enablePlugin);
+    program.command('disable <plugin>')
+        .description('disable plugin')
+        .action(disablePlugin);
+    program.command('state')
+        .description('print current state file')
+        .action(printState);
+    program.command('list').alias('ls')
+        .description('list installed plugins')
+        .action(listPlugins);
+    program.command('build <plugin>').alias('b')
+        .description('build a single plugin')
+        .option('-s, --skip-install', 'skip "npm install" step')
+        .action(buildPlugin);
+    program.command('update <plugin>')
+        .description('update a single plugin')
+        .action(updatePlugin);
+    program.command('build-all')
+        .description('build all plugins')
+        .option('-s, --skip-install', 'skip "npm install" step')
+        .action(buildAllPlugins);
+    program.command('uninstall <plugin>').alias('rm')
+        .description('uninstall plugin')
+        .action(uninstall);
+    program.parse();
+})();
+//# sourceMappingURL=hpm.js.map

scripts/hpm.ts

@@ -0,0 +1,432 @@
+import {Command, InvalidArgumentError} from 'commander';
+import path from 'path'
+import {readdir, rm} from "fs/promises";
+import fs, {existsSync, readdirSync, readFileSync, writeFileSync} from "fs";
+import {execSync, spawn} from "child_process";
+import crypto from "crypto";
+
+const package_json = JSON.parse(readFileSync('./package.json').toString());
+const debug = process.env.HPM_DEBUG;
+const program = new Command();
+const pluginDirAbsolutePath = path.join(path.resolve(), 'plugins', 'repos');
+const pluginStatePath = path.join(path.resolve(), 'plugins', '.state.json');
+
+let stateJson;
+
+const ignoredDirsForHash = ['.git', 'node_modules'];
+
+program.version(package_json.version);
+
+function checkUrl(value) {
+    if (value.startsWith('https://') || value.startsWith('git@')) {
+        return value;
+    } else {
+        throw new InvalidArgumentError('Invalid url.');
+    }
+}
+
+function throwAndQuit(message: string) {
+    console.error('⚠ >>', message);
+    process.exit(1);
+}
+
+function getCurrentBranch(dir): string {
+    const gitCmd = `(cd ${dir} && git rev-parse --abbrev-ref HEAD)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return execSync(gitCmd).toString().trim();
+}
+
+function checkoutBranch(dir, branchName): string {
+    const gitCmd = `(cd ${dir} && git checkout ${branchName})`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return execSync(gitCmd).toString();
+}
+
+function gitPull(dir): string {
+    const gitCmd = `(cd ${dir} && git pull)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return execSync(gitCmd).toString();
+}
+
+function gitReset(dir): string {
+    const gitCmd = `(cd ${dir} && git reset --hard)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return execSync(gitCmd).toString();
+}
+
+async function buildPlugin(name, flags) {
+
+    const pluginDir = path.join(pluginDirAbsolutePath, name);
+
+    if (!existsSync(pluginDir)) {
+        throwAndQuit(`plugin ${name} not found!`);
+    }
+
+    const currentBranch = getCurrentBranch(pluginDir);
+
+    console.log(`-------- ⚒ ${name} @ ${currentBranch} -----------`);
+
+    if (!flags.skipInstall) {
+        // run npm install
+        await new Promise(resolve => {
+            const npmInstall = spawn('npm', ['install'], {
+                cwd: pluginDir,
+                stdio: 'inherit',
+            });
+            npmInstall.on('close', (code) => {
+                if (code !== 0) {
+                    console.log(`process exited with code ${code}`);
+                    process.exit(code);
+                }
+                resolve(true);
+            });
+        });
+    }
+
+    // typescript build
+    await new Promise(resolve => {
+        const npmInstall = spawn('npm', ['run', 'build'], {
+            cwd: pluginDir,
+            stdio: 'inherit',
+        });
+        npmInstall.on('close', (code) => {
+            if (code !== 0) {
+                console.log(`process exited with code ${code}`);
+                process.exit(code);
+            }
+            resolve(true);
+        });
+    });
+
+    console.log('-------------------------------');
+
+    // save build date
+    if (stateJson) {
+
+        if (!stateJson.plugins[name]) {
+            stateJson.plugins[name] = {};
+        }
+
+        stateJson.plugins[name].last_build_date = new Date().toISOString();
+        const hash = await hashItem(pluginDir);
+        console.log(`SHA1 Hash: ${hash}`);
+        stateJson.plugins[name].hash = hash;
+        stateJson.plugins[name].branch = currentBranch;
+        saveState();
+    }
+}
+
+async function verifyInstalledPlugin(pluginName: string, options: any): Promise<void> {
+    const dir = path.join(pluginDirAbsolutePath, pluginName);
+    const dirExists = existsSync(dir);
+    if (dirExists) {
+        if (options.branch && options.branch !== '') {
+            const currentBranch = getCurrentBranch(dir);
+            if (currentBranch === options.branch) {
+                console.log('✅ Plugin already installed, branch check OK');
+                process.exit(0);
+            } else {
+                // checkout new branch
+                const checkoutResults = checkoutBranch(dir, options.branch);
+                console.log(checkoutResults);
+                console.log('done!');
+                process.exit(0);
+            }
+        }
+        throwAndQuit('plugin already installed');
+    }
+}
+
+async function hashItem(item) {
+    if (fs.lstatSync(item).isDirectory()) {
+        const dir = fs.readdirSync(item);
+        const sha = crypto.createHash('sha1');
+        for (const file of dir) {
+            if (ignoredDirsForHash.includes(file) && fs.lstatSync(path.join(item, file)).isDirectory()) {
+                sha.update('');
+            } else {
+                const hash = await hashItem(path.join(item, file));
+                sha.update(hash.toString());
+            }
+        }
+        return sha.digest('hex');
+    } else {
+        const sha = crypto.createHash('sha1');
+        const reader = fs.createReadStream(item);
+        return new Promise(resolve => {
+            reader.on('data', chunk => {
+                sha.update(chunk);
+            });
+            reader.on('close', () => {
+                resolve(sha.digest('hex'));
+            });
+        });
+    }
+}
+
+async function verifyInstallation(installPath, pluginName, opts) {
+    console.log('Verifying integrity...');
+    const hash = await hashItem(installPath);
+    console.log(`SHA1 Hash: ${hash}`);
+
+    if (stateJson && stateJson.plugins) {
+        stateJson.plugins[pluginName].hash = hash;
+        saveState();
+    }
+
+    if (!opts.skipValidation) {
+
+        // if (hash !== data['integrityHash']) {
+        //     console.log('WARNING! Integrity hash dos not match!');
+        //     console.log(`Expected:\t${data['integrityHash']}`);
+        //     console.log(`Current:\t${hash}`);
+        //     process.exit(0);
+        // }
+
+        const installedFiles = fs.readdirSync(installPath);
+        if (!installedFiles.includes('package.json')) {
+            console.log('package.json not found!');
+            process.exit(0);
+        }
+    }
+}
+
+async function clonePluginRepo(pluginName: string, options: any): Promise<void> {
+    const dir = path.join(pluginDirAbsolutePath, pluginName);
+    await new Promise<void>((resolve, reject) => {
+        const gitClone = spawn('git', ['clone', options.repository, dir]);
+        gitClone.stdout.on('data', chunk => {
+            console.log(`->> ${chunk}`);
+        });
+        gitClone.stderr.on('data', chunk => {
+            console.log(`->> ${chunk}`);
+        });
+        gitClone.on('close', (code) => {
+            if (code !== 0) {
+                console.log(`process exited with code ${code}`);
+            }
+            verifyInstallation(dir, pluginName, options)
+                .then(() => {
+                    resolve();
+                })
+                .catch((err) => {
+                    throwAndQuit(err.message);
+                    reject();
+                });
+        });
+    });
+}
+
+function init() {
+    const check = existsSync(pluginStatePath);
+    if (check) {
+        try {
+            stateJson = JSON.parse(readFileSync(pluginStatePath).toString());
+        } catch (e) {
+            console.log(e);
+        }
+    } else {
+        console.log('Creating new state file...');
+        stateJson = {
+            update_ts: Date.now(),
+            hyperion_version: package_json.version,
+            plugins: {}
+        };
+        writeFileSync(pluginStatePath, JSON.stringify(stateJson, null, 2));
+    }
+}
+
+async function installPlugin(plugin: string, options: any) {
+    console.log(`Plugin Name: ${plugin}`);
+    console.log(`Repository Url: ${options.repository}`);
+
+    if (options.branch) {
+        console.log(`Branch Name: ${options.branch}`);
+    }
+
+    // check installation
+    await verifyInstalledPlugin(plugin, options);
+
+    if (stateJson && stateJson.plugins) {
+        stateJson.plugins[plugin] = {};
+    } else {
+        throwAndQuit('fatal error on state');
+    }
+
+    // clone repository
+    await clonePluginRepo(plugin, options);
+
+    // checkout branch
+    if (options.branch) {
+        const dir = path.join(pluginDirAbsolutePath, plugin);
+        const results = checkoutBranch(dir, options.branch);
+        stateJson.plugins[plugin].branch = options.branch;
+        console.log(results);
+    }
+
+    // build plugin
+    await buildPlugin(plugin, {
+        skipInstall: false
+    });
+}
+
+async function uninstall(plugin) {
+    const pluginDir = path.join(pluginDirAbsolutePath, plugin);
+    try {
+        if (!existsSync(pluginDir)) {
+            throwAndQuit(`plugin ${plugin} not found!`);
+        }
+        console.log(`Removing ${pluginDir}...`);
+        await rm(pluginDir, {
+            recursive: true
+        });
+        if (stateJson && stateJson.plugins) {
+            delete stateJson.plugins[plugin];
+            saveState();
+        }
+        console.log(`${plugin} removed!`);
+    } catch (e) {
+        throwAndQuit(e.message);
+    }
+}
+
+async function listPlugins() {
+    const dirs = await readdir(pluginDirAbsolutePath);
+    const results: any[] = [];
+    for (const dir of dirs) {
+        try {
+            const packageJsonPath = path.join(pluginDirAbsolutePath, dir, 'package.json');
+            const package_json = JSON.parse(readFileSync(packageJsonPath).toString());
+            const {name, version, description} = package_json;
+            // console.table([dir, name, version, description].join('\t'));
+
+            let pluginState = false;
+            let branch = '';
+
+            if (stateJson.plugins && stateJson.plugins[dir]) {
+                if (stateJson.plugins[dir].enabled) {
+                    pluginState = true;
+                }
+                branch = stateJson.plugins[dir].branch;
+            }
+
+            results.push({
+                alias: dir,
+                enabled: pluginState,
+                plugin_name: name,
+                version,
+                branch,
+                description
+            });
+        } catch (e) {
+            console.log(e.message);
+        }
+    }
+    results.sort((a, b) => b.ative - a.ative);
+    console.table(results);
+}
+
+function saveState() {
+    stateJson.update_ts = Date.now();
+    writeFileSync(pluginStatePath, JSON.stringify(stateJson, null, 2));
+}
+
+async function setPluginState(pluginName: string, newState: boolean) {
+    if (stateJson) {
+        if (stateJson.plugins[pluginName]) {
+            stateJson.plugins[pluginName].enabled = newState;
+        } else {
+            stateJson.plugins[pluginName] = {
+                enabled: newState
+            }
+        }
+    }
+    saveState();
+    console.log(`[${pluginName}] plugin ${newState ? 'enabled' : 'disabled'} successfully!`);
+}
+
+function enablePlugin(pluginName: string) {
+    return setPluginState(pluginName, true);
+}
+
+function disablePlugin(pluginName: string) {
+    return setPluginState(pluginName, false);
+}
+
+function printState() {
+    console.log(JSON.stringify(stateJson, null, 2));
+}
+
+async function buildAllPlugins(flags) {
+    const names = readdirSync(pluginDirAbsolutePath);
+    for (const name of names) {
+        await buildPlugin(name, flags)
+    }
+}
+
+async function updatePlugin(name: string) {
+    const pluginDir = path.join(pluginDirAbsolutePath, name);
+    const resetStatus = gitReset(pluginDir);
+    console.log(resetStatus);
+    const pullStatus = gitPull(pluginDir);
+    console.log(pullStatus);
+    await buildPlugin(name, {});
+}
+
+(() => {
+
+    init();
+
+    program.description('Command-line utility to manage Hyperion plugins\nMade with ♥  by EOS Rio');
+
+    program.command('install <plugin>').alias('i')
+        .description('install new plugin')
+        .option('-r, --repository <url>', 'custom repository url', checkUrl)
+        .option('-b, --branch <branch>', 'checkout specific branch')
+        .option('-s, --skip-validation', 'skip plugin integrity validation')
+        .action(installPlugin);
+
+    program.command('enable <plugin>')
+        .description('enable plugin')
+        .action(enablePlugin);
+
+    program.command('disable <plugin>')
+        .description('disable plugin')
+        .action(disablePlugin);
+
+    program.command('state')
+        .description('print current state file')
+        .action(printState);
+
+    program.command('list').alias('ls')
+        .description('list installed plugins')
+        .action(listPlugins);
+
+    program.command('build <plugin>').alias('b')
+        .description('build a single plugin')
+        .option('-s, --skip-install', 'skip "npm install" step')
+        .action(buildPlugin);
+
+    program.command('update <plugin>')
+        .description('update a single plugin')
+        .action(updatePlugin);
+
+    program.command('build-all')
+        .description('build all plugins')
+        .option('-s, --skip-install', 'skip "npm install" step')
+        .action(buildAllPlugins);
+
+    program.command('uninstall <plugin>').alias('rm')
+        .description('uninstall plugin')
+        .action(uninstall);
+
+    program.parse();
+})();

scripts/hyp-config.js

@@ -0,0 +1,259 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const path_1 = __importDefault(require("path"));
+const promises_1 = require("fs/promises");
+const fs_1 = require("fs");
+const eosjs_1 = require("eosjs");
+const cross_fetch_1 = __importDefault(require("cross-fetch"));
+const ws_1 = __importDefault(require("ws"));
+const program = new commander_1.Command();
+const chainsDir = path_1.default.join(path_1.default.resolve(), 'chains');
+async function getConnections() {
+    const connectionsJsonFile = await (0, promises_1.readFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'));
+    return JSON.parse(connectionsJsonFile.toString());
+}
+async function getExampleConfig() {
+    const exampleChainData = await (0, promises_1.readFile)(path_1.default.join(chainsDir, 'example.config.json'));
+    return JSON.parse(exampleChainData.toString());
+}
+async function listChains(flags) {
+    const dirdata = await (0, promises_1.readdir)(chainsDir);
+    const connections = await getConnections();
+    const exampleChain = await getExampleConfig();
+    const configuredTable = [];
+    const pendingTable = [];
+    for (const file of dirdata.filter(f => f.endsWith('.json'))) {
+        if (file === 'example.config.json') {
+            continue;
+        }
+        try {
+            const fileData = await (0, promises_1.readFile)(path_1.default.join(chainsDir, file));
+            const jsonData = JSON.parse(fileData.toString());
+            const chainName = jsonData.settings.chain;
+            const chainConn = connections.chains[chainName];
+            const fullChainName = jsonData.api.chain_name;
+            // validate configs with example
+            let missingSections = 0;
+            let missingFields = 0;
+            for (const key in exampleChain) {
+                if (!jsonData[key]) {
+                    if (flags.fixMissingFields) {
+                        jsonData[key] = exampleChain[key];
+                    }
+                    else {
+                        console.log(`⚠ Section ${key} missing on ${file}`);
+                        missingSections++;
+                    }
+                }
+                else {
+                    for (const subKey in exampleChain[key]) {
+                        if (!jsonData[key].hasOwnProperty(subKey)) {
+                            if (flags.fixMissingFields) {
+                                jsonData[key][subKey] = exampleChain[key][subKey];
+                            }
+                            else {
+                                console.log(`⚠ Field ${subKey} missing from ${key} on ${file} (default = ${exampleChain[key][subKey]})`);
+                                missingFields++;
+                            }
+                        }
+                    }
+                }
+            }
+            if (flags.fixMissingFields) {
+                await (0, promises_1.writeFile)(path_1.default.join(chainsDir, file), JSON.stringify(jsonData, null, 2));
+            }
+            const common = {
+                full_name: fullChainName,
+                name: chainName,
+                parser: jsonData.settings.parser,
+                abi_scan: jsonData.indexer.abi_scan_mode,
+                live_reader: jsonData.indexer.live_reader,
+                start_on: jsonData.indexer.start_on,
+                stop_on: jsonData.indexer.stop_on
+            };
+            if (chainConn) {
+                configuredTable.push({
+                    ...common,
+                    nodeos_http: chainConn?.http,
+                    nodeos_ship: chainConn?.ship,
+                    chain_id: chainConn?.chain_id.substr(0, 16) + '...'
+                });
+            }
+            else {
+                pendingTable.push(common);
+            }
+        }
+        catch (e) {
+            console.log(`Failed to parse ${file} - ${e.message}`);
+        }
+    }
+    console.log(' >>>> Fully Configured Chains');
+    console.table(configuredTable);
+    if (!flags.valid) {
+        console.log(' >>>> Pending Configuration Chains');
+        console.table(pendingTable);
+    }
+}
+async function newChain(shortName, options) {
+    console.log(`Creating new config for ${shortName}...`);
+    const targetPath = path_1.default.join(chainsDir, `${shortName}.config.json`);
+    if ((0, fs_1.existsSync)(targetPath)) {
+        console.error(`Chain config for ${shortName} already defined!`);
+        process.exit(0);
+    }
+    // read example
+    const exampleChain = await getExampleConfig();
+    // read connections.json
+    const connections = await getConnections();
+    if (connections.chains[shortName]) {
+        console.log('Connections already defined!');
+        console.log(connections.chains[shortName]);
+        process.exit(0);
+    }
+    else {
+        connections.chains[shortName] = {
+            name: '',
+            ship: '',
+            http: '',
+            chain_id: '',
+            WS_ROUTER_HOST: '127.0.0.1',
+            WS_ROUTER_PORT: 7001
+        };
+    }
+    const jsonData = exampleChain;
+    jsonData.settings.chain = shortName;
+    if (options.http) {
+        if (options.http.startsWith('http://') || options.http.startsWith('https://')) {
+            console.log(`Testing connection on ${options.http}`);
+        }
+        else {
+            console.error(`Invalid HTTP Endpoint [${options.http}] - Url must start with either http:// or https://`);
+            process.exit(1);
+        }
+        // test nodeos availability
+        const jsonRpc = new eosjs_1.JsonRpc(options.http, { fetch: cross_fetch_1.default });
+        const info = await jsonRpc.get_info();
+        jsonData.api.chain_api = options.http;
+        connections.chains[shortName].chain_id = info.chain_id;
+        connections.chains[shortName].http = options.http;
+        if (info.server_version_string.includes('2.1')) {
+            jsonData.settings.parser = '2.1';
+        }
+        else {
+            jsonData.settings.parser = '1.8';
+        }
+        console.log(`Parser version set to ${jsonData.settings.parser}`);
+        console.log(`Chain ID: ${info.chain_id}`);
+    }
+    if (options.ship) {
+        if (options.ship.startsWith('ws://') || options.ship.startsWith('wss://')) {
+            console.log(`Testing connection on ${options.ship}`);
+        }
+        else {
+            console.error(`Invalid WS Endpoint [${options.ship}] - Url must start with either ws:// or wss://`);
+            process.exit(1);
+        }
+        // test ship availability
+        const status = await new Promise(resolve => {
+            const ws = new ws_1.default(options.ship);
+            ws.on("message", (data) => {
+                try {
+                    const abi = JSON.parse(data.toString());
+                    if (abi.version) {
+                        console.log(`Received SHIP Abi Version: ${abi.version}`);
+                        resolve(true);
+                    }
+                    else {
+                        resolve(false);
+                    }
+                }
+                catch (e) {
+                    console.log(e.message);
+                    resolve(false);
+                }
+                ws.close();
+            });
+            ws.on("error", err => {
+                console.log(err);
+                ws.close();
+                resolve(false);
+            });
+        });
+        if (status) {
+            connections.chains[shortName].ship = options.ship;
+        }
+        else {
+            console.log(`Invalid SHIP Endpoint [${options.ship}]`);
+        }
+    }
+    const fullNameArr = [];
+    shortName.split('-').forEach((word) => {
+        fullNameArr.push(word[0].toUpperCase() + word.substr(1));
+    });
+    const fullChainName = fullNameArr.join(' ');
+    jsonData.api.chain_name = fullChainName;
+    connections.chains[shortName].name = fullChainName;
+    console.log(connections.chains[shortName]);
+    console.log('Saving connections.json...');
+    await (0, promises_1.writeFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    console.log(`Saving chains/${shortName}.config.json...`);
+    await (0, promises_1.writeFile)(targetPath, JSON.stringify(jsonData, null, 2));
+}
+async function rmChain(shortName) {
+    console.log(`Removing config for ${shortName}...`);
+    const targetPath = path_1.default.join(chainsDir, `${shortName}.config.json`);
+    if (!(0, fs_1.existsSync)(targetPath)) {
+        console.error(`Chain config for ${shortName} not found!`);
+        process.exit(0);
+    }
+    // create backups
+    const backupDir = path_1.default.join(path_1.default.resolve(), 'configuration_backups');
+    if (!(0, fs_1.existsSync)(backupDir)) {
+        await (0, promises_1.mkdir)(backupDir);
+    }
+    const connections = await getConnections();
+    try {
+        const chainConn = connections.chains[shortName];
+        const now = Date.now();
+        if (chainConn) {
+            await (0, promises_1.writeFile)(path_1.default.join(backupDir, `${shortName}_${now}_connections.json`), JSON.stringify(chainConn, null, 2));
+        }
+        await (0, promises_1.cp)(targetPath, path_1.default.join(backupDir, `${shortName}_${now}_config.json`));
+    }
+    catch (e) {
+        console.log(e.message);
+        console.log('Failed to create backups! Aborting!');
+        process.exit(1);
+    }
+    console.log(`Deleting ${targetPath}`);
+    await (0, promises_1.rm)(targetPath);
+    delete connections.chains[shortName];
+    console.log('Saving connections.json...');
+    await (0, promises_1.writeFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    console.log(`✅ ${shortName} removal completed!`);
+}
+// main program
+(() => {
+    const list = program.command('list').alias('ls');
+    list.command('chains')
+        .option('--valid', 'only show valid chains')
+        .option('--fix-missing-fields', 'set defaults on missing fields')
+        .description('list configured chains')
+        .action(listChains);
+    const newCmd = program.command('new').alias('n');
+    newCmd.command('chain <shortName>')
+        .description('initialize new chain config based on example')
+        .option('--http <http_endpoint>', 'define chain api http endpoint')
+        .option('--ship <ship_endpoint>', 'define state history ws endpoint')
+        .action(newChain);
+    const remove = program.command('remove').alias('rm');
+    remove.command('chain <shortName>')
+        .description('backup and delete chain configuration')
+        .action(rmChain);
+    program.parse(process.argv);
+})();
+//# sourceMappingURL=hyp-config.js.map

scripts/hyp-config.ts

@@ -0,0 +1,283 @@
+import {Command} from "commander";
+import path from "path";
+import {cp, mkdir, readdir, readFile, rm, writeFile} from "fs/promises";
+import {HyperionConfig} from "../interfaces/hyperionConfig";
+import {HyperionConnections} from "../interfaces/hyperionConnections";
+import {existsSync} from "fs";
+import {JsonRpc} from "eosjs";
+import fetch from "cross-fetch";
+import WebSocket from 'ws';
+
+const program = new Command();
+const chainsDir = path.join(path.resolve(), 'chains');
+
+async function getConnections(): Promise<HyperionConnections> {
+    const connectionsJsonFile = await readFile(path.join(path.resolve(), 'connections.json'));
+    return JSON.parse(connectionsJsonFile.toString());
+}
+
+async function getExampleConfig(): Promise<HyperionConfig> {
+    const exampleChainData = await readFile(path.join(chainsDir, 'example.config.json'));
+    return JSON.parse(exampleChainData.toString());
+}
+
+
+async function listChains(flags) {
+
+    const dirdata = await readdir(chainsDir);
+    const connections = await getConnections();
+    const exampleChain = await getExampleConfig();
+
+    const configuredTable = [];
+    const pendingTable = [];
+    for (const file of dirdata.filter(f => f.endsWith('.json'))) {
+        if (file === 'example.config.json') {
+            continue;
+        }
+        try {
+            const fileData = await readFile(path.join(chainsDir, file));
+            const jsonData: HyperionConfig = JSON.parse(fileData.toString());
+            const chainName = jsonData.settings.chain;
+            const chainConn = connections.chains[chainName];
+            const fullChainName = jsonData.api.chain_name;
+
+            // validate configs with example
+            let missingSections = 0;
+            let missingFields = 0;
+            for (const key in exampleChain) {
+                if (!jsonData[key]) {
+                    if (flags.fixMissingFields) {
+                        jsonData[key] = exampleChain[key];
+                    } else {
+                        console.log(`⚠ Section ${key} missing on ${file}`);
+                        missingSections++;
+                    }
+                } else {
+                    for (const subKey in exampleChain[key]) {
+                        if (!jsonData[key].hasOwnProperty(subKey)) {
+                            if (flags.fixMissingFields) {
+                                jsonData[key][subKey] = exampleChain[key][subKey];
+                            } else {
+                                console.log(`⚠ Field ${subKey} missing from ${key} on ${file} (default = ${exampleChain[key][subKey]})`);
+                                missingFields++;
+                            }
+                        }
+                    }
+                }
+            }
+
+            if (flags.fixMissingFields) {
+                await writeFile(path.join(chainsDir, file), JSON.stringify(jsonData, null, 2));
+            }
+
+            const common = {
+                full_name: fullChainName,
+                name: chainName,
+                parser: jsonData.settings.parser,
+                abi_scan: jsonData.indexer.abi_scan_mode,
+                live_reader: jsonData.indexer.live_reader,
+                start_on: jsonData.indexer.start_on,
+                stop_on: jsonData.indexer.stop_on
+            };
+
+            if (chainConn) {
+                configuredTable.push({
+                    ...common,
+                    nodeos_http: chainConn?.http,
+                    nodeos_ship: chainConn?.ship,
+                    chain_id: chainConn?.chain_id.substr(0,16) + '...'
+                });
+            } else {
+                pendingTable.push(common);
+            }
+        } catch (e) {
+            console.log(`Failed to parse ${file} - ${e.message}`);
+        }
+    }
+
+    console.log(' >>>> Fully Configured Chains');
+    console.table(configuredTable);
+
+    if (!flags.valid) {
+        console.log(' >>>> Pending Configuration Chains');
+        console.table(pendingTable);
+    }
+}
+
+async function newChain(shortName, options) {
+    console.log(`Creating new config for ${shortName}...`);
+    const targetPath = path.join(chainsDir, `${shortName}.config.json`);
+
+    if (existsSync(targetPath)) {
+        console.error(`Chain config for ${shortName} already defined!`);
+        process.exit(0);
+    }
+
+    // read example
+    const exampleChain = await getExampleConfig();
+
+    // read connections.json
+    const connections = await getConnections();
+
+    if (connections.chains[shortName]) {
+        console.log('Connections already defined!');
+        console.log(connections.chains[shortName]);
+        process.exit(0);
+    } else {
+        connections.chains[shortName] = {
+            name: '',
+            ship: '',
+            http: '',
+            chain_id: '',
+            WS_ROUTER_HOST: '127.0.0.1',
+            WS_ROUTER_PORT: 7001
+        };
+    }
+
+    const jsonData = exampleChain;
+    jsonData.settings.chain = shortName;
+
+    if (options.http) {
+
+        if (options.http.startsWith('http://') || options.http.startsWith('https://')) {
+            console.log(`Testing connection on ${options.http}`);
+        } else {
+            console.error(`Invalid HTTP Endpoint [${options.http}] - Url must start with either http:// or https://`);
+            process.exit(1);
+        }
+
+        // test nodeos availability
+        const jsonRpc = new JsonRpc(options.http, {fetch});
+        const info = await jsonRpc.get_info();
+        jsonData.api.chain_api = options.http;
+        connections.chains[shortName].chain_id = info.chain_id;
+        connections.chains[shortName].http = options.http;
+        if (info.server_version_string.includes('2.1')) {
+            jsonData.settings.parser = '2.1';
+        } else {
+            jsonData.settings.parser = '1.8';
+        }
+        console.log(`Parser version set to ${jsonData.settings.parser}`);
+        console.log(`Chain ID: ${info.chain_id}`)
+    }
+
+    if (options.ship) {
+
+        if (options.ship.startsWith('ws://') || options.ship.startsWith('wss://')) {
+            console.log(`Testing connection on ${options.ship}`);
+        } else {
+            console.error(`Invalid WS Endpoint [${options.ship}] - Url must start with either ws:// or wss://`);
+            process.exit(1);
+        }
+
+        // test ship availability
+        const status = await new Promise<boolean>(resolve => {
+            const ws = new WebSocket(options.ship);
+            ws.on("message", (data: Buffer) => {
+                try {
+                    const abi = JSON.parse(data.toString());
+                    if (abi.version) {
+                        console.log(`Received SHIP Abi Version: ${abi.version}`);
+                        resolve(true);
+                    } else {
+                        resolve(false);
+                    }
+                } catch (e) {
+                    console.log(e.message);
+                    resolve(false);
+                }
+                ws.close();
+            });
+            ws.on("error", err => {
+                console.log(err);
+                ws.close();
+                resolve(false);
+            });
+        });
+        if (status) {
+            connections.chains[shortName].ship = options.ship;
+        } else {
+            console.log(`Invalid SHIP Endpoint [${options.ship}]`);
+        }
+    }
+
+    const fullNameArr = [];
+    shortName.split('-').forEach((word: string) => {
+        fullNameArr.push(word[0].toUpperCase() + word.substr(1));
+        });
+    const fullChainName = fullNameArr.join(' ');
+
+    jsonData.api.chain_name = fullChainName;
+    connections.chains[shortName].name = fullChainName;
+
+    console.log(connections.chains[shortName]);
+
+    console.log('Saving connections.json...');
+    await writeFile(path.join(path.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    console.log(`Saving chains/${shortName}.config.json...`);
+    await writeFile(targetPath, JSON.stringify(jsonData, null, 2));
+}
+
+async function rmChain(shortName) {
+    console.log(`Removing config for ${shortName}...`);
+    const targetPath = path.join(chainsDir, `${shortName}.config.json`);
+
+    if (!existsSync(targetPath)) {
+        console.error(`Chain config for ${shortName} not found!`);
+        process.exit(0);
+    }
+
+    // create backups
+    const backupDir = path.join(path.resolve(), 'configuration_backups');
+    if (!existsSync(backupDir)) {
+        await mkdir(backupDir);
+    }
+
+    const connections = await getConnections();
+    try {
+        const chainConn = connections.chains[shortName];
+        const now = Date.now();
+        if (chainConn) {
+            await writeFile(
+                path.join(backupDir, `${shortName}_${now}_connections.json`),
+                JSON.stringify(chainConn, null, 2)
+            );
+        }
+        await cp(targetPath, path.join(backupDir, `${shortName}_${now}_config.json`));
+    } catch (e) {
+        console.log(e.message);
+        console.log('Failed to create backups! Aborting!');
+        process.exit(1);
+    }
+
+    console.log(`Deleting ${targetPath}`);
+    await rm(targetPath);
+    delete connections.chains[shortName];
+    console.log('Saving connections.json...');
+    await writeFile(path.join(path.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    console.log(`✅ ${shortName} removal completed!`);
+}
+
+// main program
+(() => {
+    const list = program.command('list').alias('ls');
+    list.command('chains')
+        .option('--valid', 'only show valid chains')
+        .option('--fix-missing-fields', 'set defaults on missing fields')
+        .description('list configured chains')
+        .action(listChains);
+
+    const newCmd = program.command('new').alias('n');
+    newCmd.command('chain <shortName>')
+        .description('initialize new chain config based on example')
+        .option('--http <http_endpoint>', 'define chain api http endpoint')
+        .option('--ship <ship_endpoint>', 'define state history ws endpoint')
+        .action(newChain)
+
+    const remove = program.command('remove').alias('rm');
+    remove.command('chain <shortName>')
+        .description('backup and delete chain configuration')
+        .action(rmChain);
+
+    program.parse(process.argv);
+})();

types/fastify/index.d.ts

@@ -1,21 +1,23 @@
 import {Client} from "@elastic/elasticsearch";
-import {Redis} from "ioredis";
 import {ConnectionManager} from "../../connections/manager.class";
 import {Api, JsonRpc} from "eosjs";
+import {CacheManager} from "../../api/helpers/cacheManager";
+import {FastifyRedis} from "@fastify/redis";
 
 declare module 'fastify' {
-	export interface FastifyInstance {
-		manager: ConnectionManager;
-		redis: Redis;
-		elastic: Client;
-		eosjs: {
-			rpc: JsonRpc;
-			api: Api;
-		};
-		chain_api: string;
-		push_api: string;
-		tokenCache: Map<string, any>;
-		allowedActionQueryParamSet: Set<string>;
-		routeSet: Set<string>;
-	}
+    export interface FastifyInstance {
+        manager: ConnectionManager;
+        cacheManager: CacheManager;
+        redis: FastifyRedis;
+        elastic: Client;
+        eosjs: {
+            rpc: JsonRpc;
+            api: Api;
+        };
+        chain_api: string;
+        push_api: string;
+        tokenCache: Map<string, any>;
+        allowedActionQueryParamSet: Set<string>;
+        routeSet: Set<string>;
+    }
 }

workers/deserializer.ts

@@ -1145,21 +1145,8 @@ export default class MainDSWorker extends HyperionWorker {
                     }
                 }
 
-                let jsonRow;
-                // forced deserialization
-                try {
-                    if (payload.code === 'm.federation' && payload.table === 'leaders') {
-                        payload['data'] = {};
-                        jsonRow = payload;
-                    }
-                } catch (e) {
-                    console.log(e);
-                }
-
                 // decode contract data
-                if (!jsonRow) {
-                    jsonRow = await this.processContractRowNative(payload, block_num);
-                }
+                let jsonRow = await this.processContractRowNative(payload, block_num);
 
                 if (jsonRow?.value && !jsonRow['_blacklisted']) {
                     debugLog('Delta DS failed ->>', jsonRow);
@@ -1471,7 +1458,6 @@ export default class MainDSWorker extends HyperionWorker {
                                 await this.deltaStructHandlers[key](data[1], block_num, block_ts, row, block_id);
                             } catch (e) {
                                 hLog(`Delta struct [${key}] processing error: ${e.message}`);
-                                // hLog(data);
                             }
                         }
                     }
@@ -1509,7 +1495,7 @@ export default class MainDSWorker extends HyperionWorker {
             proposal_name: data['@approvals']['proposal_name'],
             requested_approvals: data['@approvals']['requested_approvals'],
             provided_approvals: data['@approvals']['provided_approvals'],
-            executed: data.present === false,
+            executed: data.present === false || data.present === 0,
             primary_key: data['primary_key'],
             block_num: data['block_num']
         };

workers/ds-pool.ts

@@ -21,6 +21,48 @@ interface CustomAbiDef {
     endingBlock: number;
 }
 
+function cleanActionTrace(t: any) {
+    try {
+        if (t.return_value === '') {
+            delete t.return_value;
+        }
+        if (t.context_free === false) {
+            delete t.context_free;
+        }
+        if (t.elapsed === '0') {
+            delete t.elapsed;
+        }
+
+        // remove act_digest from grouped receipts since it was written to the action
+        if (t.receipts && t.receipts.length > 0) {
+            t.act_digest = t.receipts[0].act_digest;
+            for (let receipt of t.receipts) {
+                delete receipt.act_digest;
+            }
+        } else {
+            delete t.receipts;
+        }
+
+        delete t.console;
+        delete t.receiver;
+
+        // onblock action case
+        if (t.signatures && t.signatures.length === 0) {
+            delete t.signatures;
+        }
+
+        if (t.inline_count === 0) {
+            delete t.inline_count;
+        }
+
+        if (t.net_usage_words === 0) {
+            delete t.net_usage_words;
+        }
+    } catch (e) {
+        console.log(e);
+    }
+}
+
 export default class DSPoolWorker extends HyperionWorker {
 
     abi;
@@ -261,7 +303,7 @@ export default class DSPoolWorker extends HyperionWorker {
                 debugLog(`(abieos) ${_action.account}::${_action.name} @ ${block_num} >>> ${e.message}`);
             }
         }
-        return await self.deserializeActionAtBlock(_action, block_num);
+        return self.deserializeActionAtBlock(_action, block_num);
     }
 
     async getAbiFromHeadBlock(code) {
@@ -380,9 +422,9 @@ export default class DSPoolWorker extends HyperionWorker {
                         action.data,
                         this.txEnc,
                         this.txDec
-                    );
+                    ).data;
                 } catch (e) {
-                    debugLog(`(eosjs)  ${action.account}::${action.name} @ ${block_num} >>> ${e.message}`);
+                    debugLog(`(eosjs) ${action.account}::${action.name} @ ${block_num} >>> ${e.message}`);
                     return null;
                 }
             } else {
@@ -464,20 +506,12 @@ export default class DSPoolWorker extends HyperionWorker {
             }
 
             const _finalTraces = [];
+
             if (_processedTraces.length > 1) {
                 const act_digests = {};
 
                 // collect digests & receipts
                 for (const _trace of _processedTraces) {
-
-                    if (this.conf.settings.dsp_parser) {
-                        if (_trace.console !== '') {
-                            await parseDSPEvent(this, _trace);
-                        }
-                    } else {
-                        delete _trace.console;
-                    }
-
                     if (act_digests[_trace.receipt.act_digest]) {
                         act_digests[_trace.receipt.act_digest].push(_trace.receipt);
                     } else {
@@ -488,18 +522,17 @@ export default class DSPoolWorker extends HyperionWorker {
                 // Apply notified accounts to first trace instance
                 for (const _trace of _processedTraces) {
                     if (act_digests[_trace.receipt.act_digest]) {
-                        const notifiedSet = new Set();
+                        // const notifiedSet = new Set();
                         _trace['receipts'] = [];
                         for (const _receipt of act_digests[_trace.receipt.act_digest]) {
-                            notifiedSet.add(_receipt.receiver);
+                            // notifiedSet.add(_receipt.receiver);
                             _trace['code_sequence'] = _receipt['code_sequence'];
                             delete _receipt['code_sequence'];
                             _trace['abi_sequence'] = _receipt['abi_sequence'];
                             delete _receipt['abi_sequence'];
-                            delete _receipt['act_digest'];
                             _trace['receipts'].push(_receipt);
                         }
-                        _trace['notified'] = [...notifiedSet];
+                        // _trace['notified'] = [...notifiedSet];
                         delete act_digests[_trace.receipt.act_digest];
                         delete _trace['receipt'];
                         delete _trace['receiver'];
@@ -514,10 +547,12 @@ export default class DSPoolWorker extends HyperionWorker {
                 _trace['code_sequence'] = _trace['receipt'].code_sequence;
                 _trace['abi_sequence'] = _trace['receipt'].abi_sequence;
                 _trace['act_digest'] = _trace['receipt'].act_digest;
-                _trace['notified'] = [_trace['receipt'].receiver];
+
+                // notified array is not required since receipts.receiver can be indexed directly
+                // _trace['notified'] = [_trace['receipt'].receiver];
+
                 delete _trace['receipt']['code_sequence'];
                 delete _trace['receipt']['abi_sequence'];
-                delete _trace['receipt']['act_digest'];
                 _trace['receipts'] = [_trace['receipt']];
                 delete _trace['receipt'];
                 _finalTraces.push(_trace);
@@ -528,6 +563,7 @@ export default class DSPoolWorker extends HyperionWorker {
             const redisPayload = new Map<string, IORedis.ValueType>();
 
             for (const uniqueAction of _finalTraces) {
+                cleanActionTrace(uniqueAction);
                 const payload = Buffer.from(flatstr(JSON.stringify(uniqueAction)));
                 redisPayload.set(uniqueAction.global_sequence.toString(), payload);
                 this.actionDsCounter++;
@@ -566,20 +602,24 @@ export default class DSPoolWorker extends HyperionWorker {
 
     pushToActionStreamingQueue(payload, uniqueAction) {
         if (this.allowStreaming && this.conf.features['streaming'].traces) {
-            const notifArray = new Set();
-            uniqueAction.act.authorization.forEach(auth => {
-                notifArray.add(auth.actor);
-            });
-            uniqueAction.notified.forEach(acc => {
-                notifArray.add(acc);
-            });
-            const headers = {
-                event: 'trace',
-                account: uniqueAction['act']['account'],
-                name: uniqueAction['act']['name'],
-                notified: [...notifArray].join(",")
-            };
-            this.ch.publish('', this.chain + ':stream', payload, {headers});
+            try {
+                const notifArray = new Set();
+                uniqueAction.act.authorization.forEach(auth => {
+                    notifArray.add(auth.actor);
+                });
+                uniqueAction.receipts.forEach(rec => {
+                    notifArray.add(rec.receiver);
+                });
+                const headers = {
+                    event: 'trace',
+                    account: uniqueAction['act']['account'],
+                    name: uniqueAction['act']['name'],
+                    notified: [...notifArray].join(",")
+                };
+                this.ch.publish('', this.chain + ':stream', payload, {headers});
+            } catch (e) {
+                hLog(e);
+            }
         }
     }
 

workers/ws-router.ts

@@ -8,429 +8,436 @@ const greylist = ['eosio.token'];
 
 export default class WSRouter extends HyperionWorker {
 
-	q: string;
-	totalRoutedMessages = 0;
-	firstData = false;
-	relays = {};
-	clientIndex = new Map();
-	codeActionMap = new Map();
-	notifiedMap = new Map();
-	codeDeltaMap = new Map();
-	payerMap = new Map();
-	activeRequests = new Map();
-	private io: Server;
-	private totalClients = 0;
+    q: string;
+    totalRoutedMessages = 0;
+    firstData = false;
+    relays = {};
+    clientIndex = new Map();
+    codeActionMap = new Map();
+    notifiedMap = new Map();
+    codeDeltaMap = new Map();
+    payerMap = new Map();
+    activeRequests = new Map();
+    private io: Server;
+    private totalClients = 0;
 
-	constructor() {
-		super();
-		this.q = this.chain + ':stream';
-		this.activeRequests.set('*', {
-			sockets: []
-		});
-	}
+    constructor() {
+        super();
+        this.q = this.chain + ':stream';
+        this.activeRequests.set('*', {
+            sockets: []
+        });
+    }
 
-	assertQueues(): void {
-		this.ch.assertQueue(this.q);
-		this.ch.consume(this.q, this.onConsume.bind(this));
-	}
+    assertQueues(): void {
+        this.ch.assertQueue(this.q);
+        this.ch.consume(this.q, this.onConsume.bind(this));
+    }
 
 
-	onIpcMessage(msg: any): void {
-		switch (msg.event) {
-			case 'lib_update': {
-				this.io.emit('lib_update', {
-					chain_id: this.manager.conn.chains[this.chain]?.chain_id,
-					...msg.data
-				});
-				break;
-			}
-			case 'fork_event': {
-				this.io.emit('fork_event', {
-					chain_id: this.manager.conn.chains[this.chain]?.chain_id,
-					...msg.data
-				});
-				break;
-			}
-		}
-	}
+    onIpcMessage(msg: any): void {
+        switch (msg.event) {
+            case 'lib_update': {
+                this.io.emit('lib_update', {
+                    chain_id: this.manager.conn.chains[this.chain]?.chain_id,
+                    ...msg.data
+                });
+                break;
+            }
+            case 'fork_event': {
+                this.io.emit('fork_event', {
+                    chain_id: this.manager.conn.chains[this.chain]?.chain_id,
+                    ...msg.data
+                });
+                break;
+            }
+        }
+    }
 
-	async run(): Promise<void> {
-		this.initRoutingServer();
-		this.startRoutingRateMonitor();
-		return undefined;
-	}
+    async run(): Promise<void> {
+        this.initRoutingServer();
+        this.startRoutingRateMonitor();
+        return undefined;
+    }
 
-	onConsume(msg) {
-		if (!this.firstData) {
-			this.firstData = true;
-		}
-		switch (msg.properties.headers.event) {
-			case 'trace': {
-				const actHeader = msg.properties.headers;
-				const code = actHeader.account;
-				const name = actHeader.name;
-				const notified = actHeader.notified.split(',');
-				let decodedMsg;
+    onConsume(msg) {
+        if (!this.firstData) {
+            this.firstData = true;
+        }
 
-				// send to contract subscribers
-				if (this.codeActionMap.has(code)) {
-					const codeReq = this.codeActionMap.get(code);
-					decodedMsg = Buffer.from(msg.content).toString();
+        // push to plugin handlers
+        this.mLoader.processStreamEvent(msg);
 
-					// send to action subscribers
-					if (codeReq.has(name)) {
-						for (const link of codeReq.get(name).links) {
-							this.forwardActionMessage(decodedMsg, link, notified);
-						}
-					}
-					// send to wildcard subscribers
-					if (codeReq.has("*")) {
-						for (const link of codeReq.get("*").links) {
-							this.forwardActionMessage(decodedMsg, link, notified);
-						}
-					}
-				}
+        switch (msg.properties.headers.event) {
+            case 'trace': {
+                const actHeader = msg.properties.headers;
+                const code = actHeader.account;
+                const name = actHeader.name;
+                const notified = actHeader.notified.split(',');
+                let decodedMsg;
 
-				// send to notification subscribers
-				notified.forEach((acct) => {
-					if (this.notifiedMap.has(acct)) {
-						if (!decodedMsg) {
-							decodedMsg = Buffer.from(msg.content).toString();
-						}
-						for (const link of this.notifiedMap.get(acct).links) {
-							this.forwardActionMessage(decodedMsg, link, notified);
-						}
-					}
-				});
-				break;
-			}
+                // send to contract subscribers
+                if (this.codeActionMap.has(code)) {
+                    const codeReq = this.codeActionMap.get(code);
+                    decodedMsg = Buffer.from(msg.content).toString();
 
-			case 'delta': {
-				const deltaHeader = msg.properties.headers;
-				const code = deltaHeader.code;
-				const table = deltaHeader.table;
-				// const scope = deltaHeader.scope;
-				const payer = deltaHeader.payer;
-				// console.log(code, table, scope, payer);
-				let decodedDeltaMsg;
-				// Forward to CODE/TABLE listeners
-				if (this.codeDeltaMap.has(code)) {
-					decodedDeltaMsg = Buffer.from(msg.content).toString();
+                    // send to action subscribers
+                    if (codeReq.has(name)) {
+                        for (const link of codeReq.get(name).links) {
+                            this.forwardActionMessage(decodedMsg, link, notified);
+                        }
+                    }
+                    // send to wildcard subscribers
+                    if (codeReq.has("*")) {
+                        for (const link of codeReq.get("*").links) {
+                            this.forwardActionMessage(decodedMsg, link, notified);
+                        }
+                    }
+                }
 
-					const tableDeltaMap = this.codeDeltaMap.get(code);
-					// Send specific table
-					if (tableDeltaMap.has(table)) {
-						for (const link of tableDeltaMap.get(table).links) {
-							this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
-						}
-					}
-					// Send any table
-					if (tableDeltaMap.has("*")) {
-						for (const link of tableDeltaMap.get("*").links) {
-							this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
-						}
-					}
-				}
-				// Forward to PAYER listeners
-				if (this.payerMap.has(payer)) {
-					decodedDeltaMsg = Buffer.from(msg.content).toString();
-					for (const link of this.payerMap.get(payer).links) {
-						this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
-					}
-				}
-				break;
-			}
+                // send to notification subscribers
+                notified.forEach((acct) => {
+                    if (this.notifiedMap.has(acct)) {
+                        if (!decodedMsg) {
+                            decodedMsg = Buffer.from(msg.content).toString();
+                        }
+                        for (const link of this.notifiedMap.get(acct).links) {
+                            this.forwardActionMessage(decodedMsg, link, notified);
+                        }
+                    }
+                });
+                break;
+            }
 
-			default: {
-				console.log('Unindentified message!');
-				console.log(msg);
-			}
-		}
-		this.ch.ack(msg);
-	}
+            case 'delta': {
+                const deltaHeader = msg.properties.headers;
+                const code = deltaHeader.code;
+                const table = deltaHeader.table;
+                // const scope = deltaHeader.scope;
+                const payer = deltaHeader.payer;
+                // console.log(code, table, scope, payer);
+                let decodedDeltaMsg;
+                // Forward to CODE/TABLE listeners
+                if (this.codeDeltaMap.has(code)) {
+                    decodedDeltaMsg = Buffer.from(msg.content).toString();
 
-	startRoutingRateMonitor() {
-		setInterval(() => {
-			if (this.totalRoutedMessages > 0) {
-				hLog('[Router] Routing rate: ' + (this.totalRoutedMessages / 20) + ' msg/s');
-				this.totalRoutedMessages = 0;
-			}
-		}, 20000);
-	}
+                    const tableDeltaMap = this.codeDeltaMap.get(code);
+                    // Send specific table
+                    if (tableDeltaMap.has(table)) {
+                        for (const link of tableDeltaMap.get(table).links) {
+                            this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
+                        }
+                    }
+                    // Send any table
+                    if (tableDeltaMap.has("*")) {
+                        for (const link of tableDeltaMap.get("*").links) {
+                            this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
+                        }
+                    }
+                }
+                // Forward to PAYER listeners
+                if (this.payerMap.has(payer)) {
+                    decodedDeltaMsg = Buffer.from(msg.content).toString();
+                    for (const link of this.payerMap.get(payer).links) {
+                        this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
+                    }
+                }
+                break;
+            }
 
-	countClients() {
-		let total = 0;
-		for (let key in this.relays) {
-			if (this.relays.hasOwnProperty(key)) {
-				if (this.relays[key].connected) {
-					total += this.relays[key].clients;
-				}
-			}
-		}
-		this.totalClients = total;
-		hLog('Total WS clients:', this.totalClients);
-	}
+            default: {
+                console.log('Unindentified message!');
+                console.log(msg);
+            }
+        }
+        this.ch.ack(msg);
+    }
 
-	appendToL1Map(target, primary, link) {
-		if (target.has(primary)) {
-			target.get(primary).links.push(link);
-		} else {
-			target.set(primary, {links: [link]});
-		}
-	}
+    startRoutingRateMonitor() {
+        setInterval(() => {
+            if (this.totalRoutedMessages > 0) {
+                hLog('[Router] Routing rate: ' + (this.totalRoutedMessages / 20) + ' msg/s');
+                this.totalRoutedMessages = 0;
+            }
+        }, 20000);
+    }
 
-	appendToL2Map(target, primary, secondary, link) {
-		if (target.has(primary)) {
-			const pMap = target.get(primary);
-			if (pMap.has(secondary)) {
-				const pLinks = pMap.get(secondary);
-				pLinks.links.push(link);
-			} else {
-				pMap.set(secondary, {
-					links: [link]
-				});
-			}
-		} else {
-			const sMap = new Map();
-			sMap.set(secondary, {
-				links: [link]
-			});
-			target.set(primary, sMap);
-		}
-	}
+    countClients() {
+        let total = 0;
+        for (let key in this.relays) {
+            if (this.relays.hasOwnProperty(key)) {
+                if (this.relays[key].connected) {
+                    total += this.relays[key].clients;
+                }
+            }
+        }
+        this.totalClients = total;
+        hLog('Total WS clients:', this.totalClients);
+    }
 
-	addActionRequest(data, id) {
-		const req = data.request;
-		if (typeof req.account !== 'string') {
-			return {status: 'FAIL', reason: 'invalid request'};
-		}
-		if (greylist.indexOf(req.contract) !== -1) {
-			if (req.account === '' || req.account === req.contract) {
-				return {
-					status: 'FAIL',
-					reason: 'request too broad, please be more specific'
-				};
-			}
-		}
-		const link = {
-			type: 'action',
-			relay: id,
-			client: data.client_socket,
-			filters: req.filters,
-			account: req.account,
-			added_on: Date.now()
-		};
-		if (req.contract !== '' && req.contract !== '*') {
-			this.appendToL2Map(this.codeActionMap, req.contract, req.action, link);
-		} else {
-			if (req.account !== '') {
-				this.appendToL1Map(this.notifiedMap, req.account, link);
-			} else {
-				return {status: 'FAIL', reason: 'invalid request'};
-			}
-		}
-		this.addToClientIndex(data, id, [req.contract, req.action, req.account]);
-		return {
-			status: 'OK'
-		};
-	}
+    appendToL1Map(target, primary, link) {
+        if (target.has(primary)) {
+            target.get(primary).links.push(link);
+        } else {
+            target.set(primary, {links: [link]});
+        }
+    }
 
-	addToClientIndex(data, id, path) {
-		// register client on index
-		if (this.clientIndex.has(data.client_socket)) {
-			this.clientIndex.get(data.client_socket).set(id, path);
-			// console.log('new relay added to existing client');
-		} else {
-			const list = new Map();
-			list.set(id, path);
-			this.clientIndex.set(data.client_socket, list);
-			// console.log('new client added to index');
-		}
-	}
+    appendToL2Map(target, primary, secondary, link) {
+        if (target.has(primary)) {
+            const pMap = target.get(primary);
+            if (pMap.has(secondary)) {
+                const pLinks = pMap.get(secondary);
+                pLinks.links.push(link);
+            } else {
+                pMap.set(secondary, {
+                    links: [link]
+                });
+            }
+        } else {
+            const sMap = new Map();
+            sMap.set(secondary, {
+                links: [link]
+            });
+            target.set(primary, sMap);
+        }
+    }
 
-	addDeltaRequest(data, id) {
-		const req = data.request;
-		const link = {
-			type: 'delta',
-			relay: id,
-			client: data.client_socket,
-			filters: data.request.filters,
-			payer: data.request.payer,
-			added_on: Date.now()
-		};
-		if (req.code !== '' && req.code !== '*') {
-			this.appendToL2Map(this.codeDeltaMap, req.code, req.table, link);
-		} else {
-			if (req.payer !== '' && req.payer !== '*') {
-				this.appendToL1Map(this.payerMap, req.payer, link);
-			} else {
-				return {status: 'FAIL', reason: 'invalid request'};
-			}
-		}
-		this.addToClientIndex(data, id, [req.code, req.table, req.payer]);
-		return {
-			status: 'OK'
-		};
-	}
+    addActionRequest(data, id) {
+        const req = data.request;
+        if (typeof req.account !== 'string') {
+            return {status: 'FAIL', reason: 'invalid request'};
+        }
+        if (greylist.indexOf(req.contract) !== -1) {
+            if (req.account === '' || req.account === req.contract) {
+                return {
+                    status: 'FAIL',
+                    reason: 'request too broad, please be more specific'
+                };
+            }
+        }
+        const link = {
+            type: 'action',
+            relay: id,
+            client: data.client_socket,
+            filters: req.filters,
+            account: req.account,
+            added_on: Date.now()
+        };
+        if (req.contract !== '' && req.contract !== '*') {
+            this.appendToL2Map(this.codeActionMap, req.contract, req.action, link);
+        } else {
+            if (req.account !== '') {
+                this.appendToL1Map(this.notifiedMap, req.account, link);
+            } else {
+                return {status: 'FAIL', reason: 'invalid request'};
+            }
+        }
+        this.addToClientIndex(data, id, [req.contract, req.action, req.account]);
+        return {
+            status: 'OK'
+        };
+    }
 
-	removeDeepLinks(map, path, key, id) {
-		if (map.has(path[0])) {
-			if (map.get(path[0]).has(path[1])) {
-				const currentLinks = map.get(path[0]).get(path[1]).links;
-				currentLinks.forEach((item, index) => {
-					if (item.relay === key && item.client === id) {
-						currentLinks.splice(index, 1);
-					}
-				});
-			}
-		}
-	}
+    addToClientIndex(data, id, path) {
+        // register client on index
+        if (this.clientIndex.has(data.client_socket)) {
+            this.clientIndex.get(data.client_socket).set(id, path);
+            // console.log('new relay added to existing client');
+        } else {
+            const list = new Map();
+            list.set(id, path);
+            this.clientIndex.set(data.client_socket, list);
+            // console.log('new client added to index');
+        }
+    }
 
-	removeSingleLevelLinks(map, path, key, id) {
-		if (map.has(path[2])) {
-			const _links = map.get(path[2]).links;
-			_links.forEach((item, index) => {
-				if (item.relay === key && item.client === id) {
-					_links.splice(index, 1);
-				}
-			});
-		}
-	}
+    addDeltaRequest(data, id) {
+        const req = data.request;
+        const link = {
+            type: 'delta',
+            relay: id,
+            client: data.client_socket,
+            filters: data.request.filters,
+            payer: data.request.payer,
+            added_on: Date.now()
+        };
+        if (req.code !== '' && req.code !== '*') {
+            this.appendToL2Map(this.codeDeltaMap, req.code, req.table, link);
+        } else {
+            if (req.payer !== '' && req.payer !== '*') {
+                this.appendToL1Map(this.payerMap, req.payer, link);
+            } else {
+                return {status: 'FAIL', reason: 'invalid request'};
+            }
+        }
+        this.addToClientIndex(data, id, [req.code, req.table, req.payer]);
+        return {
+            status: 'OK'
+        };
+    }
 
-	removeLinks(id) {
-		// console.log(`Removing links for ${id}...`);
-		if (this.clientIndex.has(id)) {
-			const links = this.clientIndex.get(id);
-			links.forEach((path, key) => {
-				this.removeDeepLinks(this.codeActionMap, path, key, id);
-				this.removeDeepLinks(this.codeDeltaMap, path, key, id);
-				this.removeSingleLevelLinks(this.notifiedMap, path, key, id);
-				this.removeSingleLevelLinks(this.payerMap, path, key, id);
-			});
-		}
-	}
+    removeDeepLinks(map, path, key, id) {
+        if (map.has(path[0])) {
+            if (map.get(path[0]).has(path[1])) {
+                const currentLinks = map.get(path[0]).get(path[1]).links;
+                currentLinks.forEach((item, index) => {
+                    if (item.relay === key && item.client === id) {
+                        currentLinks.splice(index, 1);
+                    }
+                });
+            }
+        }
+    }
 
-	initRoutingServer() {
-		const server = createServer();
-		this.io = new Server(server, {
-			path: '/router',
-			serveClient: false,
-			cookie: false
-		});
+    removeSingleLevelLinks(map, path, key, id) {
+        if (map.has(path[2])) {
+            const _links = map.get(path[2]).links;
+            _links.forEach((item, index) => {
+                if (item.relay === key && item.client === id) {
+                    _links.splice(index, 1);
+                }
+            });
+        }
+    }
 
-		this.io.on('connection', (socket: Socket) => {
-			console.log(`[ROUTER] New relay connected with ID = ${socket.id}`);
-			this.relays[socket.id] = {clients: 0, connected: true};
-			socket.on('event', (data, callback) => {
-				switch (data.type) {
-					case 'client_count': {
-						this.relays[socket.id]['clients'] = data.counter;
-						this.countClients();
-						break;
-					}
-					case 'action_request': {
-						const result = this.addActionRequest(data, socket.id);
-						if (result.status === 'OK') {
-							callback(result);
-						} else {
-							callback(result.reason);
-						}
-						break;
-					}
-					case 'delta_request': {
-						const result = this.addDeltaRequest(data, socket.id);
-						if (result.status === 'OK') {
-							callback(result);
-						} else {
-							callback(result.reason);
-						}
-						break;
-					}
-					case 'client_disconnected': {
-						this.removeLinks(data.id);
-						break;
-					}
-					default: {
-						console.log(data);
-					}
-				}
-			});
-			socket.on('disconnect', () => {
-				this.relays[socket.id].connected = false;
-				this.countClients();
-			});
-		});
+    removeLinks(id) {
+        // console.log(`Removing links for ${id}...`);
+        if (this.clientIndex.has(id)) {
+            const links = this.clientIndex.get(id);
+            links.forEach((path, key) => {
+                this.removeDeepLinks(this.codeActionMap, path, key, id);
+                this.removeDeepLinks(this.codeDeltaMap, path, key, id);
+                this.removeSingleLevelLinks(this.notifiedMap, path, key, id);
+                this.removeSingleLevelLinks(this.payerMap, path, key, id);
+            });
+        }
+    }
 
-		const connOpts = this.manager.conn.chains[this.chain];
+    /*
+    WS Routing server, API will connect to it.
+     */
+    initRoutingServer() {
+        const server = createServer();
+        this.io = new Server(server, {
+            path: '/router',
+            serveClient: false,
+            cookie: false
+        });
 
-		let _port = 57200;
-		if (connOpts.WS_ROUTER_PORT) {
-			_port = connOpts.WS_ROUTER_PORT;
-		}
+        this.io.on('connection', (socket: Socket) => {
+            console.log(`[ROUTER] New relay connected with ID = ${socket.id}`);
+            this.relays[socket.id] = {clients: 0, connected: true};
+            socket.on('event', (data, callback) => {
+                switch (data.type) {
+                    case 'client_count': {
+                        this.relays[socket.id]['clients'] = data.counter;
+                        this.countClients();
+                        break;
+                    }
+                    case 'action_request': {
+                        const result = this.addActionRequest(data, socket.id);
+                        if (result.status === 'OK') {
+                            callback(result);
+                        } else {
+                            callback(result.reason);
+                        }
+                        break;
+                    }
+                    case 'delta_request': {
+                        const result = this.addDeltaRequest(data, socket.id);
+                        if (result.status === 'OK') {
+                            callback(result);
+                        } else {
+                            callback(result.reason);
+                        }
+                        break;
+                    }
+                    case 'client_disconnected': {
+                        this.removeLinks(data.id);
+                        break;
+                    }
+                    default: {
+                        console.log(data);
+                    }
+                }
+            });
+            socket.on('disconnect', () => {
+                this.relays[socket.id].connected = false;
+                this.countClients();
+            });
+        });
 
-		let _host = "127.0.0.1";
-		if (connOpts.WS_ROUTER_HOST) {
-			_host = connOpts.WS_ROUTER_HOST;
-		}
+        const connOpts = this.manager.conn.chains[this.chain];
 
-		server.listen(_port, _host, () => {
-			this.ready();
-			setTimeout(() => {
-				if (!this.firstData) {
-					this.ready();
-				}
-			}, 5000);
-		});
+        let _port = 57200;
+        if (connOpts.WS_ROUTER_PORT) {
+            _port = connOpts.WS_ROUTER_PORT;
+        }
 
-	}
+        let _host = "127.0.0.1";
+        if (connOpts.WS_ROUTER_HOST) {
+            _host = connOpts.WS_ROUTER_HOST;
+        }
 
-	ready() {
-		process.send({event: 'router_ready'});
-	}
+        server.listen(_port, _host, () => {
+            this.ready();
+            setTimeout(() => {
+                if (!this.firstData) {
+                    this.ready();
+                }
+            }, 5000);
+        });
 
-	private forwardActionMessage(msg: any, link: any, notified: string[]) {
-		let allow = false;
-		const relay = this.io.of('/').sockets.get(link.relay);
-		if (relay) {
-			if (link.account !== '') {
-				allow = notified.indexOf(link.account) !== -1;
-			} else {
-				allow = true;
-			}
-			if (link.filters?.length > 0) {
-				// check filters
-				const _parsedMsg = JSON.parse(msg);
-				allow = link.filters.every(filter => {
-					return checkFilter(filter, _parsedMsg);
-				});
-			}
-			if (allow) {
-				relay.emit('trace', {client: link.client, message: msg});
-				this.totalRoutedMessages++;
-			}
-		}
-	}
+    }
 
-	private forwardDeltaMessage(msg, link, payer) {
-		let allow = false;
-		const relay = this.io.of('/').sockets.get(link.relay);
-		if (relay) {
-			if (link.payer !== '') {
-				allow = link.payer === payer;
-			} else {
-				allow = true;
-			}
-			// if (link.filters?.length > 0) {
-			//     // check filters
-			//     const _parsedMsg = JSON.parse(msg);
-			//     allow = link.filters.every(filter => {
-			//         return checkDeltaFilter(filter, _parsedMsg);
-			//     });
-			// }
-			if (allow) {
-				relay.emit('delta', {client: link.client, message: msg});
-				this.totalRoutedMessages++;
-			}
-		}
-	}
+    ready() {
+        process.send({event: 'router_ready'});
+    }
+
+    private forwardActionMessage(msg: any, link: any, notified: string[]) {
+        let allow = false;
+        const relay = this.io.of('/').sockets.get(link.relay);
+        if (relay) {
+            if (link.account !== '') {
+                allow = notified.indexOf(link.account) !== -1;
+            } else {
+                allow = true;
+            }
+            if (link.filters?.length > 0) {
+                // check filters
+                const _parsedMsg = JSON.parse(msg);
+                allow = link.filters.every(filter => {
+                    return checkFilter(filter, _parsedMsg);
+                });
+            }
+            if (allow) {
+                relay.emit('trace', {client: link.client, message: msg});
+                this.totalRoutedMessages++;
+            }
+        }
+    }
+
+    private forwardDeltaMessage(msg, link, payer) {
+        let allow = false;
+        const relay = this.io.of('/').sockets.get(link.relay);
+        if (relay) {
+            if (link.payer !== '') {
+                allow = link.payer === payer;
+            } else {
+                allow = true;
+            }
+            // if (link.filters?.length > 0) {
+            //     // check filters
+            //     const _parsedMsg = JSON.parse(msg);
+            //     allow = link.filters.every(filter => {
+            //         return checkDeltaFilter(filter, _parsedMsg);
+            //     });
+            // }
+            if (allow) {
+                relay.emit('delta', {client: link.client, message: msg});
+                this.totalRoutedMessages++;
+            }
+        }
+    }
 }