implement tx caching on v1 get_transaction

- update ws to v8
- add indexer.disable_delta_rm option

.gitignore

@@ -31,6 +31,7 @@ node_modules/
 
 .idea
 *.pem
+ecosystem.config.js
 package-lock.json
 connections.json
 connections.json2
@@ -68,18 +69,13 @@ helpers/**/*.js.map
 interfaces/**/*.js
 interfaces/**/*.js.map
 
-#modules/**/*.js
-#modules/**/*.js.map
+modules/**/*.js
+modules/**/*.js.map
 
 addons/**/*.js
 addons/**/*.js.map
 
-scripts/hpm.js
-scripts/hpm.js.map
-
-plugins/.state.json
 plugins/repos
-plugins/*.js
 .gitmodules
 
 docker/redis/data
@@ -87,13 +83,3 @@ docker/elasticsearch/data
 docker/rabbitmq/data
 docker/eosio/data
 
-configuration_backups
-
-scripts/hyp-config.js
-scripts/hyp-config.js.map
-
-scripts/hyp-repair.js
-scripts/hyp-repair.js.map
-
-scripts/repair-cli/*.js
-scripts/repair-cli/*.js.map

README.md

@@ -1,99 +1,54 @@
-<!--suppress HtmlUnknownTarget, HtmlDeprecatedAttribute -->
-<br></br>
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="https://eosrio.io/hyperion-white.png">
-    <img alt="Hyperion Logo"
-         src="https://eosrio.io/hyperion.png">
-  </picture>
-</p>
+# Hyperion History API
 
-<h4 align="center">
-    Scalable Full History API Solution for 
-    <a href="https://antelope.io">
-        Antelope
-    </a>
-    (former EOSIO) based blockchains
-</h4>
+<img alt="hyperion logo" height="64" src="https://eosrio.io/hyperion.png">
+<br/>
+Scalable Full History API Solution for EOSIO based blockchains
 
-<br>
+Made with ♥ by [EOS Rio](https://eosrio.io/)
 
-Made with ♥ by [Rio Blocks / EOS Rio](https://rioblocks.io/?lang=en)
-
-### [Official documentation](https://hyperion.docs.eosrio.io)
-
-### How to use
-
- - [For Providers](https://hyperion.docs.eosrio.io/manual_installation/)
-
- - [For Developers](https://hyperion.docs.eosrio.io/howtouse/)
-
-### Official plugins:
-
-- [Hyperion Lightweight Explorer](https://github.com/eosrio/hyperion-explorer-plugin)
+Official documentation: https://hyperion.docs.eosrio.io/
 
 ### 1. Overview
 
-Hyperion is a full history solution for indexing, storing and retrieving Antelope blockchain's historical data.
-Antelope protocol is highly scalable reaching up to tens of thousands of transactions per second demanding high
-performance indexing and optimized storage and querying solutions. Hyperion is developed to tackle those challenges
-providing open source software to be operated by block producers, infrastructure providers and dApp developers.
+Hyperion is a full history solution for indexing, storing and retrieving EOSIO blockchain`s historical data. EOSIO protocol is highly scalable reaching up to tens of thousands of transactions per second demanding high performance indexing and optimized storage and querying solutions. Hyperion is developed to tackle those challenges providing open source software to be operated by block producers, infrastructure providers and dApp developers.
+
+Focused on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers, Hyperion implements an improved data structure
+actions are stored in a flattened format
+a parent field is added to the inline actions to point to the parent global sequence
+if the inline action data is identical to the parent it is considered a notification and thus removed from the database
+no blocks or transaction data is stored, all information can be reconstructed from actions
 
-Focused on delivering faster search times, lower bandwidth overhead and easier usability for UI/UX developers,
-Hyperion implements an improved data structure. Actions are stored in a flattened format, transaction ids are added to 
-all inline actions, allowing to group by transaction without storing a full transaction index. Besides that if the inline 
-action data is identical to the parent, it is considered a notification and thus removed from the database.
-No full block or transaction data is stored, all information can be reconstructed from actions and deltas, only a block 
-header index is stored.
 
 ### 2. Architecture
-
-The following components are required in order to have a fully functional Hyperion API deployment.
-* For small use cases, it is absolutely fine to run all components on a single machine.
-* For larger chains and production environments, we recommend setting them up into different servers under a high-speed local network.
+The following components are required in order to have a fully functional Hyperion API deployment, 
+for small use cases its absolutely fine to run all components on a single machine. For larger chains and 
+production environments we recommend setting them up into different servers under a high-speed local network.
 
 #### 2.1 Elasticsearch Cluster
-
 The ES cluster is responsible for storing all indexed data.
-Direct access to the Hyperion API and Indexer must be provided. We recommend nodes in the
-cluster to have at least 32 GB of RAM and 8 cpu cores. SSD/NVME drives are recommended for
-maximum indexing throughput, although HDDs can be used for cold storage nodes.
-For production environments, a multi-node cluster is highly recommended.
+Direct access to the Hyperion API and Indexer must be provided. We recommend nodes in the 
+cluster to have at least 32 GB of RAM and 8 cpu cores. SSD/NVME drives are recommended for 
+maximum indexing throughput. For production environments a multi-node cluster is highly recommended.
 
 #### 2.2 Hyperion Indexer
-
 The Indexer is a Node.js based app that process data from the state history plugin and allows it to be indexed.
-The [PM2 process manager](https://pm2.keymetrics.io) is used to launch and operate the indexer. The configuration
-flexibility is very extensive,
-so system recommendations will depend on the use case and data load. It will require access to at least one ES node,
+The PM2 process manager is used to launch and operate the indexer. The configuration flexibility is very extensive, 
+so system recommendations will depend on the use case and data load. It will require access to at least one ES node, 
 RabbitMQ and the state history node.
 
 #### 2.3 Hyperion API
-
 Parallelizable API server that provides the V2 and V1 (legacy history plugin) endpoints.
-It is launched by PM2 and can also operate in cluster mode. It requires direct access to
+It is launched by PM2 and can also operate in cluster mode. It requires direct access to 
 at least one ES node for the queries and all other services for full healthcheck
 
 #### 2.4 RabbitMQ
+Use as messaging queue and data transport between the indexer stages
 
-Used as messaging queue and data transport between the indexer stages and for real-time data streaming
+#### 2.5 EOSIO State History
+Nodeos plugin used to collect action traces and state deltas. Provides data via websocket to the indexer
 
-#### 2.5 Redis
+### 3. How to use
 
-Used for transient data storage across processes and for the preemptive transaction caching used on
-the `v2/history/get_transaction` and `v2/history/check_transaction` endpoints
+#### 3.1 [For Providers](https://eosrio.github.io/hyperion-docs/quickstart/)
 
-#### 2.6 Leap State History
-
-[Leap / Nodeos](https://github.com/AntelopeIO/leap/tree/main/plugins/state_history_plugin) plugin used
-to collect action traces and state deltas. Provides data via websocket to the indexer
-
-#### 2.7 Hyperion Stream Client (optional)
-
-Web and Node.js client for real-time streaming on enabled hyperion
-providers. [Documentation](https://hyperion.docs.eosrio.io/dev/stream_client/)
-
-#### 2.8 Hyperion Plugins (optional)
-
-Hyperion includes a flexible plugin architecture to allow further customization.
-Plugins are managed by the `hpm` (hyperion plugin manager) command line tool.
+#### 3.2 [For Developers](https://eosrio.github.io/hyperion-docs/howtouse/)

api/config/open_api.ts

@@ -4,7 +4,6 @@ export function generateOpenApiConfig(config: HyperionConfig) {
     const packageData = require('../../package');
     const health_link = `https://${config.api.server_name}/v2/health`;
     const explorer_link = `https://${config.api.server_name}/v2/explore`;
-
     let description = `
 <img height="64" src="https://eosrio.io/hyperion.png">
 ### Scalable Full History API Solution for EOSIO based blockchains
@@ -14,15 +13,9 @@ export function generateOpenApiConfig(config: HyperionConfig) {
 #### Provided by [${config.api.provider_name}](${config.api.provider_url})
 #### Health API: <a target="_blank" href="${health_link}">${health_link}</a>
 `;
-
-    if (config.plugins) {
-        if (config.plugins.explorer) {
-            if (config.plugins.explorer.enabled) {
-                description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
-            }
-        }
+    if(config.api.enable_explorer) {
+        description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
     }
-
     return {
         routePrefix: '/v2/docs',
         exposeRoute: true,

api/helpers/cacheManager.ts

@@ -1,86 +0,0 @@
-import {HyperionConfig} from "../../interfaces/hyperionConfig";
-import {createHash} from "crypto";
-import {FastifyRequest} from "fastify";
-
-export interface CacheConfig {
-    ttl: number;
-}
-
-export interface CachedEntry {
-    data: string;
-    exp: number;
-}
-
-export class CacheManager {
-
-    v1CacheConfigs: Map<string, CacheConfig> = new Map<string, CacheConfig>();
-    v1Caches: Map<string, Map<string, CachedEntry>> = new Map<string, Map<string, CachedEntry>>();
-
-    constructor(conf: HyperionConfig) {
-        if (conf.api.v1_chain_cache) {
-            conf.api.v1_chain_cache.forEach(value => {
-                this.v1CacheConfigs.set(value.path, {
-                    ttl: value.ttl
-                });
-                this.v1Caches.set(value.path, new Map<string, CachedEntry>());
-            });
-        }
-        setInterval(() => {
-            try {
-                // remove expired entries
-                // let removeCount = 0;
-                const now = Date.now();
-                this.v1Caches.forEach((pathCacheMap: Map<string, CachedEntry>, pathKey: string) => {
-                    pathCacheMap.forEach((cache: CachedEntry, entryKey: string, map: Map<string, CachedEntry>) => {
-                        if (cache.exp < now) {
-                            map.delete(entryKey);
-                            // removeCount++;
-                        }
-                    });
-                });
-                // if (removeCount > 0) {
-                //     console.log(`${removeCount} expired cache entries removed`);
-                // }
-            } catch (e) {
-                console.log(e);
-            }
-        }, 5000);
-    }
-
-    hashPayload(input: string): string {
-        return createHash('sha256').update(input).digest().toString('hex');
-    }
-
-    setCachedData(hash: string, path: string, payload: any): void {
-        if (this.v1CacheConfigs.has(path) && this.v1Caches.has(path)) {
-            this.v1Caches.get(path).set(hash, {
-                data: payload,
-                exp: this.v1CacheConfigs.get(path).ttl + Date.now()
-            });
-        }
-    }
-
-    getCachedData(request: FastifyRequest): [string | null, string, string] {
-        const urlParts = request.url.split("?");
-        const pathComponents = urlParts[0].split('/');
-        const path = pathComponents.at(-1);
-        let payload = '';
-        if (request.method === 'POST') {
-            payload = JSON.stringify(request.body);
-        } else if (request.method === 'GET') {
-            payload = request.url;
-        }
-        const hashedString = this.hashPayload(payload);
-        if (this.v1Caches.has(path)) {
-            const entry = this.v1Caches.get(path).get(hashedString);
-            if (entry && entry.exp > Date.now()) {
-                return [entry.data, hashedString, path];
-            } else {
-                return [null, hashedString, path];
-            }
-        } else {
-            return [null, hashedString, path];
-        }
-    }
-
-}

api/plugins.ts

@@ -3,31 +3,21 @@ import {IncomingMessage, Server, ServerResponse} from "http";
 
 // fastify plugins
 import * as fastifyElasticsearch from 'fastify-elasticsearch';
-import fastifySwagger from '@fastify/swagger';
-import fastifyCors from '@fastify/cors';
-import formBodyPlugin from '@fastify/formbody';
-import fastifyRedis from '@fastify/redis';
-import fastifyRateLimit from '@fastify/rate-limit';
+import fastifySwagger from 'fastify-swagger';
+import fastifyCors from 'fastify-cors';
+import fastifyFormbody from 'fastify-formbody';
+import fastifyRedis from 'fastify-redis';
+import fastifyRateLimit from 'fastify-rate-limit';
 
 // custom plugins
 import fastify_eosjs from "./plugins/fastify-eosjs";
 
 export function registerPlugins(server: Fastify.FastifyInstance<Server, IncomingMessage, ServerResponse>, params: any) {
-    server.register(fastifyElasticsearch, params.fastify_elasticsearch);
-
-    if (params.fastify_swagger) {
-        server.register(fastifySwagger, params.fastify_swagger);
-    }
-
-    server.register(fastifyCors);
-
-    server.register(formBodyPlugin);
-
-    server.register(fastifyRedis, params.fastify_redis);
-
-    if (params.fastify_rate_limit) {
-        server.register(fastifyRateLimit, params.fastify_rate_limit);
-    }
-
-    server.register(fastify_eosjs, params.fastify_eosjs);
+	server.register(fastifyElasticsearch, params.fastify_elasticsearch);
+	server.register(fastifySwagger, params.fastify_swagger);
+	server.register(fastifyCors);
+	server.register(fastifyFormbody);
+	server.register(fastifyRedis, params.fastify_redis);
+	server.register(fastifyRateLimit, params.fastify_rate_limit);
+	server.register(fastify_eosjs, params.fastify_eosjs);
 }

api/plugins/fastify-eosjs.ts

@@ -3,16 +3,17 @@ import {Api} from "eosjs/dist";
 import fp from "fastify-plugin";
 
 export default fp(async (fastify: FastifyInstance, options: FastifyPluginOptions): Promise<void> => {
-    const rpc = fastify.manager.nodeosJsonRPC;
-    const api = new Api({
-        rpc,
-        signatureProvider: null,
-        chainId: options.chain_id,
-        textDecoder: new TextDecoder(),
-        textEncoder: new TextEncoder(),
-    });
-    fastify.decorate('eosjs', {api, rpc});
+	const rpc = fastify.manager.nodeosJsonRPC;
+	const chain_data = await rpc.get_info();
+	const api = new Api({
+		rpc,
+		signatureProvider: null,
+		chainId: chain_data.chain_id,
+		textDecoder: new TextDecoder(),
+		textEncoder: new TextEncoder(),
+	});
+	fastify.decorate('eosjs', {api, rpc});
 }, {
-    fastify: '>=2.0.0',
-    name: 'fastify-eosjs'
+	fastify: '>=2.0.0',
+	name: 'fastify-eosjs'
 });

api/routes.ts

@@ -2,139 +2,118 @@ import {join} from "path";
 import {FastifyError, FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {createReadStream} from "fs";
 import {addSharedSchemas, handleChainApiRedirect} from "./helpers/functions";
-import autoLoad from '@fastify/autoload';
+import autoLoad from 'fastify-autoload';
 import got from "got";
 
 function addRedirect(server: FastifyInstance, url: string, redirectTo: string) {
-    server.route({
-        url,
-        method: 'GET',
-        schema: {
-            hide: true
-        },
-        handler: async (request: FastifyRequest, reply: FastifyReply) => {
-            reply.redirect(redirectTo);
-        }
-    });
+	server.route({
+		url,
+		method: 'GET',
+		schema: {
+			hide: true
+		},
+		handler: async (request: FastifyRequest, reply: FastifyReply) => {
+			reply.redirect(redirectTo);
+		}
+	});
 }
 
 function addRoute(server: FastifyInstance, handlersPath: string, prefix: string) {
-    server.register(autoLoad, {
-        dir: join(__dirname, 'routes', handlersPath),
-        ignorePattern: /.*(handler|schema).js/,
-        dirNameRoutePrefix: false,
-        options: {prefix}
-    });
+	server.register(autoLoad, {
+		dir: join(__dirname, 'routes', handlersPath),
+		ignorePattern: /.*(handler|schema).js/,
+		dirNameRoutePrefix: false,
+		options: {prefix}
+	});
 }
 
 export function registerRoutes(server: FastifyInstance) {
 
-    // build internal map of routes
-    const routeSet = new Set<string>();
-    server.decorate('routeSet', routeSet);
-    const ignoreList = [
-        '/v2',
-        '/v2/history',
-        '/v2/state',
-        '/v1/chain/*',
-        '/v1/chain'
-    ];
-    server.addHook('onRoute', opts => {
-        if (!ignoreList.includes(opts.url)) {
-            if (opts.url.startsWith('/v')) {
-                routeSet.add(opts.url);
-            }
-        }
-    });
+	// build internal map of routes
+	const routeSet = new Set<string>();
+	server.decorate('routeSet', routeSet);
+	const ignoreList = [
+		'/v2',
+		'/v2/history',
+		'/v2/state',
+		'/v1/chain/*',
+		'/v1/chain'
+	];
+	server.addHook('onRoute', opts => {
+		if (!ignoreList.includes(opts.url)) {
+			if (opts.url.startsWith('/v')) {
+				routeSet.add(opts.url);
+			}
+		}
+	});
 
-    // Register fastify api routes
-    addRoute(server, 'v2', '/v2');
-    addRoute(server, 'v2-history', '/v2/history');
-    addRoute(server, 'v2-state', '/v2/state');
-    addRoute(server, 'v2-stats', '/v2/stats');
+	// Register fastify api routes
+	addRoute(server, 'v2', '/v2');
+	addRoute(server, 'v2-history', '/v2/history');
+	addRoute(server, 'v2-state', '/v2/state');
+	addRoute(server, 'v2-stats', '/v2/stats');
 
-    // legacy routes
-    addRoute(server, 'v1-history', '/v1/history');
-    addRoute(server, 'v1-trace', '/v1/trace_api');
+	// legacy routes
+	addRoute(server, 'v1-history', '/v1/history');
+	addRoute(server, 'v1-trace', '/v1/trace_api');
 
-    addSharedSchemas(server);
+	addSharedSchemas(server);
 
-    // chain api redirects
-    addRoute(server, 'v1-chain', '/v1/chain');
+	// chain api redirects
+	addRoute(server, 'v1-chain', '/v1/chain');
+	server.route({
+		url: '/v1/chain/*',
+		method: ["GET", "POST"],
+		schema: {
+			summary: "Wildcard chain api handler",
+			tags: ["chain"]
+		},
+		handler: async (request: FastifyRequest, reply: FastifyReply) => {
+			await handleChainApiRedirect(request, reply, server);
+		}
+	});
 
-    // other v1 requests
-    server.route({
-        url: '/v1/chain/*',
-        method: ["GET", "POST"],
-        schema: {
-            summary: "Wildcard chain api handler",
-            tags: ["chain"]
-        },
-        handler: async (request: FastifyRequest, reply: FastifyReply) => {
-            await handleChainApiRedirect(request, reply, server);
-        }
-    });
+	// /v1/node/get_supported_apis
+	server.route({
+		url: '/v1/node/get_supported_apis',
+		method: ["GET"],
+		schema: {
+			summary: "Get list of supported APIs",
+			tags: ["node"]
+		},
+		handler: async (request: FastifyRequest, reply: FastifyReply) => {
+			const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
+			if (data.apis && data.apis.length > 0) {
+				const apiSet = new Set(server.routeSet);
+				data.apis.forEach((a) => apiSet.add(a));
+				reply.send({apis: [...apiSet]});
+			} else {
+				reply.send({apis: [...server.routeSet], error: 'nodeos did not send any data'});
+			}
+		}
+	});
 
-    // /v1/node/get_supported_apis
-    server.route({
-        url: '/v1/node/get_supported_apis',
-        method: ["GET"],
-        schema: {
-            summary: "Get list of supported APIs",
-            tags: ["node"]
-        },
-        handler: async (request: FastifyRequest, reply: FastifyReply) => {
-            const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
-            if (data.apis && data.apis.length > 0) {
-                const apiSet = new Set(server.routeSet);
-                data.apis.forEach((a) => apiSet.add(a));
-                reply.send({apis: [...apiSet]});
-            } else {
-                reply.send({apis: [...server.routeSet], error: 'nodeos did not send any data'});
-            }
-        }
-    });
+	server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
+		console.log(`[${request.headers['x-real-ip']}] ${request.method} ${request.url} failed with error: ${error.message}`);
+		done();
+	});
 
-    // global onRequest hook to collect hourly statistics for each path on redis
-    server.addHook('onResponse', (request: FastifyRequest, reply: FastifyReply, done) => {
-        const path = request.url.split('?')[0].replace(/\/$/, '');
-        const now = new Date();
-        const ts = now.getTime();
-        // normalized time in hours
-        const factor = 1000 * 60 * 60;
-        const ts_hour_unix = Math.floor(ts / factor) * factor;
-        const key = `stats:${server.manager.chain}:H:${ts_hour_unix}`;
-        // console.log(reply.statusCode, key, new Date(ts_hour_unix));
-        server.redis.multi()
-            .incr(`stats:${server.manager.chain}:total:${reply.statusCode}:${path}`)
-            .zincrby(key, 1, `[${reply.statusCode}]${path}`)
-            .expire(key, 60 * 60 * 24 * 7)
-            .exec()
-            .catch(console.log);
-        done();
-    });
+	if (server.manager.config.features.streaming) {
+		// steam client lib
+		server.get(
+			'/stream-client.js',
+			{schema: {tags: ['internal']}},
+			(request: FastifyRequest, reply: FastifyReply) => {
+				const stream = createReadStream('./hyperion-stream-client.js');
+				reply.type('application/javascript').send(stream);
+			});
+	}
 
-    server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
-        console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
-        done();
-    });
-
-    if (server.manager.config.features.streaming) {
-        // steam client lib
-        server.get(
-            '/stream-client.js',
-            {schema: {tags: ['internal']}},
-            (request: FastifyRequest, reply: FastifyReply) => {
-                const stream = createReadStream('./hyperion-stream-client.js');
-                reply.type('application/javascript').send(stream);
-            });
-    }
-
-    // Redirect routes to documentation
-    addRedirect(server, '/v2', '/v2/docs');
-    addRedirect(server, '/v2/history', '/v2/docs/index.html#/history');
-    addRedirect(server, '/v2/state', '/v2/docs/index.html#/state');
-    addRedirect(server, '/v1/chain', '/v2/docs/index.html#/chain');
-    addRedirect(server, '/explorer', '/v2/explore');
-    addRedirect(server, '/explore', '/v2/explore');
+	// Redirect routes to documentation
+	addRedirect(server, '/v2', '/v2/docs');
+	addRedirect(server, '/v2/history', '/v2/docs/index.html#/history');
+	addRedirect(server, '/v2/state', '/v2/docs/index.html#/state');
+	addRedirect(server, '/v1/chain', '/v2/docs/index.html#/chain');
+	addRedirect(server, '/explorer', '/v2/explore');
+	addRedirect(server, '/explore', '/v2/explore');
 }

api/routes/v1-chain/abi_bin_to_json/index.ts

@@ -9,7 +9,11 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         {
             "code": {$ref: 'AccountName#'},
             "action": {$ref: 'AccountName#'},
-            "binargs": {"type": "string"}
+            "binargs": {
+                "type": "string",
+                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
+                "title": "Hex"
+            }
         },
         ["code", "action", "binargs"]
     );

api/routes/v1-chain/abi_json_to_bin/index.ts

@@ -7,14 +7,13 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Convert JSON object to binary',
         {
-            "code": {$ref: 'AccountName#'},
-            "action": {$ref: 'AccountName#'},
-            "args": {
-                type: 'object',
-                additionalProperties: true
+            "binargs": {
+                "type": "string",
+                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
+                "title": "Hex"
             }
         },
-        ["code", "action", "args"]
+        ["binargs"]
     );
     next();
 }

api/routes/v1-chain/get_table_rows/index.ts

@@ -36,8 +36,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
             "lower_bound": {
                 "type": "string"
             }
-        },
-        ["code", "table", "scope"]
+        }
     );
     next();
 }

api/routes/v1-history/get_actions/get_actions.ts

@@ -1,150 +1,8 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
-import {Serialize} from "eosjs";
-import {hLog} from "../../../../helpers/common_functions";
-import * as AbiEOS from "@eosrio/node-abieos";
-import {ApiResponse} from "@elastic/elasticsearch";
-import {TextDecoder, TextEncoder} from "util";
-import {JsonRpc} from "eosjs/dist";
-import {terms} from "../../v2-history/get_actions/definitions";
-
-const abi_remapping = {
-    "_Bool": "bool"
-};
-
-const txEnc = new TextEncoder();
-const txDec = new TextDecoder();
-
-async function fetchAbiHexAtBlockElastic(client, chain, contract_name, last_block, get_json) {
-    try {
-        const _includes = ["actions", "tables", "block"];
-        if (get_json) {
-            _includes.push("abi");
-        } else {
-            _includes.push("abi_hex");
-        }
-        const query = {
-            bool: {
-                must: [
-                    {term: {account: contract_name}},
-                    {range: {block: {lte: last_block}}}
-                ]
-            }
-        };
-        const queryResult: ApiResponse = await client.search({
-            index: `${chain}-abi-*`,
-            body: {
-                size: 1, query,
-                sort: [{block: {order: "desc"}}],
-                _source: {includes: _includes}
-            }
-        });
-        const results = queryResult.body.hits.hits;
-        if (results.length > 0) {
-            const nextRefResponse: ApiResponse = await client.search({
-                index: `${chain}-abi-*`,
-                body: {
-                    size: 1,
-                    query: {
-                        bool: {
-                            must: [
-                                {term: {account: contract_name}},
-                                {range: {block: {gte: last_block}}}
-                            ]
-                        }
-                    },
-                    sort: [{block: {order: "asc"}}],
-                    _source: {includes: ["block"]}
-                }
-            });
-            const nextRef = nextRefResponse.body.hits.hits;
-            if (nextRef.length > 0) {
-                return {
-                    valid_until: nextRef[0]._source.block,
-                    ...results[0]._source
-                };
-            }
-            return results[0]._source;
-        } else {
-            return null;
-        }
-    } catch (e) {
-        hLog(e);
-        return null;
-    }
-}
-
-async function getAbiFromHeadBlock(rpc: JsonRpc, code) {
-    let _abi;
-    try {
-        _abi = (await rpc.get_abi(code)).abi;
-    } catch (e) {
-        hLog(e);
-    }
-    return {abi: _abi, valid_until: null, valid_from: null};
-}
-
-async function getContractAtBlock(esClient, rpc, chain, accountName: string, block_num: number, check_action?: string) {
-    let savedAbi, abi;
-    savedAbi = await fetchAbiHexAtBlockElastic(esClient, chain, accountName, block_num, true);
-    if (savedAbi === null || (savedAbi.actions && !savedAbi.actions.includes(check_action))) {
-        savedAbi = await getAbiFromHeadBlock(rpc, accountName);
-        if (!savedAbi) return [null, null];
-        abi = savedAbi.abi;
-    } else {
-        try {
-            abi = JSON.parse(savedAbi.abi);
-        } catch (e) {
-            hLog(e);
-            return [null, null];
-        }
-    }
-    if (!abi) return [null, null];
-    const initialTypes = Serialize.createInitialTypes();
-    let types;
-    try {
-        types = Serialize.getTypesFromAbi(initialTypes, abi);
-    } catch (e) {
-        let remapped = false;
-        for (const struct of abi.structs) {
-            for (const field of struct.fields) {
-                if (abi_remapping[field.type]) {
-                    field.type = abi_remapping[field.type];
-                    remapped = true;
-                }
-            }
-        }
-        if (remapped) {
-            try {
-                types = Serialize.getTypesFromAbi(initialTypes, abi);
-            } catch (e) {
-                hLog('failed after remapping abi');
-                hLog(accountName, block_num, check_action);
-                hLog(e);
-            }
-        } else {
-            hLog(accountName, block_num);
-            hLog(e);
-        }
-    }
-    const actions = new Map();
-    for (const {name, type} of abi.actions) {
-        actions.set(name, Serialize.getType(types, type));
-    }
-    const result = {types, actions, tables: abi.tables};
-    if (check_action) {
-        if (actions.has(check_action)) {
-            try {
-                AbiEOS['load_abi'](accountName, JSON.stringify(abi));
-            } catch (e) {
-                hLog(e);
-            }
-        }
-    }
-    return [result, abi];
-}
-
+import flatstr from 'flatstr';
 
+const terms = ["notified", "act.authorization.actor"];
 const extendedActions = new Set(["transfer", "newaccount", "updateauth"]);
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
@@ -182,8 +40,8 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
     }
     pos = parseInt(reqBody.pos || 0, 10);
     offset = parseInt(reqBody.offset || 20, 10);
-    let from = 0;
-    let size = 0;
+    let from, size;
+    from = size = 0;
     if (pos === -1) {
         if (offset < 0) {
             from = 0;
@@ -272,7 +130,6 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
         }
     };
-    // console.log(JSON.stringify(esOpts, null, 2));
     const pResults = await Promise.all([fastify.eosjs.rpc.get_info(), fastify.elastic['search'](esOpts)]);
     const results = pResults[1];
     const response = {
@@ -297,13 +154,11 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
             actions = actions.slice(index);
         }
-
-        for (let i = 0; i < actions.length; i++) {
-            let action = actions[i];
+        actions.forEach((action, index) => {
             action = action._source;
             let act: any = {
                 "global_action_seq": action.global_sequence,
-                "account_action_seq": i,
+                "account_action_seq": index,
                 "block_num": action.block_num,
                 "block_time": action['@timestamp'],
                 "action_trace": {
@@ -319,31 +174,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             };
             mergeActionMeta(action);
             act.action_trace.act = action.act;
-
-            if (reqBody.hex_data) {
-                try {
-                    const [contract, _] = await getContractAtBlock(
-                        fastify.elastic,
-                        fastify.eosjs.rpc,
-                        fastify.manager.chain,
-                        action.act.account,
-                        action.block_num
-                    );
-                    action.act.hex_data = Serialize.serializeActionData(
-                        contract,
-                        action.act.account,
-                        action.act.name,
-                        action.act.data,
-                        txEnc,
-                        txDec
-                    );
-                } catch (e: any) {
-                    console.log(e);
-                }
-            }
-
-            // TODO: Optionally re-encode using the original ABI, will increase query time
-            // act.action_trace.act.hex_data = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+            act.action_trace.act.hex_data = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
             if (action.act.account_ram_deltas) {
                 act.action_trace.account_ram_deltas = action.account_ram_deltas
             }
@@ -353,7 +184,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             act.action_trace.receiver = receipt.receiver;
             act.account_action_seq = receipt['recv_sequence'];
             response.actions.push(act);
-        }
+        });
     }
     return response;
 }

api/routes/v1-history/get_transaction/get_transaction.ts

@@ -1,190 +1,263 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
+import {createHash} from "crypto";
+import flatstr from 'flatstr';
 import {GetInfoResult} from "eosjs/dist/eosjs-rpc-interfaces";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
+	if (typeof request.body === 'string') {
+		request.body = JSON.parse(request.body)
+	}
+	const body: any = request.body;
+	const redis = fastify.redis;
+	const trxId = body.id.toLowerCase();
+	const conf = fastify.manager.config;
+	const cachedData = await redis.hgetall('trx_' + trxId);
 
-    if (request.method === 'POST' && !request.body) {
-        throw new Error("missing POST body");
-    }
+	const response: any = {
+		"id": body.id,
+		"trx": {
+			"receipt": {
+				"status": "executed",
+				"cpu_usage_us": 0,
+				"net_usage_words": 0,
+				"trx": [1, {
+					"signatures": "",
+					"compression": "none",
+					"packed_context_free_data": "",
+					"packed_trx": ""
+				}]
+			},
+			"trx": {}
+		},
+		"block_num": 0,
+		"block_time": "",
+		"last_irreversible_block": undefined,
+		"traces": []
+	};
 
-    if (typeof request.body === 'string') {
-        request.body = JSON.parse(request.body)
-    }
-    const body: any = request.body;
-    const redis = fastify.redis;
-    const trxId = body.id.toLowerCase();
+	let hits;
 
-    const conf = fastify.manager.config;
-    const cachedData = await redis.hgetall('trx_' + trxId);
+	// build get_info request with caching
+	const $getInfo = new Promise<GetInfoResult>(resolve => {
+		const key = `${fastify.manager.chain}_get_info`;
+		fastify.redis.get(key).then(value => {
+			if (value) {
+				resolve(JSON.parse(value));
+			} else {
+				fastify.eosjs.rpc.get_info().then(value1 => {
+					fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
+					resolve(value1);
+				}).catch((reason) => {
+					console.log(reason);
+					response.error = 'failed to get last_irreversible_block_num'
+					resolve(null);
+				});
+			}
+		});
+	});
 
-    const response: any = {
-        "id": body.id,
-        "trx": {
-            "receipt": {
-                "status": "executed",
-                "cpu_usage_us": 0,
-                "net_usage_words": 0,
-                "trx": [1, {
-                    "signatures": "",
-                    "compression": "none",
-                    "packed_context_free_data": "",
-                    "packed_trx": ""
-                }]
-            },
-            "trx": {}
-        },
-        "block_num": 0,
-        "block_time": "",
-        "last_irreversible_block": undefined,
-        "traces": []
-    };
+	// reconstruct hits from cached data
+	if (cachedData && Object.keys(cachedData).length > 0) {
+		const gsArr = [];
+		for (let cachedDataKey in cachedData) {
+			gsArr.push(cachedData[cachedDataKey]);
+		}
+		gsArr.sort((a, b) => {
+			return a.global_sequence - b.global_sequence;
+		});
+		hits = gsArr.map(value => {
+			return {
+				_source: JSON.parse(value)
+			};
+		});
+		const promiseResults = await Promise.all([
+			redis.ttl('trx_' + trxId),
+			$getInfo
+		]);
+		response.cache_expires_in = promiseResults[0];
+		response.last_irreversible_block = promiseResults[1].last_irreversible_block_num;
+	}
 
-    let hits;
+	// search on ES if cache is not present
+	if (!hits) {
+		const _size = conf.api.limits.get_trx_actions || 100;
+		const blockHint = parseInt(body.block_num_hint, 10);
+		let indexPattern = '';
+		if (blockHint) {
+			const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
+			indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
+		} else {
+			indexPattern = fastify.manager.chain + '-action-*';
+		}
+		let pResults;
+		try {
 
-    // build get_info request with caching
-    const $getInfo = new Promise<GetInfoResult>(resolve => {
-        const key = `${fastify.manager.chain}_get_info`;
-        fastify.redis.get(key).then(value => {
-            if (value) {
-                resolve(JSON.parse(value));
-            } else {
-                fastify.eosjs.rpc.get_info().then(value1 => {
-                    fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
-                    resolve(value1);
-                }).catch((reason) => {
-                    console.log(reason);
-                    response.error = 'failed to get last_irreversible_block_num'
-                    resolve(null);
-                });
-            }
-        });
-    });
+			// build search request
+			const $search = fastify.elastic.search({
+				index: indexPattern,
+				size: _size,
+				body: {
+					query: {bool: {must: [{term: {trx_id: trxId}}]}},
+					sort: {global_sequence: "asc"}
+				}
+			});
 
-    // reconstruct hits from cached data
-    if (cachedData && Object.keys(cachedData).length > 0) {
-        const gsArr = [];
-        for (let cachedDataKey in cachedData) {
-            gsArr.push(cachedData[cachedDataKey]);
-        }
-        gsArr.sort((a, b) => {
-            return a.global_sequence - b.global_sequence;
-        });
-        hits = gsArr.map(value => {
-            return {
-                _source: JSON.parse(value)
-            };
-        });
-        const promiseResults = await Promise.all([
-            redis.ttl('trx_' + trxId),
-            $getInfo
-        ]);
-        response.cache_expires_in = promiseResults[0];
-        response.last_irreversible_block = promiseResults[1].last_irreversible_block_num;
-    }
-
-    // search on ES if cache is not present
-    if (!hits) {
-        const _size = conf.api.limits.get_trx_actions || 100;
-        const blockHint = parseInt(body.block_num_hint, 10);
-        let indexPattern = '';
-        if (blockHint) {
-            const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
-            indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
-        } else {
-            indexPattern = fastify.manager.chain + '-action-*';
-        }
-        let pResults;
-        try {
-
-            // build search request
-            const $search = fastify.elastic.search({
-                index: indexPattern,
-                size: _size,
-                body: {
-                    query: {bool: {must: [{term: {trx_id: trxId}}]}},
-                    sort: {global_sequence: "asc"}
-                }
-            });
-
-            // execute in parallel
-            pResults = await Promise.all([$getInfo, $search]);
-        } catch (e) {
-            console.log(e.message);
-            if (e.meta.statusCode === 404) {
-                return response;
-            }
-        }
-        hits = pResults[1]['body']['hits']['hits'];
-        response.last_irreversible_block = pResults[0].last_irreversible_block_num;
-    }
+			// execute in parallel
+			pResults = await Promise.all([$getInfo, $search]);
+		} catch (e) {
+			console.log(e.message);
+			if (e.meta.statusCode === 404) {
+				return response;
+			}
+		}
+		hits = pResults[1]['body']['hits']['hits'];
+		response.last_irreversible_block = pResults[0].last_irreversible_block_num;
+	}
 
 
-    if (hits.length > 0) {
-        const actions = hits;
-        response.trx.trx = {
-            "expiration": "",
-            "ref_block_num": 0,
-            "ref_block_prefix": 0,
-            "max_net_usage_words": 0,
-            "max_cpu_usage_ms": 0,
-            "delay_sec": 0,
-            "context_free_actions": [],
-            "actions": [],
-            "transaction_extensions": [],
-            "signatures": [],
-            "context_free_data": []
-        };
-        for (let action of actions) {
-            action = action._source;
-            mergeActionMeta(action);
-            response.block_num = action.block_num;
-            response.block_time = action['@timestamp'];
-            for (const receipt of action.receipts) {
-                if (action.act_digest) {
-                    receipt.act_digest = action.act_digest.toLowerCase();
-                }
-                response.traces.push({
-                    receipt: receipt,
-                    act: action.act,
-                    account_ram_deltas: action.account_ram_deltas || [],
-                    context_free: false,
-                    block_num: action.block_num,
-                    block_time: action['@timestamp'],
-                    console: "",
-                    elapsed: 0,
-                    except: null,
-                    inline_traces: [],
-                    producer_block_id: "",
-                    trx_id: body.id
-                });
-            }
-        }
-    } else {
-        const errmsg = "Transaction " + body.id.toLowerCase() + " not found in history and no block hint was given";
-        return {
-            code: 500,
-            message: "Internal Service Error",
-            error: {
-                code: 3040011,
-                name: "tx_not_found",
-                what: "The transaction can not be found",
-                details: [
-                    {
-                        message: errmsg,
-                        file: "",
-                        line_number: 1,
-                        method: "get_transaction"
-                    }
-                ]
-            }
-        }
-    }
-    return response;
+	if (hits.length > 0) {
+		const actions = hits;
+		response.trx.trx = {
+			"expiration": "",
+			"ref_block_num": 0,
+			"ref_block_prefix": 0,
+			"max_net_usage_words": 0,
+			"max_cpu_usage_ms": 0,
+			"delay_sec": 0,
+			"context_free_actions": [],
+			"actions": [],
+			"transaction_extensions": [],
+			"signatures": [],
+			"context_free_data": []
+		};
+		let traces = {};
+		let seqNum = 0;
+		for (let action of actions) {
+			action = action._source;
+			mergeActionMeta(action);
+			action.act['hex_data'] = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
+			if (action.parent === 0) {
+				response.trx.trx.actions.push(action.act);
+			}
+			response.block_num = action.block_num;
+			response.block_time = action['@timestamp'];
+			seqNum += 1;
+			let trace = {
+				receipt: {
+					receiver: action.act.account,
+					global_sequence: String(action.global_sequence),
+					auth_sequence: [
+						action.act.authorization[0].actor,
+						seqNum
+					],
+					act_digest: '',
+					recv_sequence: seqNum,
+					code_sequence: 1,
+					abi_sequence: 1
+				},
+				act: action.act,
+				account_ram_deltas: action.account_ram_deltas || [],
+				context_free: false,
+				block_num: action.block_num,
+				block_time: action['@timestamp'],
+				console: "",
+				elapsed: 0,
+				except: null,
+				inline_traces: [],
+				producer_block_id: "",
+				trx_id: body.id,
+				notified: action.notified
+			};
+			let hash = createHash('sha256');
+			hash.update(JSON.stringify(action.act));
+			trace.receipt.act_digest = hash.digest('hex');
+			traces[action.global_sequence] = trace;
+		}
+
+		actions.forEach(action => {
+			action = action._source;
+			for (let i = 0; i < traces[action.global_sequence].notified.length; i++) {
+				if (traces[action.global_sequence].notified[i] === action.act.account) {
+					traces[action.global_sequence].notified.splice(i, 1);
+					break;
+				}
+			}
+			if (action.parent !== 0 && action.parent) {
+				if (traces[action.parent]) {
+					for (let i = 0; i < traces[action.parent].notified.length; i++) {
+						if (traces[action.parent].notified[i] === action.act.account) {
+							traces[action.parent].notified.splice(i, 1);
+							break;
+						}
+					}
+					traces[action.parent].inline_traces.push(traces[action.global_sequence]);
+				}
+			}
+		});
+
+		actions.forEach(action => {
+			action = action._source;
+			response.traces.push(traces[action.global_sequence]);
+			if (traces[action.global_sequence] && traces[action.global_sequence].notified) {
+				traces[action.global_sequence].notified.forEach((note, index) => {
+					seqNum += 1;
+					let trace = {
+						receipt: {
+							receiver: note,
+							global_sequence: String(action.global_sequence + index + 1),
+							auth_sequence: [
+								action.act.authorization[0].actor,
+								seqNum
+							],
+							act_digest: traces[action.global_sequence].receipt.act_digest,
+							recv_sequence: seqNum,
+							code_sequence: 1,
+							abi_sequence: 1
+						},
+						account_ram_deltas: action.account_ram_deltas || [],
+						act: action.act,
+						block_num: action.block_num,
+						block_time: action['@timestamp'],
+						console: "",
+						context_free: false,
+						elapsed: 0,
+						except: null,
+						inline_traces: [],
+						producer_block_id: "",
+						trx_id: body.id,
+					};
+					traces[action.global_sequence].inline_traces.unshift(trace);
+					response.traces.push(trace);
+				});
+				delete traces[action.global_sequence].notified;
+			}
+		});
+	} else {
+		const errmsg = "Transaction " + body.id.toLowerCase() + " not found in history and no block hint was given";
+		return {
+			code: 500,
+			message: "Internal Service Error",
+			error: {
+				code: 3040011,
+				name: "tx_not_found",
+				what: "The transaction can not be found",
+				details: [
+					{
+						message: errmsg,
+						file: "",
+						line_number: 1,
+						method: "get_transaction"
+					}
+				]
+			}
+		}
+	}
+	return response;
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply) => {
-        reply.send(await timedQuery(getTransaction, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getTransaction, fastify, request, route));
+	}
 }

api/routes/v2-history/get_abi_snapshot/get_abi_snapshot.ts

@@ -22,7 +22,7 @@ async function getAbiSnapshot(fastify: FastifyInstance, request: FastifyRequest)
     }
 
     const results = await fastify.elastic.search({
-        index: fastify.manager.chain + '-abi-*',
+        index: fastify.manager.chain + '-abi',
         size: 1,
         body: {
             query: {bool: {must: mustArray}},

api/routes/v2-history/get_actions/definitions.ts

@@ -1,6 +1,5 @@
 export const terms = [
     "notified",
-    "receipts.receiver",
     "act.authorization.actor"
 ];
 
@@ -12,19 +11,12 @@ export const extendedActions = new Set([
     "buyrambytes",
     "delegatebw",
     "undelegatebw",
-    "voteproducer",
-    'statement',
-    'decision',
-    'batch',
-    'act',
-    'exec',
-    'votefor',
-    'voteagainst'
+    "voteproducer"
 ]);
 
 export const primaryTerms = [
+    "notified",
     "block_num",
-    "block_id",
     "global_sequence",
     "producer",
     "@timestamp",
@@ -32,10 +24,5 @@ export const primaryTerms = [
     "action_ordinal",
     "cpu_usage_us",
     "net_usage_words",
-    "trx_id",
-    "receipts.receiver",
-    "receipts.global_sequence",
-    "receipts.recv_sequence",
-    "receipts.auth_sequence.account",
-    "receipts.auth_sequence.sequence"
+    "trx_id"
 ];

api/routes/v2-history/get_actions/functions.ts

@@ -1,266 +1,234 @@
 import {primaryTerms, terms} from "./definitions";
 
 export function addSortedBy(query, queryBody, sort_direction) {
-    if (query['sortedBy']) {
-        const opts = query['sortedBy'].split(":");
-        const sortedByObj = {};
-        sortedByObj[opts[0]] = opts[1];
-        queryBody['sort'] = sortedByObj;
-    } else {
-        queryBody['sort'] = {
-            "global_sequence": sort_direction
-        };
-    }
+	if (query['sortedBy']) {
+		const opts = query['sortedBy'].split(":");
+		const sortedByObj = {};
+		sortedByObj[opts[0]] = opts[1];
+		queryBody['sort'] = sortedByObj;
+	} else {
+		queryBody['sort'] = {
+			"global_sequence": sort_direction
+		};
+	}
 }
 
 export function processMultiVars(queryStruct, parts, field) {
-    const must = [];
-    const mustNot = [];
+	const must = [];
+	const mustNot = [];
 
-    parts.forEach(part => {
-        if (part.startsWith("!")) {
-            mustNot.push(part.replace("!", ""));
-        } else {
-            must.push(part);
-        }
-    });
+	parts.forEach(part => {
+		if (part.startsWith("!")) {
+			mustNot.push(part.replace("!", ""));
+		} else {
+			must.push(part);
+		}
+	});
 
-    if (must.length > 1) {
-        queryStruct.bool.must.push({
-            bool: {
-                should: must.map(elem => {
-                    const _q = {};
-                    _q[field] = elem;
-                    return {term: _q}
-                })
-            }
-        });
-    } else if (must.length === 1) {
-        const mustQuery = {};
-        mustQuery[field] = must[0];
-        queryStruct.bool.must.push({term: mustQuery});
-    }
+	if (must.length > 1) {
+		queryStruct.bool.must.push({
+			bool: {
+				should: must.map(elem => {
+					const _q = {};
+					_q[field] = elem;
+					return {term: _q}
+				})
+			}
+		});
+	} else if (must.length === 1) {
+		const mustQuery = {};
+		mustQuery[field] = must[0];
+		queryStruct.bool.must.push({term: mustQuery});
+	}
 
-    if (mustNot.length > 1) {
-        queryStruct.bool.must_not.push({
-            bool: {
-                should: mustNot.map(elem => {
-                    const _q = {};
-                    _q[field] = elem;
-                    return {term: _q}
-                })
-            }
-        });
-    } else if (mustNot.length === 1) {
-        const mustNotQuery = {};
-        mustNotQuery[field] = mustNot[0].replace("!", "");
-        queryStruct.bool.must_not.push({term: mustNotQuery});
-    }
+	if (mustNot.length > 1) {
+		queryStruct.bool.must_not.push({
+			bool: {
+				should: mustNot.map(elem => {
+					const _q = {};
+					_q[field] = elem;
+					return {term: _q}
+				})
+			}
+		});
+	} else if (mustNot.length === 1) {
+		const mustNotQuery = {};
+		mustNotQuery[field] = mustNot[0].replace("!", "");
+		queryStruct.bool.must_not.push({term: mustNotQuery});
+	}
 }
 
 function addRangeQuery(queryStruct, prop, pkey, query) {
-    const _termQuery = {};
-    const parts = query[prop].split("-");
-    _termQuery[pkey] = {
-        "gte": parts[0],
-        "lte": parts[1]
-    };
-    queryStruct.bool.must.push({range: _termQuery});
+	const _termQuery = {};
+	const parts = query[prop].split("-");
+	_termQuery[pkey] = {
+		"gte": parts[0],
+		"lte": parts[1]
+	};
+	queryStruct.bool.must.push({range: _termQuery});
 }
 
 export function applyTimeFilter(query, queryStruct) {
-    if (query['after'] || query['before']) {
-
-        if (query['after']?.includes(' ')) {
-            query['after'] = query['after'].replace(' ', 'T');
-        }
-
-        if (query['before']?.includes(' ')) {
-            query['before'] = query['before'].replace(' ', 'T');
-        }
-
-        if (query['after']?.includes('T') || query['before']?.includes('T')) {
-            let _lte = "now";
-            let _gte = "0";
-            if (query['before']) {
-                try {
-                    _lte = new Date(query['before']).toISOString();
-                } catch (e) {
-                    throw new Error(e.message + ' [before]');
-                }
-            }
-            if (query['after']) {
-                try {
-                    _gte = new Date(query['after']).toISOString();
-                } catch (e) {
-                    throw new Error(e.message + ' [after]');
-                }
-            }
-            if (!queryStruct.bool['filter']) {
-                queryStruct.bool['filter'] = [];
-            }
-            queryStruct.bool['filter'].push({
-                range: {
-                    "@timestamp": {
-                        "gte": _gte,
-                        "lte": _lte
-                    }
-                }
-            });
-        } else {
-            // search by block number
-            const rangeObj = {
-                range: {
-                    block_num: {}
-                }
-            };
-            if (parseInt(query['after']) > 0) {
-                rangeObj.range.block_num['gte'] = query['after'];
-            }
-            if (parseInt(query['before']) > 0) {
-                rangeObj.range.block_num['lte'] = query['before'];
-            }
-            if (Object.keys(rangeObj.range.block_num).length > 0) {
-                if (!queryStruct.bool['filter']) {
-                    queryStruct.bool['filter'] = [];
-                }
-                queryStruct.bool['filter'].push(rangeObj);
-            }
-        }
-    }
+	if (query['after'] || query['before']) {
+		let _lte = "now";
+		let _gte = "0";
+		if (query['before']) {
+			_lte = query['before'];
+			if (!_lte.endsWith("Z")) {
+				_lte += "Z";
+			}
+		}
+		if (query['after']) {
+			_gte = query['after'];
+			if (!_gte.endsWith("Z")) {
+				_gte += "Z";
+			}
+		}
+		if (!queryStruct.bool['filter']) {
+			queryStruct.bool['filter'] = [];
+		}
+		queryStruct.bool['filter'].push({
+			range: {
+				"@timestamp": {
+					"gte": _gte,
+					"lte": _lte
+				}
+			}
+		});
+	}
 }
 
 export function applyGenericFilters(query, queryStruct, allowedExtraParams: Set<string>) {
-    for (const prop in query) {
-        if (Object.prototype.hasOwnProperty.call(query, prop)) {
-            const pair = prop.split(".");
-            if (pair.length > 1 || primaryTerms.includes(pair[0])) {
-                let pkey;
-                if (pair.length > 1 && allowedExtraParams) {
-                    pkey = allowedExtraParams.has(pair[0]) ? "@" + prop : prop;
-                } else {
-                    pkey = prop;
-                }
-                if (query[prop].indexOf("-") !== -1) {
-                    addRangeQuery(queryStruct, prop, pkey, query);
-                } else {
-                    const _qObj = {};
-                    const parts = query[prop].split(",");
-                    if (parts.length > 1) {
-                        processMultiVars(queryStruct, parts, prop);
-                    } else if (parts.length === 1) {
+	for (const prop in query) {
+		if (Object.prototype.hasOwnProperty.call(query, prop)) {
+			const pair = prop.split(".");
+			if (pair.length > 1 || primaryTerms.includes(pair[0])) {
+				let pkey;
+				if (pair.length > 1 && allowedExtraParams) {
+					pkey = allowedExtraParams.has(pair[0]) ? "@" + prop : prop;
+				} else {
+					pkey = prop;
+				}
+				if (query[prop].indexOf("-") !== -1) {
+					addRangeQuery(queryStruct, prop, pkey, query);
+				} else {
+					const _qObj = {};
+					const parts = query[prop].split(",");
+					if (parts.length > 1) {
+						processMultiVars(queryStruct, parts, prop);
+					} else if (parts.length === 1) {
 
-                        // @transfer.memo special case
-                        if (pkey === '@transfer.memo') {
-                            _qObj[pkey] = {
-                                query: parts[0]
-                            };
+						// @transfer.memo special case
+						if (pkey === '@transfer.memo') {
+							_qObj[pkey] = {
+								query: parts[0]
+							};
 
-                            if (query.match_fuzziness) {
-                                _qObj[pkey].fuzziness = query.match_fuzziness;
-                            }
+							if (query.match_fuzziness) {
+								_qObj[pkey].fuzziness = query.match_fuzziness;
+							}
 
-                            if (query.match_operator) {
-                                _qObj[pkey].operator = query.match_operator;
-                            }
+							if (query.match_operator) {
+								_qObj[pkey].operator = query.match_operator;
+							}
 
-                            queryStruct.bool.must.push({
-                                match: _qObj
-                            });
-                        } else {
-                            const andParts = parts[0].split(" ");
-                            if (andParts.length > 1) {
-                                andParts.forEach(value => {
-                                    const _q = {};
-                                    _q[pkey] = value;
-                                    queryStruct.bool.must.push({term: _q});
-                                });
-                            } else {
-                                if (parts[0].startsWith("!")) {
-                                    _qObj[pkey] = parts[0].replace("!", "");
-                                    queryStruct.bool.must_not.push({term: _qObj});
-                                } else {
-                                    _qObj[pkey] = parts[0];
-                                    queryStruct.bool.must.push({term: _qObj});
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
+							queryStruct.bool.must.push({
+								match: _qObj
+							});
+						} else {
+							const andParts = parts[0].split(" ");
+							if (andParts.length > 1) {
+								andParts.forEach(value => {
+									const _q = {};
+									_q[pkey] = value;
+									queryStruct.bool.must.push({term: _q});
+								});
+							} else {
+								if (parts[0].startsWith("!")) {
+									_qObj[pkey] = parts[0].replace("!", "");
+									queryStruct.bool.must_not.push({term: _qObj});
+								} else {
+									_qObj[pkey] = parts[0];
+									queryStruct.bool.must.push({term: _qObj});
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
 }
 
 export function makeShouldArray(query) {
-    const should_array = [];
-    for (const entry of terms) {
-        const tObj = {term: {}};
-        tObj.term[entry] = query.account;
-        should_array.push(tObj);
-    }
-    return should_array;
+	const should_array = [];
+	for (const entry of terms) {
+		const tObj = {term: {}};
+		tObj.term[entry] = query.account;
+		should_array.push(tObj);
+	}
+	return should_array;
 }
 
 export function applyCodeActionFilters(query, queryStruct) {
-    let filterObj = [];
-    if (query.filter) {
-        for (const filter of query.filter.split(',')) {
-            if (filter !== '*:*') {
-                const _arr = [];
-                const parts = filter.split(':');
-                if (parts.length === 2) {
-                    const [code, method] = parts;
-                    if (code && code !== "*") {
-                        _arr.push({'term': {'act.account': code}});
-                    }
-                    if (method && method !== "*") {
-                        _arr.push({'term': {'act.name': method}});
-                    }
-                }
-                if (_arr.length > 0) {
-                    filterObj.push({bool: {must: _arr}});
-                }
-            }
-        }
-        if (filterObj.length > 0) {
-            queryStruct.bool['should'] = filterObj;
-            queryStruct.bool['minimum_should_match'] = 1;
-        }
-    }
+	let filterObj = [];
+	if (query.filter) {
+		for (const filter of query.filter.split(',')) {
+			if (filter !== '*:*') {
+				const _arr = [];
+				const parts = filter.split(':');
+				if (parts.length === 2) {
+					const [code, method] = parts;
+					if (code && code !== "*") {
+						_arr.push({'term': {'act.account': code}});
+					}
+					if (method && method !== "*") {
+						_arr.push({'term': {'act.name': method}});
+					}
+				}
+				if (_arr.length > 0) {
+					filterObj.push({bool: {must: _arr}});
+				}
+			}
+		}
+		if (filterObj.length > 0) {
+			queryStruct.bool['should'] = filterObj;
+			queryStruct.bool['minimum_should_match'] = 1;
+		}
+	}
 }
 
 export function getSkipLimit(query, max?: number) {
-    let skip, limit;
-    skip = parseInt(query.skip, 10);
-    if (skip < 0) {
-        throw new Error('invalid skip parameter');
-    }
-    limit = parseInt(query.limit, 10);
-    if (limit < 1) {
-        throw new Error('invalid limit parameter');
-    } else if (limit > max) {
-        throw new Error(`limit too big, maximum: ${max}`);
-    }
-    return {skip, limit};
+	let skip, limit;
+	skip = parseInt(query.skip, 10);
+	if (skip < 0) {
+		throw new Error('invalid skip parameter');
+	}
+	limit = parseInt(query.limit, 10);
+	if (limit < 1) {
+		throw new Error('invalid limit parameter');
+	} else if (limit > max) {
+		throw new Error(`limit too big, maximum: ${max}`);
+	}
+	return {skip, limit};
 }
 
 export function getSortDir(query) {
-    let sort_direction = 'desc';
-    if (query.sort) {
-        if (query.sort === 'asc' || query.sort === '1') {
-            sort_direction = 'asc';
-        } else if (query.sort === 'desc' || query.sort === '-1') {
-            sort_direction = 'desc'
-        } else {
-            throw new Error('invalid sort direction');
-        }
-    }
-    return sort_direction;
+	let sort_direction = 'desc';
+	if (query.sort) {
+		if (query.sort === 'asc' || query.sort === '1') {
+			sort_direction = 'asc';
+		} else if (query.sort === 'desc' || query.sort === '-1') {
+			sort_direction = 'desc'
+		} else {
+			throw new Error('invalid sort direction');
+		}
+	}
+	return sort_direction;
 }
 
 export function applyAccountFilters(query, queryStruct) {
-    if (query.account) {
-        queryStruct.bool.must.push({"bool": {should: makeShouldArray(query)}});
-    }
+	if (query.account) {
+		queryStruct.bool.must.push({"bool": {should: makeShouldArray(query)}});
+	}
 }

api/routes/v2-history/get_actions/get_actions.ts

@@ -1,138 +1,118 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {getTrackTotalHits, mergeActionMeta, timedQuery} from "../../../helpers/functions";
 import {
-    addSortedBy,
-    applyAccountFilters,
-    applyCodeActionFilters,
-    applyGenericFilters,
-    applyTimeFilter,
-    getSkipLimit,
-    getSortDir
+	addSortedBy,
+	applyAccountFilters,
+	applyCodeActionFilters,
+	applyGenericFilters,
+	applyTimeFilter,
+	getSkipLimit,
+	getSortDir
 } from "./functions";
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
-    const query: any = request.query;
-    const maxActions = fastify.manager.config.api.limits.get_actions;
-    const queryStruct = {
-        "bool": {
-            must: [],
-            must_not: [],
-            boost: 1.0
-        }
-    };
+	const query: any = request.query;
+	const maxActions = fastify.manager.config.api.limits.get_actions;
+	const queryStruct = {
+		"bool": {
+			must: [],
+			must_not: [],
+			boost: 1.0
+		}
+	};
 
-    const {skip, limit} = getSkipLimit(query, maxActions);
+	const {skip, limit} = getSkipLimit(query, maxActions);
+	const sort_direction = getSortDir(query);
+	applyAccountFilters(query, queryStruct);
+	applyGenericFilters(query, queryStruct, fastify.allowedActionQueryParamSet);
+	applyTimeFilter(query, queryStruct);
+	applyCodeActionFilters(query, queryStruct);
+	// allow precise counting of total hits
+	const trackTotalHits = getTrackTotalHits(query);
 
-    const sort_direction = getSortDir(query);
+	// Prepare query body
+	const query_body = {
+		"track_total_hits": trackTotalHits,
+		"query": queryStruct
+	};
 
-    applyAccountFilters(query, queryStruct);
+	// Include sorting
+	addSortedBy(query, query_body, sort_direction);
 
-    applyGenericFilters(query, queryStruct, fastify.allowedActionQueryParamSet);
+	// Perform search
 
-    applyTimeFilter(query, queryStruct);
+	let indexPattern = fastify.manager.chain + '-action-*';
+	if (query.hot_only) {
+		indexPattern = fastify.manager.chain + '-action';
+	}
 
-    applyCodeActionFilters(query, queryStruct);
+	const esResults = await fastify.elastic.search({
+		"index": indexPattern,
+		"from": skip || 0,
+		"size": (limit > maxActions ? maxActions : limit) || 10,
+		"body": query_body
+	});
 
-    // allow precise counting of total hits
-    const trackTotalHits = getTrackTotalHits(query);
+	const results = esResults['body']['hits'];
+	const response: any = {
+		cached: false,
+		lib: 0,
+		total: results['total']
+	};
 
-    // Prepare query body
-    const query_body = {
-        "track_total_hits": trackTotalHits,
-        "query": queryStruct
-    };
+	if (query.hot_only) {
+		response.hot_only = true;
+	}
 
-    // Include sorting
-    addSortedBy(query, query_body, sort_direction);
+	if (query.checkLib) {
+		response.lib = (await fastify.eosjs.rpc.get_info()).last_irreversible_block_num;
+	}
 
-    // Perform search
+	if (query.simple) {
+		response['simple_actions'] = [];
+	} else {
+		response['actions'] = [];
+	}
 
-    let indexPattern = fastify.manager.chain + '-action-*';
-    if (query.hot_only) {
-        indexPattern = fastify.manager.chain + '-action';
-    }
+	if (results['hits'].length > 0) {
+		const actions = results['hits'];
+		for (let action of actions) {
+			action = action._source;
+			mergeActionMeta(action);
 
-    const esOpts = {
-        "index": indexPattern,
-        "from": skip || 0,
-        "size": (limit > maxActions ? maxActions : limit) || 10,
-        "body": query_body
-    };
+			if (query.noBinary === true) {
+				for (const key in action['act']['data']) {
+					if (action['act']['data'].hasOwnProperty(key)) {
+						if (typeof action['act']['data'][key] === 'string' && action['act']['data'][key].length > 256) {
+							action['act']['data'][key] = action['act']['data'][key].slice(0, 32) + "...";
+						}
+					}
+				}
+			}
 
-    // console.log(JSON.stringify(esOpts, null, 2));
-    const esResults = await fastify.elastic.search(esOpts);
+			if (query.simple) {
+				response.simple_actions.push({
+					block: action['block_num'],
+					irreversible: response.lib !== 0 ? action['block_num'] < response.lib : undefined,
+					timestamp: action['@timestamp'],
+					transaction_id: action['trx_id'],
+					actors: action['act']['authorization'].map(a => `${a.actor}@${a.permission}`).join(","),
+					notified: action['notified'].join(','),
+					contract: action['act']['account'],
+					action: action['act']['name'],
+					data: action['act']['data']
+				});
 
-    const results = esResults['body']['hits'];
-    const response: any = {
-        cached: false,
-        lib: 0,
-        total: results['total']
-    };
-
-    if (query.hot_only) {
-        response.hot_only = true;
-    }
-
-    if (query.checkLib) {
-        response.lib = (await fastify.eosjs.rpc.get_info()).last_irreversible_block_num;
-    }
-
-    if (query.simple) {
-        response['simple_actions'] = [];
-    } else {
-        response['actions'] = [];
-    }
-
-    if (results['hits'].length > 0) {
-        const actions = results['hits'];
-        for (let action of actions.map(a => a._source)) {
-
-            try {
-                if (action.act.data) {
-                    if (action.act.data.account && action.act.data.name && action.act.data.authorization) {
-                        action.act.data = action.act.data.data;
-                    }
-                }
-            } catch (e: any) {
-                console.log(e);
-            }
-
-            mergeActionMeta(action);
-
-            if (query.noBinary === true) {
-                for (const key in action['act']['data']) {
-                    if (action['act']['data'].hasOwnProperty(key)) {
-                        if (typeof action['act']['data'][key] === 'string' && action['act']['data'][key].length > 256) {
-                            action['act']['data'][key] = action['act']['data'][key].slice(0, 32) + "...";
-                        }
-                    }
-                }
-            }
-
-            if (query.simple) {
-                let notified = new Set(action.receipts.map(r => r.receiver));
-                response.simple_actions.push({
-                    block: action['block_num'],
-                    irreversible: response.lib !== 0 ? action['block_num'] < response.lib : undefined,
-                    timestamp: action['@timestamp'],
-                    transaction_id: action['trx_id'],
-                    actors: action['act']['authorization'].map(a => `${a.actor}@${a.permission}`).join(","),
-                    notified: [...notified].join(','),
-                    contract: action['act']['account'],
-                    action: action['act']['name'],
-                    data: action['act']['data']
-                });
-
-            } else {
-                response.actions.push(action);
-            }
-        }
-    }
-    return response;
+			} else {
+				response.actions.push(action);
+			}
+		}
+	}
+	return response;
 }
 
 export function getActionsHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply) => {
-        reply.send(await timedQuery(getActions, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getActions, fastify, request, route));
+	}
 }

api/routes/v2-history/get_actions/index.ts

@@ -3,158 +3,129 @@ import {getActionsHandler} from "./get_actions";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export const getActionResponseSchema = {
-    "@timestamp": {type: "string"},
-    "timestamp": {type: "string"},
-    "block_num": {type: "number"},
-    "block_id": {type: "string"},
-    "trx_id": {type: "string"},
-    "act": {
-        type: 'object',
-        properties: {
-            "account": {type: "string"},
-            "name": {type: "string"},
-            "authorization": {
-                type: "array",
-                items: {
-                    type: "object",
-                    properties: {
-                        "actor": {type: "string"},
-                        "permission": {type: "string"},
-                    }
-                }
-            }
-        },
-        additionalProperties: true
-    },
-    "receipts": {
-        type: "array",
-        items: {
-            type: "object",
-            properties: {
-                "receiver": {type: "string"},
-                "global_sequence": {type: "number"},
-                "recv_sequence": {type: "number"},
-                "auth_sequence": {
-                    type: "array",
-                    items: {
-                        type: "object",
-                        properties: {
-                            "account": {type: "string"},
-                            "sequence": {type: "number"},
-                        }
-                    }
-                }
-            }
-        }
-    },
-    "cpu_usage_us": {type: "number"},
-    "net_usage_words": {type: "number"},
-    "account_ram_deltas": {
-        type: "array",
-        items: {
-            type: "object",
-            properties: {
-                "account": {type: "string"},
-                "delta": {type: "number"}
-            },
-            additionalProperties: true
-        }
-    },
-    "global_sequence": {type: "number"},
-    "producer": {type: "string"},
-    "parent": {type: "number"},
-    "action_ordinal": {type: 'number'},
-    "creator_action_ordinal": {type: 'number'},
-    "signatures": {type: "array", items: {type: 'string'}}
+	"@timestamp": {type: "string"},
+	"timestamp": {type: "string"},
+	"block_num": {type: "number"},
+	"trx_id": {type: "string"},
+	"act": {
+		type: 'object',
+		properties: {
+			"account": {type: "string"},
+			"name": {type: "string"}
+		},
+		additionalProperties: true
+	},
+	"notified": {
+		type: "array", items: {type: "string"}
+	},
+	"cpu_usage_us": {type: "number"},
+	"net_usage_words": {type: "number"},
+	"account_ram_deltas": {
+		type: "array",
+		items: {
+			type: "object",
+			properties: {
+				"account": {type: "string"},
+				"delta": {type: "number"}
+			},
+			additionalProperties: true
+		}
+	},
+	"global_sequence": {type: "number"},
+	"receiver": {type: 'string'},
+	"producer": {type: "string"},
+	"parent": {type: "number"},
+	"action_ordinal": {type: 'number'},
+	"creator_action_ordinal": {type: 'number'}
 };
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema: FastifySchema = {
-        description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
-            ' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
-        summary: 'get root actions',
-        tags: ['history'],
-        querystring: extendQueryStringSchema({
-            "account": {
-                description: 'notified account',
-                type: 'string',
-                minLength: 1,
-                maxLength: 12
-            },
-            "track": {
-                description: 'total results to track (count) [number or true]',
-                type: 'string'
-            },
-            "filter": {
-                description: 'code:name filter',
-                type: 'string',
-                minLength: 3
-            },
-            "sort": {
-                description: 'sort direction',
-                enum: ['desc', 'asc', '1', '-1'],
-                type: 'string'
-            },
-            "after": {
-                description: 'filter after specified date (ISO8601)',
-                type: 'string'
-            },
-            "before": {
-                description: 'filter before specified date (ISO8601)',
-                type: 'string'
-            },
-            "simple": {
-                description: 'simplified output mode',
-                type: 'boolean'
-            },
-            "hot_only": {
-                description: 'search only the latest hot index',
-                type: 'boolean'
-            },
-            "noBinary": {
-                description: "exclude large binary data",
-                type: 'boolean'
-            },
-            "checkLib": {
-                description: "perform reversibility check",
-                type: 'boolean'
-            },
-        }),
-        response: extendResponseSchema({
-            "simple_actions": {
-                type: "array",
-                items: {
-                    type: "object",
-                    properties: {
-                        "block": {type: "number"},
-                        "timestamp": {type: "string"},
-                        "irreversible": {type: "boolean"},
-                        "contract": {type: "string"},
-                        "action": {type: "string"},
-                        "actors": {type: "string"},
-                        "notified": {type: "string"},
-                        "transaction_id": {type: "string"},
-                        "data": {
-                            additionalProperties: true
-                        }
-                    }
-                }
-            },
-            "actions": {
-                type: "array",
-                items: {
-                    type: 'object',
-                    properties: getActionResponseSchema
-                }
-            }
-        })
-    };
-    addApiRoute(
-        fastify,
-        'GET',
-        getRouteName(__filename),
-        getActionsHandler,
-        schema
-    );
-    next();
+	const schema: FastifySchema = {
+		description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
+			' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
+		summary: 'get root actions',
+		tags: ['history'],
+		querystring: extendQueryStringSchema({
+			"account": {
+				description: 'notified account',
+				type: 'string',
+				minLength: 1,
+				maxLength: 12
+			},
+			"track": {
+				description: 'total results to track (count) [number or true]',
+				type: 'string'
+			},
+			"filter": {
+				description: 'code:name filter',
+				type: 'string',
+				minLength: 3
+			},
+			"sort": {
+				description: 'sort direction',
+				enum: ['desc', 'asc', '1', '-1'],
+				type: 'string'
+			},
+			"after": {
+				description: 'filter after specified date (ISO8601)',
+				type: 'string'
+			},
+			"before": {
+				description: 'filter before specified date (ISO8601)',
+				type: 'string'
+			},
+			"simple": {
+				description: 'simplified output mode',
+				type: 'boolean'
+			},
+			"hot_only": {
+				description: 'search only the latest hot index',
+				type: 'boolean'
+			},
+			"noBinary": {
+				description: "exclude large binary data",
+				type: 'boolean'
+			},
+			"checkLib": {
+				description: "perform reversibility check",
+				type: 'boolean'
+			},
+		}),
+		response: extendResponseSchema({
+			"simple_actions": {
+				type: "array",
+				items: {
+					type: "object",
+					properties: {
+						"block": {type: "number"},
+						"timestamp": {type: "string"},
+						"irreversible": {type: "boolean"},
+						"contract": {type: "string"},
+						"action": {type: "string"},
+						"actors": {type: "string"},
+						"notified": {type: "string"},
+						"transaction_id": {type: "string"},
+						"data": {
+							additionalProperties: true
+						}
+					}
+				}
+			},
+			"actions": {
+				type: "array",
+				items: {
+					type: 'object',
+					properties: getActionResponseSchema
+				}
+			}
+		})
+	};
+	addApiRoute(
+		fastify,
+		'GET',
+		getRouteName(__filename),
+		getActionsHandler,
+		schema
+	);
+	next();
 }

api/routes/v2-history/get_creator/get_creator.ts

@@ -3,104 +3,98 @@ import {timedQuery} from "../../../helpers/functions";
 
 async function getCreator(fastify: FastifyInstance, request: FastifyRequest) {
 
-    const query: any = request.query;
+	const query: any = request.query;
 
-    const response = {
-        account: query.account,
-        creator: '',
-        timestamp: '',
-        block_num: 0,
-        trx_id: '',
-    };
+	const response = {
+		account: query.account,
+		creator: '',
+		timestamp: '',
+		block_num: 0,
+		trx_id: '',
+	};
 
-    if (query.account === fastify.manager.config.settings.eosio_alias) {
-        try {
-            const genesisBlock = await fastify.eosjs.rpc.get_block(1);
-            if (genesisBlock) {
-                response.timestamp = genesisBlock.timestamp;
-            }
-        } catch (e) {
-            console.log(e.message);
-        }
-        response.creator = '__self__';
-        response.block_num = 1;
-        response.trx_id = "";
-        return response;
-    }
+	if (query.account === fastify.manager.config.settings.eosio_alias) {
+		const genesisBlock = await fastify.eosjs.rpc.get_block(1);
+		if (genesisBlock) {
+			response.creator = '__self__';
+			response.timestamp = genesisBlock.timestamp;
+			response.block_num = 1;
+			response.trx_id = "";
+			return response;
+		} else {
+			throw new Error("genesis block not found");
+		}
+	}
 
-    const results = await fastify.elastic.search({
-        "index": fastify.manager.chain + '-action-*',
-        "body": {
-            size: 1,
-            query: {
-                bool: {
-                    must: [{term: {"@newaccount.newact": query.account}}]
-                }
-            }
-        }
-    });
+	const results = await fastify.elastic.search({
+		"index": fastify.manager.chain + '-action-*',
+		"body": {
+			size: 1,
+			query: {
+				bool: {
+					must: [{term: {"@newaccount.newact": query.account}}]
+				}
+			}
+		}
+	});
 
-    if (results['body']['hits']['hits'].length === 1) {
-        const result = results['body']['hits']['hits'][0]._source;
-        response.trx_id = result.trx_id;
-        response.block_num = result.block_num;
-        response.creator = result.act.data.creator;
-        response.timestamp = result['@timestamp'];
-        return response;
-    } else {
-        let accountInfo;
-        try {
-            accountInfo = await fastify.eosjs.rpc.get_account(query.account);
-        } catch (e) {
-            throw new Error("account not found");
-        }
-        if (accountInfo) {
-            try {
-                response.timestamp = accountInfo.created;
-                const blockHeader = await fastify.elastic.search({
-                    "index": fastify.manager.chain + '-block-*',
-                    "body": {
-                        size: 1,
-                        query: {
-                            bool: {
-                                must: [{term: {"@timestamp": response.timestamp}}]
-                            }
-                        }
-                    }
-                });
-                const hits = blockHeader['body']['hits']['hits'];
-                if (hits.length > 0 && hits[0]._source) {
-                    const blockId = blockHeader['body']['hits']['hits'][0]._source.block_id;
-                    const blockData = await fastify.eosjs.rpc.get_block(blockId);
-                    response.block_num = blockData.block_num;
-                    for (const transaction of blockData["transactions"]) {
-                        if (typeof transaction.trx !== 'string') {
-                            const actions = transaction.trx.transaction.actions;
-                            for (const act of actions) {
-                                if (act.name === 'newaccount') {
-                                    if (act.data.name === query.account) {
-                                        response.creator = act.data.creator;
-                                        response.trx_id = transaction.id;
-                                        return response;
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-                return response;
-            } catch (e) {
-                console.log(e.message);
-                throw new Error("account creation not found");
-            }
-        } else {
-            throw new Error("account not found");
-        }
-    }
+	if (results['body']['hits']['hits'].length === 1) {
+		const result = results['body']['hits']['hits'][0]._source;
+		response.trx_id = result.trx_id;
+		response.block_num = result.block_num;
+		response.creator = result.act.data.creator;
+		response.timestamp = result['@timestamp'];
+		return response;
+	} else {
+		let accountInfo;
+		try {
+			accountInfo = await fastify.eosjs.rpc.get_account(query.account);
+		} catch (e) {
+			throw new Error("account not found");
+		}
+		if (accountInfo) {
+			try {
+				response.timestamp = accountInfo.created;
+				const blockHeader = await fastify.elastic.search({
+					"index": fastify.manager.chain + '-block-*',
+					"body": {
+						size: 1,
+						query: {
+							bool: {
+								must: [{term: {"@timestamp": response.timestamp}}]
+							}
+						}
+					}
+				});
+				const blockId = blockHeader['body']['hits']['hits'][0]._source.block_id;
+				const blockData = await fastify.eosjs.rpc.get_block(blockId);
+				response.block_num = blockData.block_num;
+				for (const transaction of blockData["transactions"]) {
+					if (typeof transaction.trx !== 'string') {
+						const actions = transaction.trx.transaction.actions;
+						for (const act of actions) {
+							if (act.name === 'newaccount') {
+								if (act.data.name === query.account) {
+									response.creator = act.data.creator;
+									response.trx_id = transaction.id;
+									return response;
+								}
+							}
+						}
+					}
+				}
+				return response;
+			} catch (e) {
+				throw new Error("account creation not found");
+			}
+		} else {
+			throw new Error("account creation not found");
+		}
+	}
 }
 
 export function getCreatorHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply) => {
-        reply.send(await timedQuery(getCreator, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getCreator, fastify, request, route));
+	}
 }

api/routes/v2-history/get_deltas/get_deltas.ts

@@ -3,97 +3,89 @@ import {mergeDeltaMeta, timedQuery} from "../../../helpers/functions";
 import {applyTimeFilter} from "../get_actions/functions";
 
 async function getDeltas(fastify: FastifyInstance, request: FastifyRequest) {
-    let skip, limit;
-    let sort_direction = 'desc';
-    const mustArray = [];
-    const query: any = request.query;
-    for (const param in query) {
-        if (Object.prototype.hasOwnProperty.call(query, param)) {
-            const value = query[param];
-            switch (param) {
-                case 'limit': {
-                    limit = parseInt(value, 10);
-                    if (limit < 1) {
-                        return 'invalid limit parameter';
-                    }
-                    break;
-                }
-                case 'skip': {
-                    skip = parseInt(value, 10);
-                    if (skip < 0) {
-                        return 'invalid skip parameter';
-                    }
-                    break;
-                }
-                case 'sort': {
-                    if (value === 'asc' || value === '1') {
-                        sort_direction = 'asc';
-                    } else if (value === 'desc' || value === '-1') {
-                        sort_direction = 'desc'
-                    } else {
-                        return 'invalid sort direction';
-                    }
-                    break;
-                }
-                case 'before': {
-                    break;
-                }
-                case 'after': {
-                    break;
-                }
-                default: {
-                    if (typeof value === 'string') {
-                        const values = query[param].split(",");
-                        if (values.length > 1) {
-                            const terms = {};
-                            terms[param] = values;
-                            mustArray.push({terms: terms});
-                        } else {
-                            const term = {};
-                            term[param] = values[0];
-                            mustArray.push({term: term});
-                        }
-                    } else {
-                        if (typeof value === 'number') {
-                            const term = {};
-                            term[param] = value;
-                            mustArray.push({term: term});
-                        }
-                    }
-                    break;
-                }
-            }
-        }
-    }
+	let skip, limit;
+	let sort_direction = 'desc';
+	const mustArray = [];
+	const query: any = request.query;
+	for (const param in query) {
+		if (Object.prototype.hasOwnProperty.call(query, param)) {
+			const value = query[param];
+			switch (param) {
+				case 'limit': {
+					limit = parseInt(value, 10);
+					if (limit < 1) {
+						return 'invalid limit parameter';
+					}
+					break;
+				}
+				case 'skip': {
+					skip = parseInt(value, 10);
+					if (skip < 0) {
+						return 'invalid skip parameter';
+					}
+					break;
+				}
+				case 'sort': {
+					if (value === 'asc' || value === '1') {
+						sort_direction = 'asc';
+					} else if (value === 'desc' || value === '-1') {
+						sort_direction = 'desc'
+					} else {
+						return 'invalid sort direction';
+					}
+					break;
+				}
+				case 'before': {
+					break;
+				}
+				case 'after': {
+					break;
+				}
+				default: {
+					const values = query[param].split(",");
+					if (values.length > 1) {
+						const terms = {};
+						terms[param] = values;
+						mustArray.push({terms: terms});
+					} else {
+						const term = {};
+						term[param] = values[0];
+						mustArray.push({term: term});
+					}
+					break;
+				}
+			}
+		}
+	}
 
-    const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
-    const queryStruct = {bool: {must: mustArray}};
+	const maxDeltas = fastify.manager.config.api.limits.get_deltas ?? 1000;
+	const queryStruct = {bool: {must: mustArray}};
 
-    applyTimeFilter(query, queryStruct);
+	applyTimeFilter(query, queryStruct);
 
-    const results = await fastify.elastic.search({
-        "index": fastify.manager.chain + '-delta-*',
-        "from": skip || 0,
-        "size": (limit > maxDeltas ? maxDeltas : limit) || 10,
-        "body": {
-            query: queryStruct,
-            sort: {
-                "block_num": sort_direction
-            }
-        }
-    });
-    const deltas = results['body']['hits']['hits'].map((d) => {
-        return mergeDeltaMeta(d._source);
-    });
-    return {
-        query_time: null,
-        total: results['body']['hits']['total'],
-        deltas
-    };
+	const results = await fastify.elastic.search({
+		"index": fastify.manager.chain + '-delta-*',
+		"from": skip || 0,
+		"size": (limit > maxDeltas ? maxDeltas : limit) || 10,
+		"body": {
+			query: queryStruct,
+			sort: {
+				"block_num": sort_direction
+			}
+		}
+	});
+	const deltas = results['body']['hits']['hits'].map((d) => {
+		return mergeDeltaMeta(d._source);
+	});
+	return {
+		query_time: null,
+		total: results['body']['hits']['total'],
+		deltas
+	};
 }
 
 export function getDeltasHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply) => {
-        reply.send(await timedQuery(getDeltas, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getDeltas, fastify, request, route));
+	}
 }

api/routes/v2-history/get_schedule/get_schedule.ts

@@ -5,101 +5,69 @@ import {base58ToBinary, binaryToBase58} from "eosjs/dist/eosjs-numeric";
 import {Search} from "@elastic/elasticsearch/api/requestParams";
 
 function convertToLegacyKey(block_signing_key: string) {
-    if (block_signing_key.startsWith("PUB_K1_")) {
-        const buf = base58ToBinary(37, block_signing_key.substr(7));
-        const data = buf.slice(0, buf.length - 4);
-        const merged = Buffer.concat([
-            data,
-            createHash('ripemd160')
-                .update(data)
-                .digest()
-                .slice(0, 4)
-        ]);
-        return "EOS" + binaryToBase58(merged);
-    } else {
-        return block_signing_key;
-    }
+	if (block_signing_key.startsWith("PUB_K1_")) {
+		const buf = base58ToBinary(37, block_signing_key.substr(7));
+		const data = buf.slice(0, buf.length - 4);
+		const merged = Buffer.concat([
+			data,
+			createHash('ripemd160')
+				.update(data)
+				.digest()
+				.slice(0, 4)
+		]);
+		return "EOS" + binaryToBase58(merged);
+	} else {
+		return block_signing_key;
+	}
 }
 
 async function getSchedule(fastify: FastifyInstance, request: FastifyRequest) {
-    const query: any = request.query;
-    const response: any = {
-        producers: []
-    };
-    if (query.mode === 'activated') {
-        const searchParams: Search<any> = {
-            track_total_hits: true,
-            index: fastify.manager.chain + "-block-*",
-            size: 1,
-            body: {
-                query: {bool: {must: []}},
-                sort: {block_num: query.sort || "desc"}
-            }
-        };
-        if (query.version) {
-            searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": query.version}}});
-        } else {
-            searchParams.body.query.bool.must.push({"exists": {"field": "new_producers.version"}});
-        }
-        const apiResponse = await fastify.elastic.search(searchParams);
-        const results = apiResponse.body.hits.hits;
-        if (results && results.length > 0) {
-            response.timestamp = results[0]._source["@timestamp"];
-            response.block_num = results[0]._source.block_num;
-            response.version = results[0]._source.new_producers.version;
-            response.producers = results[0]._source.new_producers.producers.map(prod => {
-                return {
-                    ...prod,
-                    legacy_key: convertToLegacyKey(prod.block_signing_key)
-                }
-            });
-        }
-    } else {
-        // default "proposed" mode
-        const searchParams: Search<any> = {
-            track_total_hits: true,
-            index: fastify.manager.chain + "-schedule-*",
-            size: 1,
-            body: {
-                query: {bool: {must: []}},
-                sort: {version: query.sort || "desc"}
-            }
-        };
-        if (query.version) {
-            searchParams.body.query.bool.must.push({"term": {"version": {"value": query.version}}});
-        } else {
-            searchParams.body.query.bool.must.push({"exists": {"field": "version"}});
-        }
-        if (query.producer) {
-            searchParams.body.query.bool.must.push({"term": {"producers.name": {"value": query.producer}}});
-        }
-        if (query.key) {
-            searchParams.body.query.bool.must.push({"term": {"producers.keys": {"value": query.key}}});
-        }
-        const apiResponse = await fastify.elastic.search(searchParams);
-        const results = apiResponse.body.hits.hits;
-        if (results && results.length > 0) {
-            response.timestamp = results[0]._source["@timestamp"];
-            response.proposal_block_num = results[0]._source.block_num;
-            response.version = results[0]._source.version;
-            response.producers = results[0]._source.producers;
-            // sort producers by name
-            // response.producers.sort((a, b) => {
-            //     if (a.name < b.name) {
-            //         return -1;
-            //     }
-            //     if (a.name > b.name) {
-            //         return 1;
-            //     }
-            //     return 0;
-            // });
-        }
-    }
-    return response;
+
+	const query: any = request.query;
+
+	const response: any = {
+		producers: []
+	};
+	const searchParams: Search<any> = {
+		track_total_hits: true,
+		index: fastify.manager.chain + "-block-*",
+		size: 1,
+		body: {
+			query: {
+				bool: {
+					must: []
+				}
+			},
+			sort: {block_num: "desc"}
+		}
+	};
+	if (query.version) {
+		searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": query.version}}});
+	} else {
+		searchParams.body.query.bool.must.push({
+			"exists": {
+				"field": "new_producers.version"
+			}
+		});
+	}
+	const apiResponse = await fastify.elastic.search(searchParams);
+	const results = apiResponse.body.hits.hits;
+	if (results) {
+		response.timestamp = results[0]._source["@timestamp"];
+		response.block_num = results[0]._source.block_num;
+		response.version = results[0]._source.new_producers.version;
+		response.producers = results[0]._source.new_producers.producers.map(prod => {
+			return {
+				...prod,
+				legacy_key: convertToLegacyKey(prod.block_signing_key)
+			}
+		});
+	}
+	return response;
 }
 
 export function getScheduleHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply) => {
-        reply.send(await timedQuery(getSchedule, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getSchedule, fastify, request, route));
+	}
 }

api/routes/v2-history/get_schedule/index.ts

@@ -15,7 +15,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: 'string'
                 },
                 "key": {
-                    description: 'search by signing key',
+                    description: 'search by key',
                     type: 'string'
                 },
                 "after": {
@@ -26,10 +26,6 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     description: 'filter before specified date (ISO8601)',
                     type: 'string'
                 },
-                "mode": {
-                    description: 'search mode (activated or proposed)',
-                    type: 'string',
-                },
                 "version": {
                     description: 'schedule version',
                     type: 'integer',
@@ -40,7 +36,6 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         response: extendResponseSchema({
             "timestamp": {type: "string"},
             "block_num": {type: "number"},
-            "proposed_block_num": {type: "number"},
             "version": {type: "number"},
             "producers": {
                 type: "array",
@@ -48,10 +43,8 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: "object",
                     properties: {
                         "producer_name": {type: "string"},
-                        "name": {type: "string"},
                         "block_signing_key": {type: "string"},
-                        "legacy_key": {type: "string"},
-                        "keys": {type: "array", items: {type: "string"}}
+                        "legacy_key": {type: "string"}
                     }
                 }
             },

api/routes/v2-history/get_transaction_legacy/get_transaction_legacy.ts

@@ -1,128 +0,0 @@
-import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
-
-async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
-    const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
-    const query: any = request.query;
-
-    let indexPattern = fastify.manager.chain + '-action-*';
-    if (query.hot_only) {
-        indexPattern = fastify.manager.chain + '-action';
-    }
-
-    const pResults = await Promise.all([
-        fastify.eosjs.rpc.get_info(),
-        fastify.elastic.search({
-            index: indexPattern,
-            size: _size,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {term: {trx_id: query.id.toLowerCase()}}
-                        ]
-                    }
-                },
-                sort: {
-                    global_sequence: "asc"
-                }
-            }
-        }),
-        fastify.elastic.search({
-            index: fastify.manager.chain + '-gentrx-*',
-            size: _size,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {term: {trx_id: query.id.toLowerCase()}}
-                        ]
-                    }
-                }
-            }
-        })
-    ]);
-
-    const results = pResults[1];
-    const genTrxRes = pResults[2];
-
-    const response = {
-        "executed": false,
-        "hot_only": false,
-        "trx_id": query.id,
-        "lib": pResults[0].last_irreversible_block_num,
-        "actions": [],
-        "generated": undefined
-    };
-
-    if (query.hot_only) {
-        response.hot_only = true;
-    }
-
-    const hits = results['body']['hits']['hits'];
-
-    if (hits.length > 0) {
-
-        // const producers = {};
-        // for (let hit of hits) {
-        // 	if (hit._source.producer) {
-        // 		if (producers[hit._source.producer]) {
-        // 			producers[hit._source.producer]++;
-        // 		} else {
-        // 			producers[hit._source.producer] = 1;
-        // 		}
-        // 	}
-        // }
-        // let useBlockNumber;
-        // if (Object.keys(producers).length > 1) {
-        // 	// multiple producers of the same tx id, forked actions are present, attempt to clean-up
-        // 	let trueProd = '';
-        // 	let highestActCount = 0;
-        // 	for (const prod in producers) {
-        // 		if (producers.hasOwnProperty(prod)) {
-        // 			if(producers[prod] === highestActCount) {
-        // 				useBlockNumber = true;
-        // 			} else if (producers[prod] > highestActCount) {
-        // 				highestActCount = producers[prod];
-        // 				trueProd = prod;
-        // 			}
-        // 		}
-        // 	}
-        // }
-
-        let highestBlockNum = 0;
-        for (let action of hits) {
-            if (action._source.block_num > highestBlockNum) {
-                highestBlockNum = action._source.block_num;
-            }
-        }
-
-        for (let action of hits) {
-            if (action._source.block_num === highestBlockNum) {
-                action = action._source;
-                mergeActionMeta(action);
-                response.actions.push(action);
-            }
-        }
-
-        response.executed = true;
-    }
-
-    const hits2 = genTrxRes['body']['hits']['hits'];
-
-    if (hits2 && hits2.length > 0) {
-        if (hits2[0]._source['@timestamp']) {
-            hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
-            delete hits2[0]._source['@timestamp'];
-        }
-        response.generated = hits2[0]._source;
-    }
-
-    return response;
-}
-
-export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply) => {
-        reply.send(await timedQuery(getTransaction, fastify, request, route));
-    }
-}

api/routes/v2-history/get_transaction_legacy/index.ts

@@ -1,29 +0,0 @@
-import {FastifyInstance} from "fastify";
-import {getTransactionHandler} from "./get_transaction_legacy";
-import {addApiRoute, getRouteName} from "../../../helpers/functions";
-
-export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema = {
-        description: 'get all actions belonging to the same transaction',
-        summary: 'get transaction by id',
-        tags: ['history'],
-        querystring: {
-            type: 'object',
-            properties: {
-                "id": {
-                    description: 'transaction id',
-                    type: 'string'
-                }
-            },
-            required: ["id"]
-        }
-    };
-    addApiRoute(
-        fastify,
-        'GET',
-        getRouteName(__filename),
-        getTransactionHandler,
-        schema
-    );
-    next();
-}

api/routes/v2-state/get_tokens/get_tokens.ts

@@ -5,86 +5,69 @@ import {getSkipLimit} from "../../v2-history/get_actions/functions";
 
 async function getTokens(fastify: FastifyInstance, request: FastifyRequest) {
 
-    const query: any = request.query;
+	const query: any = request.query;
 
-    const response = {
-        account: query.account,
-        tokens: []
-    };
+	const response = {'account': query.account, 'tokens': []};
 
-    const {skip, limit} = getSkipLimit(request.query);
-    const maxDocs = fastify.manager.config.api.limits.get_tokens ?? 100;
+	const {skip, limit} = getSkipLimit(request.query);
+	const maxDocs = fastify.manager.config.api.limits.get_tokens ?? 100;
 
-    const stateResult = await fastify.elastic.search({
-        "index": fastify.manager.chain + '-table-accounts-*',
-        "size": (limit > maxDocs ? maxDocs : limit) || 50,
-        "from": skip || 0,
-        "body": {
-            query: {
-                bool: {
-                    filter: [{term: {scope: query.account}}]
-                }
-            }
-        }
-    });
+	const stateResult = await fastify.elastic.search({
+		"index": fastify.manager.chain + '-table-accounts-*',
+		"size": (limit > maxDocs ? maxDocs : limit) || 50,
+		"from": skip || 0,
+		"body": {
+			query: {
+				bool: {
+					filter: [{term: {"scope": query.account}}]
+				}
+			}
+		}
+	});
 
-    const testSet = new Set();
-    for (const hit of stateResult.body.hits.hits) {
-        const data = hit._source;
-        if (typeof data.present !== "undefined" && data.present === 0) {
-            continue;
-        }
-        let precision;
-        let token_data;
-        let errorMsg;
+	for (const hit of stateResult.body.hits.hits) {
+		const data = hit._source;
+		if (typeof data.present !== "undefined" && data.present === 0) {
+			continue;
+		}
+		let precision;
+		const key = `${data.code}_${data.symbol}`;
+		if (!fastify.tokenCache) {
+			fastify.tokenCache = new Map<string, any>();
+		}
+		if (fastify.tokenCache.has(key)) {
+			precision = fastify.tokenCache.get(key).precision;
+		} else {
+			let token_data;
+			try {
+				token_data = await fastify.eosjs.rpc.get_currency_balance(data.code, query.account, data.symbol);
+				if (token_data.length > 0) {
+					const [amount, symbol] = token_data[0].split(" ");
+					const amount_arr = amount.split(".");
+					if (amount_arr.length === 2) {
+						precision = amount_arr[1].length;
+						fastify.tokenCache.set(key, {precision});
+						// console.log('Caching token precision -', key, precision);
+					}
+				}
+			} catch (e) {
+				console.log(`get_currency_balance error - contract:${data.code} - account:${query.account}`);
+			}
+		}
 
-        const key = `${data.code}_${data.symbol}`;
+		response.tokens.push({
+			symbol: data.symbol,
+			precision: precision,
+			amount: parseFloat(data.amount),
+			contract: data.code
+		});
+	}
 
-        if (testSet.has(key)) {
-            continue;
-        }
-
-        testSet.add(key);
-        if (!fastify.tokenCache) {
-            fastify.tokenCache = new Map<string, any>();
-        }
-        if (fastify.tokenCache.has(key)) {
-            precision = fastify.tokenCache.get(key).precision;
-        } else {
-            try {
-                token_data = await fastify.eosjs.rpc.get_currency_balance(data.code, query.account, data.symbol);
-                if (token_data.length > 0) {
-                    const [amount, symbol] = token_data[0].split(" ");
-                    const amount_arr = amount.split(".");
-                    if (amount_arr.length === 2) {
-                        precision = amount_arr[1].length;
-                        fastify.tokenCache.set(key, {precision});
-                    }
-                }
-            } catch (e) {
-                errorMsg = e.message;
-            }
-        }
-
-        const resp: Record<string, any> = {
-            symbol: data.symbol,
-            precision: precision,
-            amount: parseFloat(data.amount),
-            contract: data.code,
-        };
-
-        if (errorMsg) {
-            resp.error = errorMsg;
-        }
-
-        response.tokens.push(resp);
-    }
-
-    return response;
+	return response;
 }
 
 export function getTokensHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply) => {
-        reply.send(await timedQuery(getTokens, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getTokens, fastify, request, route));
+	}
 }

api/routes/v2-stats/get_api_usage/get_api_usage.ts

@@ -1,53 +0,0 @@
-import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
-import {timedQuery} from "../../../helpers/functions";
-import {parseInt} from "lodash";
-
-async function getApiUsage(fastify: FastifyInstance, request: FastifyRequest) {
-    const response = {
-        total: {responses: {}},
-        buckets: []
-    } as any;
-    const data = await fastify.redis.keys(`stats:${fastify.manager.chain}:*`);
-    const buckets = [];
-    for (const key of data) {
-        const parts = key.split(':');
-        if (parts[2] === 'H') {
-            const bucket = {
-                timestamp: new Date(parseInt(parts[3])),
-                responses: {}
-            };
-            const sortedSet = await fastify.redis.zrange(key, 0, -1, 'WITHSCORES');
-            for (let i = 0; i < sortedSet.length; i += 2) {
-                const [status, path] = sortedSet[i].split(']');
-                if (status) {
-                    const statusCode = status.slice(1);
-                    if (!bucket.responses[statusCode]) {
-                        bucket.responses[statusCode] = {};
-                    }
-                    if (statusCode) {
-                        bucket.responses[statusCode][path] = parseInt(sortedSet[i + 1]);
-                    }
-                }
-            }
-            response.buckets.push(bucket);
-        } else {
-            if (parts[2] === 'total' && parts[4]) {
-                const statusCode = parts[3];
-                if (!response.total.responses[statusCode]) {
-                    response.total.responses[statusCode] = {};
-                }
-                const value = await fastify.redis.get(key);
-                response.total.responses[statusCode][parts[4]] = parseInt(value);
-            }
-        }
-    }
-    // reverse sorte by date
-    response.buckets.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
-    return response;
-}
-
-export function getApiUsageHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply) => {
-        reply.send(await timedQuery(getApiUsage, fastify, request, route));
-    }
-}

api/routes/v2-stats/get_api_usage/index.ts

@@ -1,41 +0,0 @@
-import {FastifyInstance} from "fastify";
-import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
-import {getApiUsageHandler} from "./get_api_usage";
-
-export default function (fastify: FastifyInstance, opts: any, next) {
-    const schema = {
-        description: 'get hyperion api usage statistics',
-        summary: 'get hyperion api usage statistics',
-        tags: ['stats'],
-        response: extendResponseSchema({
-            total: {
-                type: 'object',
-                properties: {
-                    responses: {
-                        additionalProperties: true
-                    }
-                }
-            },
-            buckets: {
-                type: 'array',
-                items: {
-                    type: 'object',
-                    properties: {
-                        timestamp: {type: 'string'},
-                        responses: {
-                            additionalProperties: true
-                        }
-                    }
-                }
-            }
-        })
-    };
-    addApiRoute(
-        fastify,
-        'GET',
-        getRouteName(__filename),
-        getApiUsageHandler,
-        schema
-    );
-    next();
-}

api/routes/v2/health/health.ts

@@ -1,144 +1,90 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {connect} from "amqplib";
 import {timedQuery} from "../../../helpers/functions";
-import {getFirstIndexedBlock, getLastIndexedBlockWithTotalBlocks} from "../../../../helpers/common_functions";
+import {getLastIndexedBlockWithTotalBlocks} from "../../../../helpers/common_functions";
 
-
-async function checkRabbit(fastify: FastifyInstance): Promise<ServiceResponse<any>> {
-    try {
-        const connection = await connect(fastify.manager.ampqUrl);
-        await connection.close();
-        return createHealth('RabbitMq', 'OK');
-    } catch (e) {
-        console.log(e);
-        return createHealth('RabbitMq', 'Error');
-    }
+async function checkRabbit(fastify: FastifyInstance) {
+	try {
+		const connection = await connect(fastify.manager.ampqUrl);
+		await connection.close();
+		return createHealth('RabbitMq', 'OK');
+	} catch (e) {
+		console.log(e);
+		return createHealth('RabbitMq', 'Error');
+	}
 }
 
-interface NodeosService {
-    head_block_num: number;
-    head_block_time: string;
-    time_offset: number;
-    last_irreversible_block: number;
-    chain_id: string;
+async function checkNodeos(fastify: FastifyInstance) {
+	const rpc = fastify.manager.nodeosJsonRPC;
+	try {
+		const results = await rpc.get_info();
+		if (results) {
+			const diff = (new Date().getTime()) - (new Date(results.head_block_time + '+00:00').getTime());
+			return createHealth('NodeosRPC', 'OK', {
+				head_block_num: results.head_block_num,
+				head_block_time: results.head_block_time,
+				time_offset: diff,
+				last_irreversible_block: results.last_irreversible_block_num,
+				chain_id: results.chain_id
+			});
+		} else {
+			return createHealth('NodeosRPC', 'Error');
+		}
+	} catch (e) {
+		return createHealth('NodeosRPC', 'Error');
+	}
 }
 
-async function checkNodeos(fastify: FastifyInstance): Promise<ServiceResponse<NodeosService>> {
-    const rpc = fastify.manager.nodeosJsonRPC;
-    try {
-        const results = await rpc.get_info();
-        if (results) {
-            const diff = (new Date().getTime()) - (new Date(results.head_block_time + '+00:00').getTime());
-            return createHealth('NodeosRPC', 'OK', {
-                head_block_num: results.head_block_num,
-                head_block_time: results.head_block_time,
-                time_offset: diff,
-                last_irreversible_block: results.last_irreversible_block_num,
-                chain_id: results.chain_id
-            });
-        } else {
-            return createHealth('NodeosRPC', 'Error');
-        }
-    } catch (e) {
-        return createHealth('NodeosRPC', 'Error');
-    }
+async function checkElastic(fastify: FastifyInstance) {
+	try {
+		let esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
+		let indexedBlocks = await getLastIndexedBlockWithTotalBlocks(fastify.elastic, fastify.manager.chain);
+		const data = {
+			last_indexed_block: indexedBlocks[0],
+			total_indexed_blocks: indexedBlocks[1],
+			active_shards: esStatus.body[0]['active_shards_percent']
+		};
+		let stat = 'OK';
+		esStatus.body.forEach(status => {
+			if (status.status === 'yellow' && stat !== 'Error') {
+				stat = 'Warning'
+			} else if (status.status === 'red') {
+				stat = 'Error'
+			}
+		});
+		return createHealth('Elasticsearch', stat, data);
+	} catch (e) {
+		console.log(e, 'Elasticsearch Error');
+		return createHealth('Elasticsearch', 'Error');
+	}
 }
 
-interface ESService {
-    first_indexed_block: number;
-    last_indexed_block: number;
-    total_indexed_blocks: number;
-    active_shards: string;
-    missing_blocks: number;
-    missing_pct: string;
-    head_offset: number;
+function createHealth(name: string, status, data?: any) {
+	let time = Date.now();
+	return {
+		service: name,
+		status: status,
+		service_data: data,
+		time: time
+	}
 }
 
-interface ServiceResponse<T> {
-    service: string;
-    time: number;
-    status: any
-    service_data?: T;
-}
-
-async function checkElastic(fastify: FastifyInstance): Promise<ServiceResponse<ESService>> {
-    try {
-        const esStatusCache = await fastify.redis.get(`${fastify.manager.chain}::es_status`);
-        let esStatus;
-        if (esStatusCache) {
-            esStatus = JSON.parse(esStatusCache);
-        } else {
-            esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
-            await fastify.redis.set(`${fastify.manager.chain}::es_status`, JSON.stringify(esStatus), 'EX', 30 * 60);
-        }
-        let firstIndexedBlock: number;
-        const fib = await fastify.redis.get(`${fastify.manager.chain}::fib`);
-        if (fib) {
-            firstIndexedBlock = parseInt(fib);
-        } else {
-            firstIndexedBlock = await getFirstIndexedBlock(fastify.elastic, fastify.manager.chain);
-            await fastify.redis.set(`${fastify.manager.chain}::fib`, firstIndexedBlock);
-        }
-        let indexedBlocks = await getLastIndexedBlockWithTotalBlocks(fastify.elastic, fastify.manager.chain);
-        const lastIndexedBlock = indexedBlocks[0];
-        const totalIndexed = indexedBlocks[1] - 1;
-        const missingCounter = (lastIndexedBlock - firstIndexedBlock) - totalIndexed;
-        const missingPct = (missingCounter * 100 / indexedBlocks[1]).toFixed(2) + "%";
-        const data: ESService = {
-            active_shards: esStatus.body[0]['active_shards_percent'],
-            head_offset: null,
-            first_indexed_block: firstIndexedBlock,
-            last_indexed_block: lastIndexedBlock,
-            total_indexed_blocks: totalIndexed,
-            missing_blocks: missingCounter,
-            missing_pct: missingPct
-        };
-        let stat = 'OK';
-        esStatus.body.forEach(status => {
-            if (status.status === 'yellow' && stat !== 'Error') {
-                stat = 'Warning'
-            } else if (status.status === 'red') {
-                stat = 'Error'
-            }
-        });
-        return createHealth('Elasticsearch', stat, data);
-    } catch (e) {
-        console.log(e, 'Elasticsearch Error');
-        return createHealth('Elasticsearch', 'Error');
-    }
-}
-
-function createHealth(name: string, status, data?: any): ServiceResponse<any> {
-    let time = Date.now();
-    return {
-        service: name,
-        status: status,
-        service_data: data,
-        time: time
-    }
-}
-
-async function getHealthQuery(fastify: FastifyInstance) {
-    let response = {
-        version: fastify.manager.current_version,
-        version_hash: fastify.manager.getServerHash(),
-        host: fastify.manager.config.api.server_name,
-        health: [],
-        features: fastify.manager.config.features
-    };
-    response.health = await Promise.all([
-        checkRabbit(fastify),
-        checkNodeos(fastify),
-        checkElastic(fastify)
-    ]);
-    const es = response.health.find(value => value.service === 'Elasticsearch');
-    const nodeos = response.health.find(value => value.service === 'NodeosRPC');
-    es.service_data.head_offset = nodeos.service_data.head_block_num - es.service_data.last_indexed_block;
-    return response;
+async function getHealthQuery(fastify: FastifyInstance, request: FastifyRequest) {
+	let response = {
+		version: fastify.manager.current_version,
+		version_hash: fastify.manager.getServerHash(),
+		host: fastify.manager.config.api.server_name,
+		health: [],
+		features: fastify.manager.config.features
+	};
+	response.health.push(await checkRabbit(fastify));
+	response.health.push(await checkNodeos(fastify));
+	response.health.push(await checkElastic(fastify));
+	return response;
 }
 
 export function healthHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply) => {
-        reply.send(await timedQuery(getHealthQuery, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply) => {
+		reply.send(await timedQuery(getHealthQuery, fastify, request, route));
+	}
 }

api/server.ts

@@ -1,4 +1,4 @@
-import {hLog, waitUntilReady} from "../helpers/common_functions";
+import {hLog} from "../helpers/common_functions";
 import {ConfigurationModule} from "../modules/config";
 import {ConnectionManager} from "../connections/manager.class";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
@@ -8,302 +8,239 @@ import {registerPlugins} from "./plugins";
 import {AddressInfo} from "net";
 import {registerRoutes} from "./routes";
 import {generateOpenApiConfig} from "./config/open_api";
-import {createWriteStream, existsSync, mkdirSync, readFileSync} from "fs";
+import {createWriteStream, existsSync, mkdirSync} from "fs";
 import {SocketManager} from "./socketManager";
 import {HyperionModuleLoader} from "../modules/loader";
 import {extendedActions} from "./routes/v2-history/get_actions/definitions";
-import {io, Socket} from "socket.io-client";
-import {CacheManager} from "./helpers/cacheManager";
-
-import {bootstrap} from 'global-agent';
+import {Socket, io} from "socket.io-client";
 
 class HyperionApiServer {
 
-    private hub: Socket;
-    private readonly fastify;
-    private readonly chain: string;
-    private readonly conf: HyperionConfig;
-    private readonly manager: ConnectionManager;
-    private readonly cacheManager: CacheManager;
+	private conf: HyperionConfig;
+	private readonly manager: ConnectionManager;
+	mLoader: HyperionModuleLoader;
+	private readonly fastify;
+	private readonly chain: string;
+	socketManager: SocketManager;
 
-    socketManager: SocketManager;
-    mLoader: HyperionModuleLoader;
-    private pluginParams: any;
+	private hub: Socket;
 
-    constructor() {
+	constructor() {
+		const cm = new ConfigurationModule();
+		this.conf = cm.config;
+		this.chain = this.conf.settings.chain;
+		process.title = `hyp-${this.chain}-api`;
+		this.manager = new ConnectionManager(cm);
+		this.manager.calculateServerHash();
+		this.mLoader = new HyperionModuleLoader(cm);
 
-        const package_json = JSON.parse(readFileSync('./package.json').toString());
-        hLog(`--------- Hyperion API ${package_json.version} ---------`);
+		if (!existsSync('./logs/' + this.chain)) {
+			mkdirSync('./logs/' + this.chain, {recursive: true});
+		}
 
-        const cm = new ConfigurationModule();
-        this.conf = cm.config;
+		const logStream = createWriteStream('./logs/' + this.chain + '/api.access.log');
 
-        if (this.conf.settings.use_global_agent) {
-            bootstrap();
-        }
+		const loggerOpts = {
+			stream: logStream,
+			redact: ['req.headers.authorization'],
+			level: 'info',
+			prettyPrint: true,
+			serializers: {
+				res: (reply) => {
+					return {
+						statusCode: reply.statusCode
+					};
+				},
+				req: (request) => {
+					return {
+						method: request.method,
+						url: request.url,
+						ip: request.headers['x-real-ip']
+					}
+				}
+			}
+		};
 
-        this.chain = this.conf.settings.chain;
-        process.title = `hyp-${this.chain}-api`;
-        this.manager = new ConnectionManager(cm);
-        this.manager.calculateServerHash();
-        this.mLoader = new HyperionModuleLoader(cm);
-        this.cacheManager = new CacheManager(this.conf);
+		this.fastify = fastify({
+			ignoreTrailingSlash: false,
+			trustProxy: true,
+			pluginTimeout: 5000,
+			logger: this.conf.api.access_log ? loggerOpts : false
+		});
 
-        if (!existsSync('./logs/' + this.chain)) {
-            mkdirSync('./logs/' + this.chain, {recursive: true});
-        }
+		this.fastify.decorate('manager', this.manager);
 
-        const logStream = createWriteStream('./logs/' + this.chain + '/api.access.log');
+		// import get_actions query params from custom modules
+		const extendedActionsSet: Set<string> = new Set([...extendedActions]);
+		for (const qPrefix of this.mLoader.extendedActions) {
+			extendedActionsSet.add(qPrefix);
+		}
+		this.fastify.decorate('allowedActionQueryParamSet', extendedActionsSet);
 
-        const loggerOpts = {
-            stream: logStream,
-            redact: ['req.headers.authorization'],
-            level: 'info',
-            prettyPrint: true,
-            serializers: {
-                res: (reply) => {
-                    return {
-                        statusCode: reply.statusCode
-                    };
-                },
-                req: (request) => {
-                    return {
-                        method: request.method,
-                        url: request.url,
-                        ip: request.headers['x-real-ip']
-                    }
-                }
-            }
-        };
+		// define chain api url for /v1/chain/ redirects
+		let chainApiUrl: string = this.conf.api.push_api;
+		if (chainApiUrl === null || chainApiUrl === "") {
+			chainApiUrl = this.manager.conn.chains[this.chain].http;
+		}
+		this.fastify.decorate('chain_api', chainApiUrl);
 
-        this.fastify = fastify({
-            ignoreTrailingSlash: false,
-            trustProxy: true,
-            pluginTimeout: 5000,
-            logger: this.conf.api.access_log ? loggerOpts : false
-        });
+		// define optional push api url for /v1/chain/push_transaction
+		if (this.conf.api.push_api) {
+			this.fastify.decorate('push_api', this.conf.api.push_api);
+		}
 
-        this.fastify.decorate('cacheManager', this.cacheManager);
+		hLog(`Chain API URL: "${this.fastify.chain_api}" | Push API URL: "${this.fastify.push_api}"`);
 
-        this.fastify.decorate('manager', this.manager);
+		const ioRedisClient = new IORedis(this.manager.conn.redis);
 
-        // import get_actions query params from custom modules
-        const extendedActionsSet: Set<string> = new Set([...extendedActions]);
-        for (const qPrefix of this.mLoader.extendedActions) {
-            extendedActionsSet.add(qPrefix);
-        }
-        this.fastify.decorate('allowedActionQueryParamSet', extendedActionsSet);
+		let rateLimiterWhitelist = ['127.0.0.1'];
 
-        // define chain api url for /v1/chain/ redirects
-        let chainApiUrl: string = this.conf.api.chain_api;
-        if (chainApiUrl === null || chainApiUrl === "") {
-            chainApiUrl = this.manager.conn.chains[this.chain].http;
-        }
-        this.fastify.decorate('chain_api', chainApiUrl);
+		if (this.conf.api.rate_limit_allow && this.conf.api.rate_limit_allow.length > 0) {
+			const tempSet = new Set<string>(['127.0.0.1', ...this.conf.api.rate_limit_allow]);
+			rateLimiterWhitelist = [...tempSet];
+		}
 
-        // define optional push api url for /v1/chain/push_transaction
-        if (this.conf.api.push_api) {
-            this.fastify.decorate('push_api', this.conf.api.push_api);
-        }
+		let rateLimiterRPM = 1000;
+		if (this.conf.api.rate_limit_rpm) {
+			rateLimiterRPM = this.conf.api.rate_limit_rpm;
+		}
 
-        hLog(`Chain API URL: "${this.fastify.chain_api}" | Push API URL: "${this.fastify.push_api}"`);
+		const api_rate_limit = {
+			max: rateLimiterRPM,
+			allowList: rateLimiterWhitelist,
+			timeWindow: '1 minute',
+			redis: ioRedisClient
+		};
 
-        const ioRedisClient = new IORedis(this.manager.conn.redis);
+		if (this.conf.features.streaming.enable) {
+			this.activateStreaming();
+		}
 
-        const pluginParams = {
-            fastify_elasticsearch: {
-                client: this.manager.elasticsearchClient
-            },
-            fastify_redis: this.manager.conn.redis,
-            fastify_eosjs: this.manager,
-            chain_id: '',
-        } as any;
+		const docsConfig = generateOpenApiConfig(this.manager.config);
 
-        if (!this.conf.api.disable_rate_limit) {
-            let rateLimiterWhitelist = ['127.0.0.1'];
-            if (this.conf.api.rate_limit_allow && this.conf.api.rate_limit_allow.length > 0) {
-                const tempSet = new Set<string>(['127.0.0.1', ...this.conf.api.rate_limit_allow]);
-                rateLimiterWhitelist = [...tempSet];
-            }
-            let rateLimiterRPM = 1000;
-            if (this.conf.api.rate_limit_rpm) {
-                rateLimiterRPM = this.conf.api.rate_limit_rpm;
-            }
-            pluginParams.fastify_rate_limit = {
-                max: rateLimiterRPM,
-                allowList: rateLimiterWhitelist,
-                timeWindow: '1 minute',
-                redis: ioRedisClient
-            }
-        }
+		registerPlugins(this.fastify, {
+			fastify_elasticsearch: {client: this.manager.elasticsearchClient},
+			fastify_swagger: docsConfig,
+			fastify_rate_limit: api_rate_limit,
+			fastify_redis: this.manager.conn.redis,
+			fastify_eosjs: this.manager,
+		});
 
-        if (this.conf.features.streaming.enable) {
-            this.activateStreaming();
-        }
+		this.addGenericTypeParsing();
+	}
 
-        const docsConfig = generateOpenApiConfig(this.manager.config);
-        if (docsConfig) {
-            pluginParams.fastify_swagger = docsConfig;
-        }
+	activateStreaming() {
+		console.log('Importing stream module');
+		import('./socketManager').then((mod) => {
+			const connOpts = this.manager.conn.chains[this.chain];
 
-        this.pluginParams = pluginParams;
-    }
+			let _port = 57200;
+			if (connOpts.WS_ROUTER_PORT) {
+				_port = connOpts.WS_ROUTER_PORT;
+			}
 
-    activateStreaming() {
-        console.log('Importing stream module...');
-        import('./socketManager').then((mod) => {
-            const connOpts = this.manager.conn.chains[this.chain];
-            let _port = 57200;
-            if (connOpts.WS_ROUTER_PORT) {
-                _port = connOpts.WS_ROUTER_PORT;
-            }
-            let _host = "127.0.0.1";
-            if (connOpts.WS_ROUTER_HOST) {
-                _host = connOpts.WS_ROUTER_HOST;
-            }
-            if (_host === "0.0.0.0") {
-                hLog(`[ERROR] WS Router Host is set to 0.0.0.0, please use a fixed IP address instead. Can't start streaming.`);
-                return;
-            }
-            this.socketManager = new mod.SocketManager(
-                this.fastify,
-                `http://${_host}:${_port}`,
-                this.manager.conn.redis
-            );
-            this.socketManager.startRelay();
-        });
-    }
+			let _host = "127.0.0.1";
+			if (connOpts.WS_ROUTER_HOST) {
+				_host = connOpts.WS_ROUTER_HOST;
+			}
 
-    private addGenericTypeParsing() {
-        this.fastify.addContentTypeParser('*', (request, payload, done) => {
-            let data = '';
-            payload.on('data', chunk => {
-                data += chunk;
-            });
-            payload.on('end', () => {
-                done(null, data);
-            });
-            payload.on('error', (err) => {
-                console.log('---- Content Parsing Error -----');
-                console.log(err);
-            });
-        });
-    }
+			this.socketManager = new mod.SocketManager(
+				this.fastify,
+				`http://${_host}:${_port}`,
+				this.manager.conn.redis
+			);
+			this.socketManager.startRelay();
+		});
+	}
 
-    async init() {
+	private addGenericTypeParsing() {
+		this.fastify.addContentTypeParser('*', (request, payload, done) => {
+			let data = '';
+			payload.on('data', chunk => {
+				data += chunk;
+			});
+			payload.on('end', () => {
+				done(null, data);
+			});
+			payload.on('error', (err) => {
+				console.log('---- Content Parsing Error -----');
+				console.log(err);
+			});
+		});
+	}
 
-        const rpc = this.manager.nodeosJsonRPC;
-        await waitUntilReady(async () => {
-            try {
-                const chain_data = await rpc.get_info();
-                if (chain_data && chain_data.chain_id) {
-                    this.pluginParams.chain_id = chain_data.chain_id;
-                    return true;
-                } else {
-                    return false;
-                }
-            } catch (e) {
-                hLog(e.message);
-                return false;
-            }
-        }, 20, 5000, () => {
-            hLog('Failed to validate chain api!');
-            process.exit(1);
-        });
-        hLog('Chain API validated!');
+	async init() {
 
-        // Wait for Elasticsearch availability
-        await waitUntilReady(async () => {
-            try {
-                const esInfo = await this.manager.elasticsearchClient.info();
-                hLog(`Elasticsearch: ${esInfo.body.version.number} | Lucene: ${esInfo.body.version.lucene_version}`);
-                return true;
-            } catch (e) {
-                console.log(e.message);
-                return false;
-            }
-        }, 10, 5000, () => {
-            hLog('Failed to check elasticsearch version!');
-            process.exit();
-        });
+		await this.mLoader.init();
 
-        hLog('Elasticsearch validated!');
-        hLog('Registering plugins...');
+		// add custom plugin routes
+		for (const plugin of this.mLoader.plugins) {
+			if (plugin.hasApiRoutes) {
+				plugin.addRoutes(this.fastify);
+				plugin.chainName = this.chain;
+			}
+		}
 
-        registerPlugins(this.fastify, this.pluginParams);
-        this.addGenericTypeParsing();
+		registerRoutes(this.fastify);
 
-        await this.mLoader.init();
+		// register documentation when ready
+		this.fastify.ready().then(async () => {
+			await this.fastify.swagger();
+		}, (err) => {
+			hLog('an error happened', err)
+		});
 
-        // add custom plugin routes
-        for (const plugin of this.mLoader.plugins) {
-            if (plugin.hasApiRoutes) {
-                hLog(`Adding routes for plugin: ${plugin.internalPluginName}`);
-                plugin.addRoutes(this.fastify);
-                plugin.chainName = this.chain;
-            }
-        }
+		try {
+			await this.fastify.listen({
+				host: this.conf.api.server_addr,
+				port: this.conf.api.server_port
+			});
+			hLog(`${this.chain} hyperion api ready and listening on port ${(this.fastify.server.address() as AddressInfo).port}`);
+			this.startHyperionHub();
+		} catch (err) {
+			hLog(err);
+			process.exit(1)
+		}
+	}
 
-        registerRoutes(this.fastify);
+	startHyperionHub() {
+		if (this.conf.hub) {
+			const url = this.conf.hub.inform_url;
+			hLog(`Connecting API to Hyperion Hub`);
+			this.hub = io(url, {
+				query: {
+					key: this.conf.hub.publisher_key,
+					client_mode: 'false'
+				}
+			});
+			this.hub.on('connect', () => {
+				hLog(`Hyperion Hub connected!`);
+				this.emitHubApiUpdate();
+			});
+		}
+	}
 
-        // register documentation when ready
-        this.fastify.ready().then(async () => {
-            await this.fastify.swagger();
-        }, (err) => {
-            hLog('an error happened', err)
-        });
-
-        try {
-            await this.fastify.listen({
-                host: this.conf.api.server_addr,
-                port: this.conf.api.server_port
-            });
-            hLog(`${this.chain} hyperion api ready and listening on port ${(this.fastify.server.address() as AddressInfo).port}`);
-            this.startHyperionHub();
-        } catch (err) {
-            hLog(err);
-            process.exit(1)
-        }
-    }
-
-    startHyperionHub() {
-        if (this.conf.hub) {
-            const url = this.conf.hub.inform_url;
-            hLog(`Connecting API to Hyperion Hub`);
-            this.hub = io(url, {
-                query: {
-                    key: this.conf.hub.publisher_key,
-                    client_mode: 'false'
-                }
-            });
-            this.hub.on('connect', () => {
-                hLog(`Hyperion Hub connected!`);
-                this.emitHubApiUpdate();
-            });
-        }
-    }
-
-    private emitHubApiUpdate() {
-        this.hub.emit('hyp_info', {
-            type: 'api',
-            production: this.conf.hub.production,
-            location: this.conf.hub.location,
-            chainId: this.manager.conn.chains[this.chain].chain_id,
-            providerName: this.conf.api.provider_name,
-            explorerEnabled: this.conf.plugins.explorer?.enabled,
-            providerUrl: this.conf.api.provider_url,
-            providerLogo: this.conf.api.provider_logo,
-            chainLogo: this.conf.api.chain_logo_url,
-            chainCodename: this.chain,
-            chainName: this.conf.api.chain_name,
-            endpoint: this.conf.api.server_name,
-            features: this.conf.features,
-            filters: {
-                blacklists: this.conf.blacklists,
-                whitelists: this.conf.whitelists
-            }
-        });
-    }
+	private emitHubApiUpdate() {
+		this.hub.emit('hyp_info', {
+			type: 'api',
+			production: this.conf.hub.production,
+			location: this.conf.hub.location,
+			chainId: this.manager.conn.chains[this.chain].chain_id,
+			providerName: this.conf.api.provider_name,
+			explorerEnabled: this.conf.api.enable_explorer,
+			providerUrl: this.conf.api.provider_url,
+			providerLogo: this.conf.api.provider_logo,
+			chainLogo: this.conf.api.chain_logo_url,
+			chainCodename: this.chain,
+			chainName: this.conf.api.chain_name,
+			endpoint: this.conf.api.server_name,
+			features: this.conf.features,
+			filters: {
+				blacklists: this.conf.blacklists,
+				whitelists: this.conf.whitelists
+			}
+		});
+	}
 }
 
 const server = new HyperionApiServer();

api/socketManager.ts

@@ -1,292 +1,384 @@
-import {hLog, sleep} from '../helpers/common_functions';
+import {checkFilter, hLog} from '../helpers/common_functions';
 import {Server, Socket} from 'socket.io';
 import {createAdapter} from 'socket.io-redis';
 import {io} from 'socket.io-client';
 import {FastifyInstance} from "fastify";
 import IORedis from "ioredis";
-import {App, TemplatedApp} from 'uWebSockets.js';
-import {streamPastActions, streamPastDeltas} from "./helpers/functions";
-import {randomUUID} from "crypto";
 
 export interface StreamDeltasRequest {
-    code: string;
-    table: string;
-    scope: string;
-    payer: string;
-    start_from: number | string;
-    read_until: number | string;
+	code: string;
+	table: string;
+	scope: string;
+	payer: string;
+	start_from: number | string;
+	read_until: number | string;
 }
 
 export interface RequestFilter {
-    field: string;
-    value: string;
+	field: string;
+	value: string;
 }
 
 export interface StreamActionsRequest {
-    contract: string;
-    account: string;
-    action: string;
-    filters: RequestFilter[];
-    start_from: number | string;
-    read_until: number | string;
+	contract: string;
+	account: string;
+	action: string;
+	filters: RequestFilter[];
+	start_from: number | string;
+	read_until: number | string;
 }
 
+async function addBlockRangeOpts(data, search_body, fastify: FastifyInstance) {
+
+	let timeRange;
+	let blockRange;
+	let head;
+
+	if (typeof data['start_from'] === 'string' && data['start_from'] !== '') {
+		if (!timeRange) {
+			timeRange = {"@timestamp": {}};
+		}
+		timeRange["@timestamp"]['gte'] = data['start_from'];
+	}
+
+	if (typeof data['read_until'] === 'string' && data['read_until'] !== '') {
+		if (!timeRange) {
+			timeRange = {"@timestamp": {}};
+		}
+		timeRange["@timestamp"]['lte'] = data['read_until'];
+	}
+
+	if (typeof data['start_from'] === 'number' && data['start_from'] !== 0) {
+		if (!blockRange) {
+			blockRange = {"block_num": {}};
+		}
+		if (data['start_from'] < 0) {
+			if (!head) {
+				head = (await fastify.eosjs.rpc.get_info()).head_block_num;
+			}
+			blockRange["block_num"]['gte'] = head + data['start_from'];
+		} else {
+			blockRange["block_num"]['gte'] = data['start_from'];
+		}
+	}
+
+	if (typeof data['read_until'] === 'number' && data['read_until'] !== 0) {
+		if (!blockRange) {
+			blockRange = {"block_num": {}};
+		}
+		if (data['read_until'] < 0) {
+			if (!head) {
+				head = (await fastify.eosjs.rpc.get_info()).head_block_num;
+			}
+			blockRange["block_num"]['lte'] = head + data['read_until'];
+		} else {
+			blockRange["block_num"]['lte'] = data['read_until'];
+		}
+	}
+
+	if (timeRange) {
+		search_body.query.bool.must.push({
+			range: timeRange,
+		});
+	}
+	if (blockRange) {
+		search_body.query.bool.must.push({
+			range: blockRange,
+		});
+	}
+}
+
+function addTermMatch(data, search_body, field) {
+	if (data[field] !== '*' && data[field] !== '') {
+		const termQuery = {};
+		termQuery[field] = data[field];
+		search_body.query.bool.must.push({'term': termQuery});
+	}
+}
+
+const deltaQueryFields = ['code', 'table', 'scope', 'payer'];
+
+async function streamPastDeltas(fastify: FastifyInstance, socket, data) {
+	const search_body = {
+		query: {bool: {must: []}},
+		sort: {block_num: 'asc'},
+	};
+	await addBlockRangeOpts(data, search_body, fastify);
+	deltaQueryFields.forEach(f => {
+		addTermMatch(data, search_body, f);
+	});
+	const responseQueue = [];
+	let counter = 0;
+	const init_response = await fastify.elastic.search({
+		index: fastify.manager.chain + '-delta-*',
+		scroll: '30s',
+		size: 20,
+		body: search_body,
+	});
+	responseQueue.push(init_response);
+	while (responseQueue.length) {
+		const {body} = responseQueue.shift();
+		counter += body['hits']['hits'].length;
+		if (socket.connected) {
+			socket.emit('message', {
+				type: 'delta_trace',
+				mode: 'history',
+				messages: body['hits']['hits'].map(doc => doc._source),
+			});
+		} else {
+			hLog('LOST CLIENT');
+			break;
+		}
+		if (body['hits'].total.value === counter) {
+			hLog(`${counter} past deltas streamed to ${socket.id}`);
+			break;
+		}
+
+		const next_response = await fastify.elastic.scroll({
+			body: {
+				scroll_id: body['_scroll_id'],
+				scroll: '30s'
+			}
+		});
+		responseQueue.push(next_response);
+	}
+}
+
+async function streamPastActions(fastify: FastifyInstance, socket, data) {
+	const search_body = {
+		query: {bool: {must: []}},
+		sort: {global_sequence: 'asc'},
+	};
+
+	await addBlockRangeOpts(data, search_body, fastify);
+
+	if (data.account !== '') {
+		search_body.query.bool.must.push({
+			bool: {
+				should: [
+					{term: {'notified': data.account}},
+					{term: {'act.authorization.actor': data.account}},
+				],
+			},
+		});
+	}
+
+	if (data.contract !== '*' && data.contract !== '') {
+		search_body.query.bool.must.push({'term': {'act.account': data.contract}});
+	}
+
+	if (data.action !== '*' && data.action !== '') {
+		search_body.query.bool.must.push({'term': {'act.name': data.action}});
+	}
+
+	const onDemandFilters = [];
+	if (data.filters.length > 0) {
+		data.filters.forEach(f => {
+			if (f.field && f.value) {
+				if (f.field.startsWith('@') && !f.field.startsWith('act.data')) {
+					const _q = {};
+					_q[f.field] = f.value;
+					search_body.query.bool.must.push({'term': _q});
+				} else {
+					onDemandFilters.push(f);
+				}
+			}
+		});
+	}
+
+	const responseQueue = [];
+	let counter = 0;
+
+	const init_response = await fastify.elastic.search({
+		index: fastify.manager.chain + '-action-*',
+		scroll: '30s',
+		size: 20,
+		body: search_body,
+	});
+	responseQueue.push(init_response);
+	while (responseQueue.length) {
+		const {body} = responseQueue.shift();
+		const enqueuedMessages = [];
+		counter += body['hits']['hits'].length;
+		for (const doc of body['hits']['hits']) {
+			let allow = false;
+			if (onDemandFilters.length > 0) {
+				allow = onDemandFilters.every(filter => {
+					return checkFilter(filter, doc._source);
+				});
+			} else {
+				allow = true;
+			}
+			if (allow) {
+				enqueuedMessages.push(doc._source);
+			}
+		}
+		if (socket.connected) {
+			socket.emit('message', {type: 'action_trace', mode: 'history', messages: enqueuedMessages});
+		} else {
+			hLog('LOST CLIENT');
+			break;
+		}
+		if (body['hits'].total.value === counter) {
+			hLog(`${counter} past actions streamed to ${socket.id}`);
+			break;
+		}
+		if (init_response.body.hits.total.value < 1000) {
+			const next_response = await fastify.elastic.scroll({
+				body: {scroll_id: body['_scroll_id'], scroll: '30s'}
+			});
+			responseQueue.push(next_response);
+		} else {
+			hLog('Request too large!');
+			socket.emit('message', {type: 'action_trace', mode: 'history', messages: []});
+		}
+	}
+}
 
 export class SocketManager {
 
-    private io: Server;
-    private relay;
-    relay_restored = true;
-    relay_down = false;
-    private readonly url;
-    private readonly server: FastifyInstance;
-    private readonly uwsApp: TemplatedApp;
-    private readonly chainId: string;
-    private currentBlockNum: number;
+	private io: Server;
+	private relay;
+	relay_restored = true;
+	relay_down = false;
+	private readonly url;
+	private readonly server: FastifyInstance;
 
-    constructor(fastify: FastifyInstance, url, redisOptions) {
-        this.server = fastify;
-        this.url = url;
-        this.uwsApp = App({});
+	constructor(fastify: FastifyInstance, url, redisOptions) {
+		this.server = fastify;
+		this.url = url;
 
-        // WS Server for public access
-        this.io = new Server({
-            transports: ['websocket'],
-            path: '/stream'
-        });
+		this.io = new Server(fastify.server, {
+			allowEIO3: true,
+			transports: ['websocket', 'polling'],
+		});
 
-        this.io.attachApp(this.uwsApp);
+		const pubClient = new IORedis(redisOptions);
+		const subClient = pubClient.duplicate();
+		this.io.adapter(createAdapter({pubClient, subClient}));
 
-        this.chainId = this.server.manager.conn.chains[this.server.manager.chain].chain_id
-        hLog(`[SocketManager] chain_id: ${this.chainId}`);
-        const pubClient = new IORedis(redisOptions);
-        const subClient = pubClient.duplicate();
-        this.io.adapter(createAdapter({pubClient, subClient, key: this.chainId}));
+		this.io.on('connection', (socket: Socket) => {
 
-        this.io.on('connection', (socket: Socket) => {
+			if (socket.handshake.headers['x-forwarded-for']) {
+				hLog(`[socket] ${socket.id} connected via ${socket.handshake.headers['x-forwarded-for']}`);
+			}
 
-            if (socket.handshake.headers['x-forwarded-for']) {
-                hLog(`[socket] ${socket.id} connected via ${socket.handshake.headers['x-forwarded-for']}`);
-            }
+			socket.emit('message', {
+				event: 'handshake',
+				chain: fastify.manager.chain,
+			});
 
-            socket.emit('message', {
-                event: 'handshake',
-                chain: fastify.manager.chain,
-            });
+			if (this.relay) {
+				this.relay.emit('event', {
+					type: 'client_count',
+					counter: this.io.sockets.sockets.size,
+				});
+			}
 
-            if (this.relay) {
-                this.relay.emit('event', {
-                    type: 'client_count',
-                    counter: this.io.sockets.sockets.size,
-                });
-            }
+			socket.on('delta_stream_request', async (data: StreamDeltasRequest, callback) => {
+				if (typeof callback === 'function' && data) {
+					try {
+						if (data.start_from) {
+							await streamPastDeltas(this.server, socket, data);
+						}
+						this.emitToRelay(data, 'delta_request', socket, callback);
+					} catch (e) {
+						console.log(e);
+					}
+				}
+			});
 
-            socket.on('delta_stream_request', async (data: StreamDeltasRequest, callback) => {
-                if (typeof callback === 'function' && data) {
-                    try {
-                        // generate random uuid
-                        socket.data.reqUUID = randomUUID();
-                        const lastHistoryBlock = await new Promise<number>((resolve) => {
-                            // start sending realtime data
-                            this.emitToRelay(data, 'delta_request', socket, (emissionResult) => {
-                                callback(emissionResult);
-                                resolve(emissionResult.currentBlockNum);
-                            });
-                        });
-                        // push history data
-                        if (data.start_from) {
-                            data.read_until = lastHistoryBlock;
-                            console.log('Performing primary scroll request...');
-                            let ltb = 0;
-                            const hStreamResult = await streamPastDeltas(this.server, socket, data);
-                            if (hStreamResult.status === false) {
-                                return;
-                            } else {
-                                ltb = hStreamResult.lastTransmittedBlock;
-                                let attempts = 0;
-                                await sleep(500);
-                                while (ltb > 0 && lastHistoryBlock > ltb && attempts < 3) {
-                                    attempts++;
-                                    console.log(`Performing fill request from ${ltb}...`);
-                                    data.start_from = hStreamResult.lastTransmittedBlock + 1;
-                                    data.read_until = lastHistoryBlock;
-                                    const r = await streamPastDeltas(this.server, socket, data);
-                                    if (r.status === false) {
-                                        console.log(r);
-                                        return;
-                                    } else {
-                                        ltb = r.lastTransmittedBlock;
-                                    }
-                                }
-                            }
-                        }
-                    } catch (e) {
-                        console.log(e);
-                    }
-                }
-            });
+			socket.on('action_stream_request', async (data: StreamActionsRequest, callback) => {
+				if (typeof callback === 'function' && data) {
+					try {
+						if (data.start_from) {
+							await streamPastActions(this.server, socket, data);
+						}
+						this.emitToRelay(data, 'action_request', socket, callback);
+					} catch (e) {
+						console.log(e);
+					}
+				}
+			});
 
-            socket.on('action_stream_request', async (data: StreamActionsRequest, callback) => {
-                if (typeof callback === 'function' && data) {
-                    try {
-                        // generate random uuid
-                        socket.data.reqUUID = randomUUID();
-                        const lastHistoryBlock = await new Promise<number>((resolve) => {
-                            // start sending realtime data
-                            this.emitToRelay(data, 'action_request', socket, (emissionResult) => {
-                                callback(emissionResult);
-                                resolve(emissionResult.currentBlockNum);
-                            });
-                        });
-                        // push history data
-                        if (data.start_from) {
-                            data.read_until = lastHistoryBlock;
-                            console.log('Performing primary scroll request...');
-                            let ltb = 0;
-                            const hStreamResult = await streamPastActions(this.server, socket, data);
-                            if (hStreamResult.status === false) {
-                                return;
-                            } else {
-                                ltb = hStreamResult.lastTransmittedBlock;
-                                let attempts = 0;
-                                await sleep(500);
-                                while (ltb > 0 && lastHistoryBlock > ltb && attempts < 3) {
-                                    attempts++;
-                                    console.log(`Performing fill request from ${hStreamResult.lastTransmittedBlock}...`);
-                                    data.start_from = hStreamResult.lastTransmittedBlock + 1;
-                                    data.read_until = lastHistoryBlock;
-                                    const r = await streamPastActions(this.server, socket, data);
-                                    if (r.status === false) {
-                                        console.log(r);
-                                        return;
-                                    } else {
-                                        ltb = r.lastTransmittedBlock;
-                                    }
-                                }
-                            }
-                        }
-                    } catch (e) {
-                        console.log(e);
-                    }
-                }
-            });
+			socket.on('disconnect', (reason) => {
+				hLog(`[socket] ${socket.id} disconnected - ${reason}`);
+				this.relay.emit('event', {
+					type: 'client_disconnected',
+					id: socket.id,
+					reason,
+				});
+			});
+		});
+		hLog('Websocket manager loaded!');
+	}
 
-            socket.on('disconnect', (reason) => {
-                hLog(`[socket] ${socket.id} disconnected - ${reason}`);
-                this.relay.emit('event', {
-                    type: 'client_disconnected',
-                    id: socket.id,
-                    reason,
-                });
-            });
-        });
+	startRelay() {
+		hLog(`starting relay - ${this.url}`);
 
-        try {
-            const port = this.server.manager.config.api.stream_port || 1234;
-            this.uwsApp.listen(port, () => {
-                hLog(`Socket.IO via uWS started on port ${port}`);
-            });
-        } catch (e) {
-            hLog(e.message);
-        }
+		this.relay = io(this.url, {path: '/router'});
 
-        hLog('Websocket manager loaded!');
-    }
+		this.relay.on('connect', () => {
+			hLog('Relay Connected!');
+			if (this.relay_down) {
+				this.relay_restored = true;
+				this.relay_down = false;
+				this.io.emit('status', 'relay_restored');
+			}
+		});
 
-    /*
-    WS Relay will connect to the indexer
-     */
-    startRelay() {
-        hLog(`starting relay - ${this.url}`);
-        this.relay = io(this.url, {path: '/router'});
+		this.relay.on('disconnect', () => {
+			hLog('Relay disconnected!');
+			this.io.emit('status', 'relay_down');
+			this.relay_down = true;
+			this.relay_restored = false;
+		});
 
-        this.relay.on('connect', () => {
-            hLog('Relay Connected!');
-            if (this.relay_down) {
-                this.relay_restored = true;
-                this.relay_down = false;
-                this.io.emit('status', 'relay_restored');
-            }
-        });
+		this.relay.on('delta', (traceData) => {
+			this.emitToClient(traceData, 'delta_trace');
+		});
 
-        this.relay.on('disconnect', () => {
-            hLog('Relay disconnected!');
-            this.io.emit('status', 'relay_down');
-            this.relay_down = true;
-            this.relay_restored = false;
-        });
+		this.relay.on('trace', (traceData) => {
+			this.emitToClient(traceData, 'action_trace');
+		});
 
-        this.relay.on('delta', (traceData) => {
-            this.emitToClient(traceData, 'delta_trace');
-        });
+		// Relay LIB info to clients;
+		this.relay.on('lib_update', (data) => {
+			if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+				this.io.emit('lib_update', data);
+			}
+		});
 
-        this.relay.on('trace', (traceData) => {
-            this.emitToClient(traceData, 'action_trace');
-        });
+		// Relay LIB info to clients;
+		this.relay.on('fork_event', (data) => {
+			hLog(data);
+			if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+				this.io.emit('fork_event', data);
+			}
+		});
+	}
 
-        this.relay.on('block', (blockData) => {
-            try {
-                // const decodedBlock = JSON.parse(blockData.content.toString());
-                // console.log(blockData.serverTime, blockData.blockNum, decodedBlock);
-                this.currentBlockNum = blockData.blockNum;
-            } catch (e) {
-                hLog(`Failed to decode incoming live block ${blockData.blockNum}: ${e.message}`);
-            }
-        });
+	emitToClient(traceData, type) {
+		if (this.io.sockets.sockets.has(traceData.client)) {
+			this.io.sockets.sockets.get(traceData.client).emit('message', {
+				type: type,
+				mode: 'live',
+				message: traceData.message,
+			});
+		}
+	}
 
-        this.addRelayForwarding('lib_update');
-
-        this.addRelayForwarding('fork_event');
-
-        // // Relay LIB info to clients;
-        // this.relay.on('lib_update', (data) => {
-        //     if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-        //         this.io.emit('lib_update', data);
-        //     }
-        // });
-        //
-        // // Relay fork info to clients;
-        // this.relay.on('fork_event', (data) => {
-        //     if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-        //         this.io.emit('fork_event', data);
-        //     }
-        // });
-    }
-
-    // Relay events to clients
-    addRelayForwarding(event: string) {
-        this.relay.on(event, (data: any) => {
-            if (data.chain_id && this.chainId === data.chain_id) {
-                this.io.emit(event, data);
-            }
-        });
-    }
-
-    emitToClient(traceData, type) {
-        if (this.io.sockets.sockets.has(traceData.client)) {
-            this.io.sockets.sockets.get(traceData.client).emit('message', {
-                type: type,
-                reqUUID: traceData.req,
-                mode: 'live',
-                message: traceData.message,
-            });
-        }
-    }
-
-    emitToRelay(data, type, socket, callback) {
-        if (this.relay.connected) {
-            this.relay.emit('event', {
-                reqUUID: socket.data.reqUUID,
-                type: type,
-                client_socket: socket.id,
-                request: data,
-            }, (response) => {
-                response['reqUUID'] = socket.data.reqUUID;
-                response['currentBlockNum'] = this.currentBlockNum;
-                callback(response);
-            });
-        } else {
-            callback('STREAMING_OFFLINE');
-        }
-    }
+	emitToRelay(data, type, socket, callback) {
+		if (this.relay.connected) {
+			this.relay.emit('event', {
+				type: type,
+				client_socket: socket.id,
+				request: data,
+			}, (response) => {
+				callback(response);
+			});
+		} else {
+			callback('STREAMING_OFFLINE');
+		}
+	}
 }

auto_setup.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+git clone https://github.com/eosrio/hyperion-history-api.git  || exit
+cd hyperion-history-api || exit
+bash ./install_env.sh

chains/example.config.json

@@ -1,14 +1,8 @@
 {
   "api": {
-    "enabled": true,
-    "pm2_scaling": 1,
-    "node_max_old_space_size": 1024,
     "chain_name": "EXAMPLE Chain",
     "server_addr": "127.0.0.1",
     "server_port": 7000,
-    "stream_port": 1234,
-    "stream_scroll_limit": -1,
-    "stream_scroll_batch": 500,
     "server_name": "127.0.0.1:7000",
     "provider_name": "Example Provider",
     "provider_url": "https://example.com",
@@ -25,28 +19,64 @@
       "get_trx_actions": 200
     },
     "access_log": false,
+    "enable_explorer": false,
     "chain_api_error_log": false,
     "custom_core_token": "",
     "enable_export_action": false,
-    "disable_rate_limit": false,
     "rate_limit_rpm": 1000,
     "rate_limit_allow": [],
     "disable_tx_cache": false,
-    "tx_cache_expiration_sec": 3600,
-    "v1_chain_cache": [
-      {
-        "path": "get_block",
-        "ttl": 3000
-      },
-      {
-        "path": "get_info",
-        "ttl": 500
-      }
-    ]
+    "tx_cache_expiration_sec": 3600
+  },
+  "settings": {
+    "preview": false,
+    "chain": "eos",
+    "eosio_alias": "eosio",
+    "parser": "1.8",
+    "auto_stop": 0,
+    "index_version": "v1",
+    "debug": false,
+    "bp_logs": false,
+    "bp_monitoring": false,
+    "ipc_debug_rate": 60000,
+    "allow_custom_abi": false,
+    "rate_monitoring": true,
+    "max_ws_payload_mb": 256,
+    "ds_profiling": false,
+    "auto_mode_switch": false,
+    "hot_warm_policy": false,
+    "custom_policy": "",
+    "bypass_index_map": false,
+    "index_partition_size": 10000000
+  },
+  "blacklists": {
+    "actions": [],
+    "deltas": []
+  },
+  "whitelists": {
+    "actions": [],
+    "deltas": [],
+    "max_depth": 10,
+    "root_only": false
+  },
+  "scaling": {
+    "readers":            1,
+    "ds_queues":          1,
+    "ds_threads":         1,
+    "ds_pool_size":       1,
+    "indexing_queues":    1,
+    "ad_idx_queues":      1,
+    "dyn_idx_queues":     1,
+    "max_autoscale":      4,
+    "batch_size":         5000,
+    "resume_trigger":     5000,
+    "auto_scale_trigger": 20000,
+    "block_queue_limit":  10000,
+    "max_queue_limit":    100000,
+    "routing_mode":       "round_robin",
+    "polling_interval":   10000
   },
   "indexer": {
-    "enabled": true,
-    "node_max_old_space_size": 4096,
     "start_on": 0,
     "stop_on": 0,
     "rewrite": false,
@@ -61,55 +91,6 @@
     "process_deltas": true,
     "disable_delta_rm": true
   },
-  "settings": {
-    "preview": false,
-    "chain": "eos",
-    "eosio_alias": "eosio",
-    "parser": "3.2",
-    "auto_stop": 0,
-    "index_version": "v1",
-    "debug": false,
-    "bp_logs": false,
-    "bp_monitoring": false,
-    "ipc_debug_rate": 60000,
-    "allow_custom_abi": false,
-    "rate_monitoring": true,
-    "max_ws_payload_mb": 256,
-    "ds_profiling": false,
-    "auto_mode_switch": false,
-    "hot_warm_policy": false,
-    "custom_policy": "",
-    "use_global_agent": false,
-    "index_partition_size": 10000000,
-    "es_replicas": 0
-  },
-  "blacklists": {
-    "actions": [],
-    "deltas": []
-  },
-  "whitelists": {
-    "actions": [],
-    "deltas": [],
-    "max_depth": 10,
-    "root_only": false
-  },
-  "scaling": {
-    "readers": 1,
-    "ds_queues": 1,
-    "ds_threads": 1,
-    "ds_pool_size": 1,
-    "indexing_queues": 1,
-    "ad_idx_queues": 1,
-    "dyn_idx_queues": 1,
-    "max_autoscale": 4,
-    "batch_size": 5000,
-    "resume_trigger": 5000,
-    "auto_scale_trigger": 20000,
-    "block_queue_limit": 10000,
-    "max_queue_limit": 100000,
-    "routing_mode": "round_robin",
-    "polling_interval": 10000
-  },
   "features": {
     "streaming": {
       "enable": false,
@@ -122,7 +103,7 @@
       "voters": true
     },
     "index_deltas": true,
-    "index_transfer_memo": true,
+    "index_transfer_memo": false,
     "index_all_deltas": true,
     "deferred_trx": false,
     "failed_trx": false,

connections-demo.json

@@ -0,0 +1,55 @@
+{
+  "amqp": {
+    "host": "127.0.0.1:5672",
+    "api": "127.0.0.1:15672",
+    "user": "username",
+    "pass": "password",
+    "vhost": "hyperion"
+  },
+  "elasticsearch": {
+    "host": "127.0.0.1:9200",
+    "ingest_nodes": [
+      "127.0.0.1:9200"
+    ],
+    "user": "elastic",
+    "pass": "password"
+  },
+  "redis": {
+    "host": "127.0.0.1",
+    "port": "6379"
+  },
+  "chains": {
+    "eos": {
+      "name": "EOS Mainnet",
+      "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
+      "http": "http://127.0.0.1:8888",
+      "ship": "ws://127.0.0.1:8080",
+      "WS_ROUTER_PORT": 7001,
+      "WS_ROUTER_HOST": "127.0.0.1"
+    },
+    "eos2": {
+      "name": "EOS Mainnet",
+      "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
+      "http": "http://127.0.0.1:8888",
+      "ship": "ws://127.0.0.1:8080",
+      "WS_ROUTER_PORT": 7001,
+      "WS_ROUTER_HOST": "127.0.0.1"
+    },
+    "demo": {
+      "name": "Demo Chain",
+      "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
+      "http": "http://api.eosrio.io",
+      "ship": "ws://200.201.191.228:58013",
+      "WS_ROUTER_PORT": 57201,
+      "WS_ROUTER_HOST": "192.168.0.1"
+    },
+    "demo2": {
+      "name": "Demo Chain",
+      "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
+      "http": "http://api.eosrio.io",
+      "ship": "ws://200.201.191.228:58013",
+      "WS_ROUTER_PORT": 57201,
+      "WS_ROUTER_HOST": "192.168.0.1"
+    }
+  }
+}

connections/amqp.ts

@@ -17,16 +17,17 @@ export async function createConnection(config): Promise<Connection> {
 }
 
 export function getAmpqUrl(config): string {
-    let frameMaxValue = '0x10000';
-    if (config.frameMax) {
-        frameMaxValue = config.frameMax;
-    }
-    const u = encodeURIComponent(config.user);
-    const p = encodeURIComponent(config.pass);
-    const v = encodeURIComponent(config.vhost);
-    return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
+	let frameMaxValue = '0x10000';
+	if (config.frameMax) {
+		frameMaxValue = config.frameMax;
+	}
+	const u = encodeURIComponent(config.user);
+	const p = encodeURIComponent(config.pass);
+	const v = encodeURIComponent(config.vhost)
+	return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
 }
 
+
 async function createChannels(connection) {
 	try {
 		const channel = await connection.createChannel();

connections/manager.class.ts

@@ -12,163 +12,167 @@ import {hLog} from "../helpers/common_functions";
 
 export class ConnectionManager {
 
-    config: HyperionConfig;
-    conn: HyperionConnections;
+	config: HyperionConfig;
+	conn: HyperionConnections;
 
-    chain: string;
-    last_commit_hash: string;
-    current_version: string;
+	chain: string;
+	last_commit_hash: string;
+	current_version: string;
 
-    esIngestClients: Client[];
-    esIngestClient: Client;
+	private readonly esIngestClients: Client[];
+	private esIngestClient: Client;
 
-    constructor(private cm: ConfigurationModule) {
-        this.config = cm.config;
-        this.conn = cm.connections;
-        if (!this.conn.amqp.protocol) {
-            this.conn.amqp.protocol = 'http';
-        }
-        this.chain = this.config.settings.chain;
-        this.esIngestClients = [];
-        this.prepareESClient();
-        this.prepareIngestClients();
-    }
+	constructor(private cm: ConfigurationModule) {
+		this.config = cm.config;
+		this.conn = cm.connections;
+		if (!this.conn.amqp.protocol) {
+			this.conn.amqp.protocol = 'http';
+		}
+		this.chain = this.config.settings.chain;
+		this.esIngestClients = [];
+		this.prepareESClient();
+		this.prepareIngestClients();
+	}
 
-    get nodeosJsonRPC() {
-        // @ts-ignore
-        return new JsonRpc(this.conn.chains[this.chain].http, {fetch});
-    }
+	get nodeosJsonRPC() {
+		// @ts-ignore
+		return new JsonRpc(this.conn.chains[this.chain].http, {fetch});
+	}
 
-    async purgeQueues() {
-        hLog(`Purging all ${this.chain} queues!`);
-        const apiUrl = `http://${this.conn.amqp.api}`;
-        const vHost = encodeURIComponent(this.conn.amqp.vhost);
-        const getAllQueuesFromVHost = apiUrl + `/api/queues/${vHost}`;
-        const opts = {
-            username: this.conn.amqp.user,
-            password: this.conn.amqp.pass
-        };
-        let result;
-        try {
-            const data = await got(getAllQueuesFromVHost, opts);
-            if (data) {
-                result = JSON.parse(data.body);
-            }
-        } catch (e) {
-            console.log(e.message);
-            console.error('failed to connect to rabbitmq http api');
-            process.exit(1);
-        }
-        if (result) {
-            for (const queue of result) {
-                if (queue.name.startsWith(this.chain + ":")) {
-                    const msg_count = parseInt(queue.messages);
-                    if (msg_count > 0) {
-                        try {
-                            await got.delete(apiUrl + `/api/queues/${vHost}/${queue.name}/contents`, opts);
-                            hLog(`${queue.messages} messages deleted on queue ${queue.name}`);
-                        } catch (e) {
-                            console.log(e.message);
-                            console.error('failed to connect to rabbitmq http api');
-                            process.exit(1);
-                        }
-                    }
-                }
-            }
-        }
-    }
+	async purgeQueues() {
+		hLog(`Purging all ${this.chain} queues!`);
+		const apiUrl = `http://${this.conn.amqp.api}`;
+		const vHost = encodeURIComponent(this.conn.amqp.vhost);
+		const getAllQueuesFromVHost = apiUrl + `/api/queues/${vHost}`;
+		const opts = {
+			username: this.conn.amqp.user,
+			password: this.conn.amqp.pass
+		};
+		let result;
+		try {
+			const data = await got(getAllQueuesFromVHost, opts);
+			if (data) {
+				result = JSON.parse(data.body);
+			}
+		} catch (e) {
+			console.log(e.message);
+			console.error('failed to connect to rabbitmq http api');
+			process.exit(1);
+		}
+		if (result) {
+			for (const queue of result) {
+				if (queue.name.startsWith(this.chain + ":")) {
+					const msg_count = parseInt(queue.messages);
+					if (msg_count > 0) {
+						try {
+							await got.delete(apiUrl + `/api/queues/${vHost}/${queue.name}/contents`, opts);
+							hLog(`${queue.messages} messages deleted on queue ${queue.name}`);
+						} catch (e) {
+							console.log(e.message);
+							console.error('failed to connect to rabbitmq http api');
+							process.exit(1);
+						}
+					}
+				}
+			}
+		}
+	}
 
-    prepareESClient() {
-        const _es = this.conn.elasticsearch;
-        if (!_es.protocol) {
-            _es.protocol = 'http';
-        }
-        this.esIngestClient = new Client({
-            node: `${_es.protocol}://${_es.host}`,
-            auth: {
-                username: _es.user,
-                password: _es.pass
-            },
-            ssl: _es.protocol === 'https' ? {
-                rejectUnauthorized: false
-            } : undefined
-        });
-    }
+	prepareESClient() {
+		let es_url;
+		const _es = this.conn.elasticsearch;
+		if (!_es.protocol) {
+			_es.protocol = 'http';
+		}
+		if (_es.user !== '') {
+			es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${_es.host}`;
+		} else {
+			es_url = `${_es.protocol}://${_es.host}`
+		}
+		this.esIngestClient = new Client({
+			node: es_url,
+			ssl: {
+				rejectUnauthorized: false
+			}
+		});
+	}
 
-    get elasticsearchClient() {
-        return this.esIngestClient;
-    }
+	get elasticsearchClient() {
+		return this.esIngestClient;
+	}
 
-    prepareIngestClients() {
-        const _es = this.conn.elasticsearch;
-        if (!_es.protocol) {
-            _es.protocol = 'http';
-        }
-        if (_es.ingest_nodes) {
-            if (_es.ingest_nodes.length > 0) {
-                for (const node of _es.ingest_nodes) {
-                    this.esIngestClient = new Client({
-                        node: `${_es.protocol}://${_es.host}`,
-                        auth: {
-                            username: _es.user,
-                            password: _es.pass
-                        },
-                        pingTimeout: 100,
-                        ssl: _es.protocol === 'https' ? {
-                            rejectUnauthorized: false
-                        } : undefined
-                    });
-                }
-            }
-        }
-    }
+	prepareIngestClients() {
+		const _es = this.conn.elasticsearch;
+		if (!_es.protocol) {
+			_es.protocol = 'http';
+		}
+		if (_es.ingest_nodes) {
+			if (_es.ingest_nodes.length > 0) {
+				for (const node of _es.ingest_nodes) {
+					let es_url;
+					if (_es.user !== '') {
+						es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${node}`;
+					} else {
+						es_url = `${_es.protocol}://${node}`
+					}
+					this.esIngestClients.push(new Client({
+						node: es_url,
+						pingTimeout: 100,
+						ssl: {
+							rejectUnauthorized: false
+						}
+					}));
+				}
+			}
+		}
+	}
 
-    get ingestClients() {
-        if (this.esIngestClients.length > 0) {
-            return this.esIngestClients;
-        } else {
-            return [this.esIngestClient];
-        }
-    }
+	get ingestClients() {
+		if (this.esIngestClients.length > 0) {
+			return this.esIngestClients;
+		} else {
+			return [this.esIngestClient];
+		}
+	}
 
-    async createAMQPChannels(onReconnect, onClose) {
-        return await amqpConnect(onReconnect, this.conn.amqp, onClose);
-    }
+	async createAMQPChannels(onReconnect, onClose) {
+		return await amqpConnect(onReconnect, this.conn.amqp, onClose);
+	}
 
-    async checkQueueSize(queue) {
-        return await checkQueueSize(queue, this.conn.amqp);
-    }
+	async checkQueueSize(queue) {
+		return await checkQueueSize(queue, this.conn.amqp);
+	}
 
-    get shipClient(): StateHistorySocket {
-        return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_mb);
-    }
+	get shipClient(): StateHistorySocket {
+		return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_mb);
+	}
 
-    get ampqUrl() {
-        return getAmpqUrl(this.conn.amqp);
-    }
+	get ampqUrl() {
+		return getAmpqUrl(this.conn.amqp);
+	}
 
-    calculateServerHash() {
-        exec('git rev-parse HEAD', (err, stdout) => {
-            if (err) {
-                // hLog(`\n ${err.message}\n >>> Failed to check last commit hash. Version hash will be "custom"`);
-                hLog(`Failed to check last commit hash. Version hash will be "custom"`);
-                this.last_commit_hash = 'custom';
-            } else {
-                hLog('Last commit hash on this branch is:', stdout);
-                this.last_commit_hash = stdout.trim();
-            }
-            this.getHyperionVersion();
-        });
-    }
+	calculateServerHash() {
+		exec('git rev-parse HEAD', (err, stdout) => {
+			if (err) {
+				// hLog(`\n ${err.message}\n >>> Failed to check last commit hash. Version hash will be "custom"`);
+				hLog(`Failed to check last commit hash. Version hash will be "custom"`);
+				this.last_commit_hash = 'custom';
+			} else {
+				hLog('Last commit hash on this branch is:', stdout);
+				this.last_commit_hash = stdout.trim();
+			}
+			this.getHyperionVersion();
+		});
+	}
 
-    getServerHash() {
-        return this.last_commit_hash;
-    }
+	getServerHash() {
+		return this.last_commit_hash;
+	}
 
-    getHyperionVersion() {
-        this.current_version = require('../package.json').version;
-        if (this.last_commit_hash === 'custom') {
-            this.current_version = this.current_version + '-dirty';
-        }
-    }
+	getHyperionVersion() {
+		this.current_version = require('../package.json').version;
+		if (this.last_commit_hash === 'custom') {
+			this.current_version = this.current_version + '-dirty';
+		}
+	}
 }

connections/state-history.ts

@@ -6,8 +6,6 @@ export class StateHistorySocket {
 	private ws;
 	private readonly shipUrl;
 	private readonly max_payload_mb;
-	retryOnDisconnect = true;
-	connected = false;
 
 	constructor(ship_url, max_payload_mb) {
 		this.shipUrl = ship_url;
@@ -19,13 +17,13 @@ export class StateHistorySocket {
 	}
 
 	connect(onMessage, onDisconnect, onError, onConnected) {
+
 		debugLog(`Connecting to ${this.shipUrl}...`);
 		this.ws = new WebSocket(this.shipUrl, {
 			perMessageDeflate: false,
 			maxPayload: this.max_payload_mb * 1024 * 1024,
 		});
 		this.ws.on('open', () => {
-			this.connected = true;
 			hLog('Websocket connected!');
 			if (onConnected) {
 				onConnected();
@@ -33,21 +31,15 @@ export class StateHistorySocket {
 		});
 		this.ws.on('message', (data) => onMessage(data));
 		this.ws.on('close', () => {
-			this.connected = false;
 			hLog('Websocket disconnected!');
-			if(this.retryOnDisconnect) {
-				onDisconnect();
-			}
+			onDisconnect();
 		});
 		this.ws.on('error', (err) => {
 			hLog(`${this.shipUrl} :: ${err.message}`);
 		});
 	}
 
-	close(graceful: boolean) {
-		if(graceful) {
-			this.retryOnDisconnect = false;
-		}
+	close() {
 		this.ws.close();
 	}
 

definitions/ecosystem_settings.js

@@ -1,51 +1,51 @@
+// max js heap in MB for each indexer subprocess
+const idx_js_heap = 4096;
+
+// max js heap in MB for each api process
+const api_js_heap = 1024;
+
 function interpreterArgs(heap) {
-    const arr = ['--trace-deprecation', '--trace-warnings'];
-    if (heap) {
-        arr.push('--max-old-space-size=' + heap);
-    } else {
-        arr.push('--max-old-space-size=2048');
-    }
-    if (process.env.INSPECT) {
-        arr.push('--inspect');
-    }
-    return arr;
+  const arr = ['--max-old-space-size=' + heap, '--trace-deprecation', '--trace-warnings'];
+  if (process.env.INSPECT) {
+    arr.push('--inspect');
+  }
+  return arr;
 }
 
-function addIndexer(chainName, heap) {
-    return {
-        script: './launcher.js',
-        name: chainName + '-indexer',
-        namespace: chainName,
-        interpreter: 'node',
-        interpreter_args: interpreterArgs(heap),
-        autorestart: false,
-        kill_timeout: 3600,
-        watch: false,
-        time: true,
-        env: {
-            CONFIG_JSON: 'chains/' + chainName + '.config.json',
-            TRACE_LOGS: 'false',
-        },
-    };
+function addIndexer(chainName) {
+  return {
+    script: './launcher.js',
+    name: chainName + '-indexer',
+    namespace: chainName,
+    interpreter: 'node',
+    interpreter_args: interpreterArgs(idx_js_heap),
+    autorestart: false,
+    kill_timeout: 3600,
+    watch: false,
+    time: true,
+    env: {
+      CONFIG_JSON: 'chains/' + chainName + '.config.json',
+      TRACE_LOGS: 'false',
+    },
+  };
 }
 
-function addApiServer(chainName, threads, heap) {
-    return {
-        script: './api/server.js',
-        name: chainName + '-api',
-        namespace: chainName,
-        node_args: interpreterArgs(heap),
-        exec_mode: 'cluster',
-        merge_logs: true,
-        instances: threads,
-        autorestart: true,
-        exp_backoff_restart_delay: 100,
-        watch: false,
-        time: true,
-        env: {
-            CONFIG_JSON: 'chains/' + chainName + '.config.json',
-        },
-    };
+function addApiServer(chainName, threads) {
+  return {
+    script: './api/server.js',
+    name: chainName + '-api',
+    namespace: chainName,
+    node_args: interpreterArgs(api_js_heap),
+    exec_mode: 'cluster',
+    merge_logs: true,
+    instances: threads,
+    autorestart: true,
+    exp_backoff_restart_delay: 100,
+    watch: false,
+    env: {
+      CONFIG_JSON: 'chains/' + chainName + '.config.json',
+    },
+  };
 }
 
 module.exports = {addIndexer, addApiServer};

ecosystem.config.js

@@ -1,30 +0,0 @@
-const {addApiServer, addIndexer} = require('./definitions/ecosystem_settings');
-const {readdirSync, readFileSync} = require("fs");
-const path = require('path');
-
-const apps = [];
-const chainsRoot = path.join(path.resolve(), 'chains');
-readdirSync(chainsRoot)
-    .filter(f => f.endsWith('.config.json') && !f.startsWith('example'))
-    .forEach(value => {
-        const configFile = readFileSync(path.join(chainsRoot, value))
-        const config = JSON.parse(configFile.toString());
-        const chainName = config.settings.chain;
-        if (config.api.enabled) {
-            const apiHeap = config.api.node_max_old_space_size;
-            apps.push(addApiServer(chainName, config.api.pm2_scaling, apiHeap));
-        }
-        if (config.indexer.enabled) {
-            const indexerHeap = config.indexer.node_max_old_space_size;
-            apps.push(addIndexer(chainName, indexerHeap));
-        }
-    });
-
-// apps.push({
-//     name: 'hyperion-governor',
-//     namespace: 'hyperion',
-//     script: 'governor/server/index.js',
-//     watch: false,
-// });
-
-module.exports = {apps};

example-connections.json

@@ -27,9 +27,8 @@
       "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
       "http": "http://127.0.0.1:8888",
       "ship": "ws://127.0.0.1:8080",
-      "WS_ROUTER_HOST": "127.0.0.1",
       "WS_ROUTER_PORT": 7001,
-      "control_port": 7002
+      "WS_ROUTER_HOST": "127.0.0.1"
     }
   }
 }

example-ecosystem.config.js

@@ -0,0 +1,14 @@
+const {addApiServer, addIndexer} = require("./definitions/ecosystem_settings");
+
+module.exports = {
+    apps: [
+        // launch indexer
+        addIndexer('eos'),
+
+        // launch a single threaded api
+        addApiServer('eos', 1)
+
+        // example: launching a 4-node cluster of apis using pm2 cluster
+        // addApiServer('eos', 4)
+    ]
+};

helpers/common_functions.ts

@@ -6,288 +6,233 @@ import {join} from "path";
 let config;
 const conf_path = join(__dirname, `../${process.env.CONFIG_JSON}`);
 if (existsSync(conf_path)) {
-    try {
-        config = require(conf_path);
-    } catch (e) {
-        console.log(e.message);
-        process.exit(1);
-    }
+	try {
+		config = require(conf_path);
+	} catch (e) {
+		console.log(e.message);
+		process.exit(1);
+	}
 }
 
 if (!config) {
-    console.log(`Configuration not found: ${conf_path}`);
-    process.exit(1);
+	console.log(`Configuration not found: ${conf_path}`);
+	process.exit(1);
 }
 
 function getLastResult(results: ApiResponse) {
-    if (results.body.hits?.hits?.length > 0) {
-        return parseInt(results.body.hits.hits[0].sort[0], 10);
-    } else {
-        return 1;
-    }
+	if (results.body.hits?.hits?.length > 0) {
+		return parseInt(results.body.hits.hits[0].sort[0], 10);
+	} else {
+		return 1;
+	}
 }
 
 export async function getLastIndexedBlockByDelta(es_client: Client, chain: string) {
-    const results: ApiResponse = await es_client.search({
-        index: chain + '-delta-*',
-        size: 1,
-        body: {
-            query: {bool: {filter: {match_all: {}}}},
-            sort: [{block_num: {order: "desc"}}]
-        }
-    });
-    return getLastResult(results);
+	const results: ApiResponse = await es_client.search({
+		index: chain + '-delta-*',
+		size: 1,
+		body: {
+			query: {bool: {filter: {match_all: {}}}},
+			sort: [{block_num: {order: "desc"}}]
+		}
+	});
+	return getLastResult(results);
 }
 
 export async function getLastIndexedBlock(es_client: Client, chain: string) {
-    const results: ApiResponse = await es_client.search({
-        index: chain + '-block-*',
-        size: 1,
-        body: {
-            query: {bool: {filter: {match_all: {}}}},
-            sort: [{block_num: {order: "desc"}}]
-        }
-    });
-    return getLastResult(results);
+	const results: ApiResponse = await es_client.search({
+		index: chain + '-block-*',
+		size: 1,
+		body: {
+			query: {bool: {filter: {match_all: {}}}},
+			sort: [{block_num: {order: "desc"}}]
+		}
+	});
+	return getLastResult(results);
 }
 
 export async function getLastIndexedBlockWithTotalBlocks(es_client: Client, chain: string): Promise<[number, number]> {
-    const results: ApiResponse = await es_client.search({
-        index: chain + '-block-*',
-        size: 1,
-        body: {
-            query: {bool: {filter: {match_all: {}}}},
-            sort: [{block_num: {order: "desc"}}],
-            track_total_hits: true
-        }
-    });
-    let lastBlock = getLastResult(results);
-    let totalBlocks = results.body.hits.total.value || 1;
-    return [lastBlock, totalBlocks];
-}
-
-export async function getFirstIndexedBlock(es_client: Client, chain: string): Promise<number> {
-    const results: ApiResponse = await es_client.search({
-        index: chain + '-block-*',
-        size: 1,
-        body: {
-            query: {bool: {filter: {match_all: {}}}},
-            sort: [{block_num: {order: "asc"}}]
-        }
-    });
-    return getLastResult(results);
+	const results: ApiResponse = await es_client.search({
+		index: chain + '-block-*',
+		size: 1,
+		body: {
+			query: {bool: {filter: {match_all: {}}}},
+			sort: [{block_num: {order: "desc"}}],
+			track_total_hits: true
+		}
+	});
+	let lastBlock = getLastResult(results);
+	let totalBlocks = results.body.hits.total.value || 1;
+	return [lastBlock, totalBlocks];
 }
 
 
 export async function getLastIndexedABI(es_client: Client, chain: string) {
-    const results: ApiResponse = await es_client.search({
-        index: chain + '-abi-*',
-        size: 1,
-        body: {
-            query: {
-                match_all: {}
-            },
-            sort: [{block: {order: "desc"}}]
-        }
-    });
-    return getLastResult(results);
+	const results: ApiResponse = await es_client.search({
+		index: chain + '-abi-*',
+		size: 1,
+		body: {
+			query: {
+				match_all: {}
+			},
+			sort: [{block: {order: "desc"}}]
+		}
+	});
+	return getLastResult(results);
 }
 
 export async function getLastIndexedBlockByDeltaFromRange(es_client: Client, chain: string, first: number, last: number) {
-    const results: ApiResponse = await es_client.search({
-        index: chain + '-delta-*',
-        size: 1,
-        body: {
-            query: {
-                range: {
-                    block_num: {
-                        "gte": first,
-                        "lt": last,
-                        "boost": 2
-                    }
-                }
-            },
-            sort: [{block_num: {order: "desc"}}]
-        }
-    });
-    return getLastResult(results);
+	const results: ApiResponse = await es_client.search({
+		index: chain + '-delta-*',
+		size: 1,
+		body: {
+			query: {
+				range: {
+					block_num: {
+						"gte": first,
+						"lt": last,
+						"boost": 2
+					}
+				}
+			},
+			sort: [{block_num: {order: "desc"}}]
+		}
+	});
+	return getLastResult(results);
 }
 
 export async function getLastIndexedBlockFromRange(es_client: Client, chain: string, first: number, last: number) {
-    const results = await es_client.search({
-        index: chain + '-block-*',
-        size: 1,
-        body: {
-            query: {
-                range: {
-                    block_num: {
-                        "gte": first,
-                        "lt": last,
-                        "boost": 2
-                    }
-                }
-            },
-            sort: [{block_num: {order: "desc"}}]
-        }
-    });
-    return getLastResult(results);
+	const results = await es_client.search({
+		index: chain + '-block-*',
+		size: 1,
+		body: {
+			query: {
+				range: {
+					block_num: {
+						"gte": first,
+						"lt": last,
+						"boost": 2
+					}
+				}
+			},
+			sort: [{block_num: {order: "desc"}}]
+		}
+	});
+	return getLastResult(results);
 }
 
 export function messageAllWorkers(cl, payload) {
-    for (const c in cl.workers) {
-        if (cl.workers.hasOwnProperty(c)) {
-            const _w = cl.workers[c];
-            if (_w) {
-                try {
-                    if (_w.isConnected()) {
-                        _w.send(payload);
-                    } else {
-                        hLog('Worker is not connected!');
-                    }
-                } catch (e) {
-                    hLog('Failed to message worker!');
-                    hLog(e);
-                }
-            } else {
-                hLog('Worker not found!');
-            }
-        }
-    }
+	for (const c in cl.workers) {
+		if (cl.workers.hasOwnProperty(c)) {
+			const _w = cl.workers[c];
+			if (_w) {
+				try {
+					if (_w.isConnected()) {
+						_w.send(payload);
+					} else {
+						hLog('Worker is not connected!');
+					}
+				} catch (e) {
+					hLog('Failed to message worker!');
+					hLog(e);
+				}
+			} else {
+				hLog('Worker not found!');
+			}
+		}
+	}
 }
 
 export function serialize(type, value, txtEnc, txtDec, types) {
-    const buffer = new Serialize.SerialBuffer({
-        textEncoder: txtEnc,
-        textDecoder: txtDec
-    });
-    Serialize.getType(types, type).serialize(buffer, value);
-    return buffer.asUint8Array();
+	const buffer = new Serialize.SerialBuffer({
+		textEncoder: txtEnc,
+		textDecoder: txtDec
+	});
+	Serialize.getType(types, type).serialize(buffer, value);
+	return buffer.asUint8Array();
 }
 
 export function deserialize(type, array, txtEnc, txtDec, types) {
-    const buffer = new Serialize.SerialBuffer({
-        textEncoder: txtEnc,
-        textDecoder: txtDec,
-        array
-    });
-    return Serialize.getType(types, type).deserialize(buffer, new Serialize.SerializerState({bytesAsUint8Array: true}));
+	const buffer = new Serialize.SerialBuffer({
+		textEncoder: txtEnc,
+		textDecoder: txtDec,
+		array
+	});
+	return Serialize.getType(types, type).deserialize(buffer, new Serialize.SerializerState({bytesAsUint8Array: true}));
 }
 
 function getNested(path_array, jsonObj) {
-    const nextPath = path_array.shift();
-    const nextValue = jsonObj[nextPath];
-    if (!nextValue) {
-        return null;
-    } else {
-        if (typeof nextValue !== 'object') {
-            return nextValue;
-        } else {
-            if (Array.isArray(nextValue)) {
-                return nextValue;
-            } else {
-                return getNested(path_array, nextValue);
-            }
-        }
-    }
-}
-
-export function checkDeltaFilter(filter, _source) {
-    if (filter.field && filter.value) {
-        let fieldValue = getNested(filter.field.split("."), _source);
-        if (!fieldValue) {
-            const fArray = filter.field.split(".");
-            if (fArray[0].startsWith('@')) {
-                const tableName = fArray[0].replace('@', '');
-                if (_source.table === tableName) {
-                    fArray[0] = 'data';
-                    fieldValue = getNested(fArray, _source);
-                }
-            }
-        }
-        if (fieldValue) {
-            if (Array.isArray(fieldValue)) {
-                return fieldValue.indexOf(filter.value) !== -1;
-            } else {
-                return fieldValue === filter.value;
-            }
-        } else {
-            return !filter.value;
-        }
-    } else {
-        return false;
-    }
+	const nextPath = path_array.shift();
+	const nextValue = jsonObj[nextPath];
+	if (!nextValue) {
+		return null;
+	} else {
+		if (typeof nextValue !== 'object') {
+			return nextValue;
+		} else {
+			if (Array.isArray(nextValue)) {
+				return nextValue;
+			} else {
+				return getNested(path_array, nextValue);
+			}
+		}
+	}
 }
 
 export function checkFilter(filter, _source) {
-    if (filter.field && filter.value) {
-        let fieldValue = getNested(filter.field.split("."), _source);
-        if (!fieldValue) {
-            const fArray = filter.field.split(".");
-            if (fArray[0].startsWith('@')) {
-                const actName = fArray[0].replace('@', '');
-                if (_source.act.name === actName) {
-                    fArray[0] = 'data';
-                    fArray.unshift('act');
-                    fieldValue = getNested(fArray, _source);
-                }
-            }
-        }
-        if (fieldValue) {
-            if (Array.isArray(fieldValue)) {
-                return fieldValue.indexOf(filter.value) !== -1;
-            } else {
-                return fieldValue === filter.value;
-            }
-        } else {
-            return !filter.value;
-        }
-    } else {
-        return false;
-    }
+	if (filter.field && filter.value) {
+		let fieldValue = getNested(filter.field.split("."), _source);
+		if (!fieldValue) {
+			const fArray = filter.field.split(".");
+			if (fArray[0].startsWith('@')) {
+				const actName = fArray[0].replace('@', '');
+				if (_source.act.name === actName) {
+					fArray[0] = 'data';
+					fArray.unshift('act');
+					fieldValue = getNested(fArray, _source);
+				}
+			}
+		}
+		if (fieldValue) {
+			if (Array.isArray(fieldValue)) {
+				return fieldValue.indexOf(filter.value) !== -1;
+			} else {
+				return fieldValue === filter.value;
+			}
+		} else {
+			return !filter.value;
+		}
+	} else {
+		return false;
+	}
 }
 
 export function hLog(input: any, ...extra: any[]) {
-    let role;
-    if (process.env.worker_role) {
-        const id = parseInt(process.env.worker_id);
-        role = `[${process.pid} - ${(id < 10 ? '0' : '') + id.toString()}_${process.env.worker_role}]`;
-    } else {
-        if (process.env.script && process.env.script === './api/server.js') {
-            role = `[${process.pid} - api]`;
-        } else {
-            role = `[${process.pid} - 00_master]`;
-        }
-    }
-    if (process.env.TRACE_LOGS === 'true') {
-        const e = new Error();
-        const frame = e.stack.split("\n")[2];
-        const where = frame.split(" ")[6].split(/[:()]/);
-        const arr = where[1].split("/");
-        const fileName = arr[arr.length - 1];
-        const lineNumber = where[2];
-        role += ` ${fileName}:${lineNumber}`;
-    }
-    console.log(role, input, ...extra);
+	let role;
+	if (process.env.worker_role) {
+		const id = parseInt(process.env.worker_id);
+		role = `[${process.pid} - ${(id < 10 ? '0' : '') + id.toString()}_${process.env.worker_role}]`;
+	} else {
+		if (process.env.script && process.env.script === './api/server.js') {
+			role = `[${process.pid} - api]`;
+		} else {
+			role = `[${process.pid} - 00_master]`;
+		}
+	}
+	if (process.env.TRACE_LOGS === 'true') {
+		const e = new Error();
+		const frame = e.stack.split("\n")[2];
+		const where = frame.split(" ")[6].split(/[:()]/);
+		const arr = where[1].split("/");
+		const fileName = arr[arr.length - 1];
+		const lineNumber = where[2];
+		role += ` ${fileName}:${lineNumber}`;
+	}
+	console.log(role, input, ...extra);
 }
 
 export function debugLog(text: any, ...extra: any[]) {
-    if (config.settings.debug) {
-        hLog(text, ...extra);
-    }
-}
-
-export async function sleep(ms): Promise<void> {
-    return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-export async function waitUntilReady(executor: () => Promise<boolean>, attempts: number, interval: number, onError: () => void): Promise<void> {
-    let i = 0;
-    while (i < attempts) {
-        if (await executor()) {
-            return;
-        }
-        await sleep(interval);
-        i++;
-    }
-    onError();
+	if (config.settings.debug) {
+		hLog(text, ...extra);
+	}
 }

helpers/elastic-routes.ts

@@ -145,14 +145,6 @@ function buildResUsageBulk(payloads, messageMap: MMap) {
     });
 }
 
-function buildScheduleBulk(payloads, messageMap: MMap) {
-    return flatMap(payloads, (payload, body) => {
-        const id = `${body.version}`;
-        messageMap.set(id, _.omit(payload, ['content']));
-        return makeScriptedOp(id, body);
-    });
-}
-
 function buildGenTrxBulk(payloads, messageMap: MMap) {
     return flatMap(payloads, (payload, body) => {
         const hash = createHash('sha256')
@@ -191,10 +183,6 @@ const generatorsMap = {
         index_name: 'userres',
         func: buildResUsageBulk
     },
-    schedule: {
-        index_name: 'schedule',
-        func: buildScheduleBulk
-    },
     generated_transaction: {
         index_name: 'gentrx',
         func: buildGenTrxBulk
@@ -205,13 +193,21 @@ const generatorsMap = {
     },
 };
 
+interface IndexDist {
+    index: string;
+    first_block: number;
+    last_block: number;
+}
+
 export class ElasticRoutes {
     public routes: any;
     cm: ConnectionManager;
     chain: string;
     ingestNodeCounters = {};
+    distributionMap: IndexDist[];
 
-    constructor(connectionManager: ConnectionManager) {
+    constructor(connectionManager: ConnectionManager, distributionMap: IndexDist[]) {
+        this.distributionMap = distributionMap;
         this.routes = {generic: this.handleGenericRoute.bind(this)};
         this.cm = connectionManager;
         this.chain = this.cm.chain;
@@ -386,27 +382,27 @@ export class ElasticRoutes {
         return Math.ceil(blockNum / this.cm.config.settings.index_partition_size).toString().padStart(6, '0');
     }
 
-    // getIndexNameByBlock(block_num) {
-    //     if (!this.distributionMap) {
-    //         return null;
-    //     }
-    //     for (let i = this.distributionMap.length - 1; i >= 0; i--) {
-    //         const test = this.distributionMap[i].first_block <= block_num && this.distributionMap[i].last_block >= block_num;
-    //         if (test) {
-    //             return this.distributionMap[i].index;
-    //         }
-    //     }
-    //     return null;
-    // }
-    //
-    // addToIndexMap(map, idx, payload) {
-    //     if (idx) {
-    //         if (!map[idx]) {
-    //             map[idx] = [];
-    //         }
-    //         map[idx].push(payload);
-    //     }
-    // }
+    getIndexNameByBlock(block_num) {
+        if (!this.distributionMap) {
+            return null;
+        }
+        for (let i = this.distributionMap.length - 1; i >= 0; i--) {
+            const test = this.distributionMap[i].first_block <= block_num && this.distributionMap[i].last_block >= block_num;
+            if (test) {
+                return this.distributionMap[i].index;
+            }
+        }
+        return null;
+    }
+
+    addToIndexMap(map, idx, payload) {
+        if (idx) {
+            if (!map[idx]) {
+                map[idx] = [];
+            }
+            map[idx].push(payload);
+        }
+    }
 
     private routeFactory(indexName: string, bulkGenerator, routerFunction) {
         return async (payloads, channel, cb) => {

hpm

@@ -1,3 +0,0 @@
-#!/bin/bash
-node ./scripts/hpm.js "$@"
-# HPM_DEBUG=1 node ./scripts/hpm.js "$@"

hyp-config

@@ -1,2 +0,0 @@
-#!/bin/bash
-node ./scripts/hyp-config.js "$@"

hyp-repair

@@ -1,2 +0,0 @@
-#!/bin/bash
-node ./scripts/hyp-repair.js "$@"

install.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash -
+sudo apt-get install -y nodejs
+sudo npm install pm2 -g
+npm install

install_env.sh

@@ -0,0 +1,303 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+#terminal colors
+COLOR_NC=$(tput sgr0)
+export COLOR_NC
+COLOR_RED=$(tput setaf 1)
+export COLOR_RED
+COLOR_GREEN=$(tput setaf 2)
+export COLOR_GREEN
+COLOR_YELLOW=$(tput setaf 3)
+export COLOR_YELLOW
+COLOR_BLUE=$(tput setaf 4)
+export COLOR_BLUE
+
+#variables
+INITIAL_PARAMS="$*"
+GLOBAL="y"
+RABBIT_USER="hyperion"
+RABBIT_PASSWORD="123456"
+NODE=false
+ELASTIC=false
+REDIS=false
+RABBIT=false
+RAM=0
+
+help_function() {
+    echo -e "\nAutomated shell script that installs all dependencies and then configure Hyperion."
+    echo -e "\nUsage: ./install.env <option>\n"
+    echo -e "Options:"
+    echo -e "  --help ==> Show this menu"
+    echo -e "  --version ==> Show Hyperion current version\n"
+    exit 0
+}
+
+version (){
+  echo -e "\nv3.0.0\n"
+  exit 0
+
+}
+
+arg_checker() {
+    if [ "${INITIAL_PARAMS}" == "--help" ]; then
+        help_function
+
+    elif [ "${INITIAL_PARAMS}" == "--version" ]; then
+        version
+    fi
+}
+
+check_ram() {
+  RAM=$(free --giga | awk '/Mem/ {print $2}')
+}
+
+# first check if is installed, after check version
+check_dependencies() {
+  echo -e "\n\n${COLOR_BLUE}Checking dependencies...${COLOR_NC}\n\n"
+  CMD=$(if (dpkg --compare-versions $(dpkg -s nodejs 2>/dev/null | awk '/Version/ {print $2}') ge "5" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
+    NODE=true
+    echo -e "\n\n${COLOR_BLUE}Nodejs compatible version already installed ${COLOR_NC}"
+  elif ("$CMD" = false); then
+    echo -e "\n\n${COLOR_RED}Nodejs installed version is < 13. Please, update and try again. ${COLOR_NC}\n\n"
+    exit 1
+  else
+    echo -e "\n\n${COLOR_BLUE}Nodejs not installed ${COLOR_NC}"
+  fi
+  CMD=$(if (dpkg --compare-versions $(dpkg -s elasticsearch 2>/dev/null | awk '/Version/ {print $2}') ge "7.6" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
+    ELASTIC=true
+    echo -e "\n${COLOR_BLUE}Elasticsearch compatible version already installed ${COLOR_NC}"
+  elif ("$CMD" = false); then
+    echo -e "\n\n${COLOR_RED}Elasticsearch installed version is < 7.6. Please, update and try again. ${COLOR_NC}\n\n"
+    exit 1
+  else
+    echo -e "\n${COLOR_BLUE}Elasticsearch not installed ${COLOR_NC}"
+  fi
+  CMD=$(if (dpkg --compare-versions $(dpkg -s redis-server 2>/dev/null | awk '/Version/ {print $2}') ge "5" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
+    REDIS=true
+    echo -e "\n${COLOR_BLUE}Redis compatible version already installed ${COLOR_NC}"
+  elif ("$CMD" = false); then
+    echo -e "\n\n${COLOR_RED}Redis installed version is < 5. Please, update and try again. ${COLOR_NC}\n\n"
+    exit 1
+  else
+    echo -e "\n${COLOR_BLUE}Redis not installed ${COLOR_NC}"
+  fi
+  CMD=$(if (dpkg --compare-versions $(dpkg -s rabbitmq-server 2>/dev/null | awk '/Version/ {print $2}') ge "3.8" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
+    RABBIT=true
+    echo -e "\n${COLOR_BLUE}RabbitMQ compatible version already installed ${COLOR_NC}"
+  elif ("$CMD" = false); then
+    echo -e "\n\n${COLOR_RED}RabbitMQ installed version is < 3.8. Please, update and try again. ${COLOR_NC}\n\n"
+    exit 1
+  else
+    echo -e "\n${COLOR_BLUE}RabbitMQ not installed ${COLOR_NC}"
+  fi
+  CMD=$(if (dpkg --compare-versions $(dpkg -s kibana 2>/dev/null | awk '/Version/ {print $2}') ge "7.6" 2>/dev/null); then echo true; else echo false; fi)
+  if ("$CMD" = true); then
+    RABBIT=true
+    echo -e "\n${COLOR_BLUE}Kibana compatible version already installed ${COLOR_NC}"
+  elif ("$CMD" = false); then
+    echo -e "\n\n${COLOR_RED}Kibana installed version is < 7.6. Please, update and try again. ${COLOR_NC}\n\n"
+    exit 1
+  else
+    echo -e "\n${COLOR_BLUE}Kibana not installed ${COLOR_NC}\n\n"
+  fi
+}
+
+# Make a directory for global installations
+configure_npm() {
+  mkdir ~/.npm-global
+  npm config set prefix "$HOME/.npm-global"
+  # export path for the current session
+  export PATH=$HOME/.npm-global/bin:$PATH
+  # make PATH persistent
+  echo "export PATH=~/.npm-global/bin:$PATH" >>"$HOME/.profile"
+  # shellcheck source=/dev/null
+  source ~/.profile
+}
+
+install_keys_sources() {
+  echo -e "\n\n${COLOR_BLUE}Configuring keys and sources...${COLOR_NC}\n\n"
+
+  PPA="https://artifacts.elastic.co/packages/7.x/apt stable main"
+  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
+    wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+    echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
+  fi
+
+  PPA="http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang"
+  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
+    curl -fsSL https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc | sudo apt-key add -
+    echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" | sudo tee /etc/apt/sources.list.d/bintray.rabbitmq.list
+    curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | sudo bash
+  fi
+
+  PPA="https://deb.nodesource.com/node_13.x bionic main"
+  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
+    curl -sL "https://deb.nodesource.com/setup_13.x" | sudo -E bash -
+  fi
+
+  sudo apt update -y
+}
+
+install_dep() {
+ echo -e "\n\n${COLOR_BLUE}Installing dependencies...${COLOR_NC}\n\n"
+ # sudo apt install -y build-essential curl
+ sudo apt install -y curl
+}
+
+install_node() {
+  echo -e "\n\n${COLOR_BLUE}Installing nodejs...${COLOR_NC}\n\n"
+  sudo apt install -y nodejs
+  read -p "Do you want to create a directory for npm global installations [Y/n] : " GLOBAL
+  GLOBAL=${GLOBAL:-y}
+  if [ "$GLOBAL" = "y" ]; then
+    configure_npm
+  fi
+}
+
+install_build_hyperion() {
+  echo -e "\n\n${COLOR_BLUE}Installing packages and building hyperion...${COLOR_NC}\n\n"
+  npm install
+}
+
+install_pm2() {
+  echo -e "\n\n${COLOR_BLUE}Installing pm2...${COLOR_NC}\n\n"
+  if [ "$GLOBAL" = "y" ]; then
+    npm install pm2@latest -g
+  else
+    sudo npm install pm2@latest -g
+  fi
+}
+
+install_redis() {
+  echo -e "\n\n${COLOR_BLUE}Installing redis...${COLOR_NC}\n\n"
+  sudo apt install -y redis-server
+  sudo sed -ie 's/supervised no/supervised systemd/' /etc/redis/redis.conf
+  sudo systemctl restart redis.service
+}
+
+install_erlang() {
+  echo -e "\n\n${COLOR_BLUE}Installing erlang...${COLOR_NC}\n\n"
+  sudo apt -y install erlang
+}
+
+#ask user for rabbit credentials
+rabbit_credentials() {
+  read -p "Enter rabbitmq user [hyperion]: " RABBIT_USER
+  RABBIT_USER=${RABBIT_USER:-hyperion}
+
+  read -p "Enter rabbitmq password [123456]: " RABBIT_PASSWORD
+  RABBIT_PASSWORD=${RABBIT_PASSWORD:-123456}
+
+}
+
+install_rabittmq() {
+  echo -e "\n\n${COLOR_BLUE}Installing rabbit-mq...${COLOR_NC}\n\n"
+  sudo apt install -y rabbitmq-server
+  #enable web gui
+  sudo rabbitmq-plugins enable rabbitmq_management
+  sudo rabbitmqctl add_vhost /hyperion
+  sudo rabbitmqctl add_user ${RABBIT_USER} ${RABBIT_PASSWORD}
+  sudo rabbitmqctl set_user_tags ${RABBIT_USER} administrator
+  sudo rabbitmqctl set_permissions -p /hyperion ${RABBIT_USER} ".*" ".*" ".*"
+}
+
+install_elastic() {
+  echo -e "\n\n${COLOR_BLUE}Installing elastic...${COLOR_NC}\n\n"
+  sudo apt install -y elasticsearch
+
+  echo -e "\n\n${COLOR_BLUE}Configuring elastic...${COLOR_NC}\n\n"
+
+  # edit configs
+  sudo sed -ie 's/#cluster.name: my-application/cluster.name: myCluster/; s/#bootstrap.memory_lock: true/bootstrap.memory_lock: true/' /etc/elasticsearch/elasticsearch.yml
+  # set jvm options based on system RAM
+  check_ram
+  if [ "$RAM" -lt 32 ]; then
+    (( RAM=RAM/2 ))
+    sudo sed -ie 's/-Xms1g/-Xms'"$RAM"'g/; s/-Xmx1g/-Xmx'"$RAM"'g/' /etc/elasticsearch/jvm.options
+  else
+    sudo sed -ie 's/-Xms1g/-Xms16g/; s/-Xmx1g/-Xmx16g/' /etc/elasticsearch/jvm.options
+  fi
+
+  sudo bash -c 'echo "xpack.security.enabled: true" >> /etc/elasticsearch/elasticsearch.yml'
+
+  sudo mkdir -p /etc/systemd/system/elasticsearch.service.d/
+  echo -e "[Service]\nLimitMEMLOCK=infinity" | sudo tee /etc/systemd/system/elasticsearch.service.d/override.conf
+  sudo systemctl daemon-reload
+  sudo service elasticsearch start
+  sudo systemctl enable elasticsearch
+
+  echo -e "\n\n${COLOR_BLUE}Generating Elasticsearch cluster passwords...${COLOR_NC}\n\n"
+
+  echo "y" | sudo /usr/share/elasticsearch/bin/elasticsearch-setup-passwords auto >elastic_pass.txt
+
+}
+
+install_kibana() {
+  echo -e "\n\n${COLOR_BLUE}Installing and configuring Kibana...${COLOR_NC}\n\n"
+  sudo apt install -y kibana
+
+  KIBANA_PASSWORD=$(awk <elastic_pass.txt '/PASSWORD kibana =/ {print $4}')
+
+  sudo sed -ie 's/#elasticsearch.password: "pass"/elasticsearch.password: '"$KIBANA_PASSWORD"'/; s/#elasticsearch.username: "kibana"/elasticsearch.username: "kibana"/' /etc/kibana/kibana.yml
+
+  sudo systemctl enable kibana
+  sudo systemctl start kibana
+}
+
+#******************
+# End of functions
+#******************
+arg_checker
+
+echo -e "\n\n${COLOR_BLUE}*** STARTING HYPERION HISTORY API CONFIGURATION ***${COLOR_NC}\n\n"
+
+check_dependencies
+
+if [ "$RABBIT" = false ]; then
+  rabbit_credentials
+fi
+
+install_dep
+
+install_keys_sources
+if [ "$NODE" = false ]; then
+  install_node
+fi
+
+install_pm2
+
+install_build_hyperion
+
+if [ "$RABBIT" = false ]; then
+  install_rabittmq
+fi
+
+if [ "$REDIS" = false ]; then
+  install_redis
+fi
+
+if [ "$ELASTIC" = false ]; then
+  install_elastic
+  install_kibana
+fi
+
+printf "
+ _   ___   ______  _____ ____  ___ ___  _   _
+| | | \ \ / /  _ \| ____|  _ \|_ _/ _ \| \ | |
+| |_| |\ V /| |_) |  _| | |_) || | | | |  \| |
+|  _  | | | |  __/| |___|  _ < | | |_| | |\  |
+|_| |_| |_| |_|   |_____|_| \_\___\___/|_| \_|
+Made with ♥ by EOS Rio
+"
+echo -e "\n\n${COLOR_GREEN}Hyperion History API successfully installed!!${COLOR_NC}"
+echo -e "\n${COLOR_YELLOW}Please, check your installation: ${COLOR_NC}"
+echo -e "${COLOR_YELLOW}- Elastic: http://localhost:9200 ${COLOR_NC}"
+echo -e "${COLOR_YELLOW}- Kibana: http://localhost:5601 ${COLOR_NC}"
+echo -e "${COLOR_YELLOW}- RabbitMQ: http://localhost:15672 ${COLOR_NC}"
+echo -e "${COLOR_YELLOW}- Refer to the elastic_pas.txt file for the elasticsearch passwords. ${COLOR_NC}"
+echo -e "${COLOR_YELLOW}- Make pm2 auto-boot at server restart: \$ pm2 startup ${COLOR_NC}\n"

interfaces/hyperionConfig.ts

@@ -1,193 +1,173 @@
 import {AlertManagerOptions} from "../modules/alertsManager";
 
 export interface ScalingConfigs {
-    polling_interval: number;
-    resume_trigger: number;
-    max_queue_limit: number;
-    block_queue_limit: number;
-    routing_mode: string;
-    batch_size: number;
-    queue_limit: number;
-    readers: number;
-    ds_queues: number;
-    ds_threads: number;
-    ds_pool_size: number;
-    indexing_queues: number;
-    ad_idx_queues: number;
-    dyn_idx_queues: number;
-    max_autoscale: number;
-    auto_scale_trigger: number;
+	polling_interval: number;
+	resume_trigger: number;
+	max_queue_limit: number;
+	block_queue_limit: number;
+	routing_mode: string;
+	batch_size: number;
+	queue_limit: number;
+	readers: number;
+	ds_queues: number;
+	ds_threads: number;
+	ds_pool_size: number;
+	indexing_queues: number;
+	ad_idx_queues: number;
+	dyn_idx_queues: number;
+	max_autoscale: number;
+	auto_scale_trigger: number;
 }
 
 export interface MainSettings {
-    use_global_agent?: boolean;
-    process_prefix?: string;
-    ignore_snapshot?: boolean;
-    ship_request_rev: string;
-    custom_policy: string;
-    bypass_index_map: boolean;
-    hot_warm_policy: boolean;
-    auto_mode_switch: boolean;
-    ds_profiling: boolean;
-    max_ws_payload_mb: number;
-    ipc_debug_rate?: number;
-    bp_monitoring?: boolean;
-    preview: boolean;
-    chain: string;
-    eosio_alias: string;
-    parser: string;
-    auto_stop: number;
-    index_version: string;
-    debug: boolean;
-    rate_monitoring: boolean;
-    bp_logs: boolean;
-    dsp_parser: boolean;
-    allow_custom_abi: boolean;
-    index_partition_size: number;
-    es_replicas: number;
+	process_prefix?: string;
+	index_partition_size: number;
+	ignore_snapshot?: boolean;
+	ship_request_rev: string;
+	custom_policy: string;
+	bypass_index_map: boolean;
+	hot_warm_policy: boolean;
+	auto_mode_switch: boolean;
+	ds_profiling: boolean;
+	max_ws_payload_mb: number;
+	ipc_debug_rate?: number;
+	bp_monitoring?: boolean;
+	preview: boolean;
+	chain: string;
+	eosio_alias: string;
+	parser: string;
+	auto_stop: number;
+	index_version: string;
+	debug: boolean;
+	rate_monitoring: boolean;
+	bp_logs: boolean;
+	dsp_parser: boolean;
+	allow_custom_abi: boolean;
 }
 
 export interface IndexerConfigs {
-    enabled?: boolean;
-    node_max_old_space_size?: number;
-    fill_state: boolean;
-    start_on: number;
-    stop_on: number;
-    rewrite: boolean;
-    purge_queues: boolean;
-    live_reader: boolean;
-    live_only_mode: boolean;
-    abi_scan_mode: boolean;
-    fetch_block: boolean;
-    fetch_traces: boolean;
-    fetch_deltas: boolean;
-    disable_reading: boolean;
-    disable_indexing: boolean;
-    process_deltas: boolean;
-    repair_mode: boolean;
-    max_inline: number;
-    disable_delta_rm?: boolean;
+	fill_state: boolean;
+	start_on: number;
+	stop_on: number;
+	rewrite: boolean;
+	purge_queues: boolean;
+	live_reader: boolean;
+	live_only_mode: boolean;
+	abi_scan_mode: boolean;
+	fetch_block: boolean;
+	fetch_traces: boolean;
+	fetch_deltas: boolean;
+	disable_reading: boolean;
+	disable_indexing: boolean;
+	process_deltas: boolean;
+	repair_mode: boolean;
+	max_inline: number;
+	disable_delta_rm?: boolean;
 }
 
 interface ApiLimits {
-    get_links?: number;
-    get_actions?: number;
-    get_blocks?: number;
-    get_created_accounts?: number;
-    get_deltas?: number;
-    get_key_accounts?: number;
-    get_proposals?: number;
-    get_tokens?: number;
-    get_transfers?: number;
-    get_voters?: number;
-    get_trx_actions?: number;
-}
-
-interface CachedRouteConfig {
-    path: string;
-    ttl: number
+	get_links?: number;
+	get_actions?: number;
+	get_blocks?: number;
+	get_created_accounts?: number;
+	get_deltas?: number;
+	get_key_accounts?: number;
+	get_proposals?: number;
+	get_tokens?: number;
+	get_transfers?: number;
+	get_voters?: number;
+	get_trx_actions?: number;
 }
 
 interface ApiConfigs {
-    stream_scroll_batch?: number;
-    stream_scroll_limit?: number;
-    enabled?: boolean;
-    pm2_scaling?: number;
-    node_max_old_space_size?: number;
-    disable_rate_limit?: boolean;
-    disable_tx_cache?: boolean;
-    tx_cache_expiration_sec?: number | string;
-    rate_limit_rpm?: number;
-    rate_limit_allow?: string[];
-    custom_core_token?: string;
-    chain_api_error_log?: boolean;
-    chain_api?: string;
-    push_api?: string;
-    access_log: boolean;
-    chain_name: string;
-    server_port: number;
-    stream_port: number;
-    server_addr: string;
-    server_name: string;
-    provider_name: string;
-    provider_url: string;
-    provider_logo: string;
-    chain_logo_url: string;
-    enable_caching: boolean,
-    cache_life: number;
-    limits: ApiLimits,
-    v1_chain_cache?: CachedRouteConfig[]
+	disable_tx_cache?: boolean;
+	tx_cache_expiration_sec?: number | string;
+	rate_limit_rpm?: number;
+	rate_limit_allow?: string[];
+	custom_core_token?: string;
+	chain_api_error_log?: boolean;
+	chain_api?: string;
+	push_api?: string;
+	enable_explorer?: boolean;
+	access_log: boolean;
+	chain_name: string;
+	server_port: number;
+	server_addr: string;
+	server_name: string;
+	provider_name: string;
+	provider_url: string;
+	provider_logo: string;
+	chain_logo_url: string;
+	enable_caching: boolean,
+	cache_life: number;
+	limits: ApiLimits
 }
 
 interface HubLocation {
-    city: string,
-    country: string,
-    lat: number,
-    lon: number
+	city: string,
+	country: string,
+	lat: number,
+	lon: number
 }
 
 interface HyperionHubConfigs {
-    location: HubLocation;
-    production: boolean;
-    publisher_key: string;
-    inform_url: string;
+	location: HubLocation;
+	production: boolean;
+	publisher_key: string;
+	inform_url: string;
 }
 
 export interface HyperionConfig {
-    api: ApiConfigs;
+	hub: HyperionHubConfigs;
+	settings: MainSettings;
+	scaling: ScalingConfigs;
+	indexer: IndexerConfigs;
 
-    settings: MainSettings;
+	api: ApiConfigs;
 
-    hub: HyperionHubConfigs;
-    scaling: ScalingConfigs;
+	blacklists: {
+		actions: string[],
+		deltas: string[]
+	};
 
-    indexer: IndexerConfigs;
+	whitelists: {
+		max_depth: number;
+		root_only: boolean,
+		actions: string[],
+		deltas: string[]
+	};
 
-    blacklists: {
-        actions: string[],
-        deltas: string[]
-    };
+	features: {
+		streaming: {
+			enable: boolean,
+			traces: boolean,
+			deltas: boolean
+		},
+		tables: {
+			proposals: boolean,
+			accounts: boolean,
+			voters: boolean,
+			userres: boolean,
+			delband: boolean
+		},
+		index_deltas: boolean,
+		index_transfer_memo: boolean,
+		index_all_deltas: boolean,
+		deferred_trx: boolean,
+		failed_trx: boolean,
+		resource_usage: boolean,
+		resource_limits: boolean,
+	};
 
-    whitelists: {
-        max_depth: number;
-        root_only: boolean,
-        actions: string[],
-        deltas: string[]
-    };
+	prefetch: {
+		read: number,
+		block: number,
+		index: number
+	};
 
-    features: {
-        streaming: {
-            enable: boolean,
-            traces: boolean,
-            deltas: boolean
-        },
-        tables: {
-            proposals: boolean,
-            accounts: boolean,
-            voters: boolean,
-            userres: boolean,
-            delband: boolean
-        },
-        index_deltas: boolean,
-        index_transfer_memo: boolean,
-        index_all_deltas: boolean,
-        deferred_trx: boolean,
-        failed_trx: boolean,
-        resource_usage: boolean,
-        resource_limits: boolean,
-    };
+	experimental: any;
 
-    prefetch: {
-        read: number,
-        block: number,
-        index: number
-    };
+	plugins: any;
 
-    experimental: any;
-
-    plugins: {
-        [key: string]: any;
-    };
-
-    alerts: AlertManagerOptions;
+	alerts: AlertManagerOptions;
 }

interfaces/hyperionConnections.ts

@@ -20,10 +20,8 @@ interface HyperionChainData {
     name: string;
     chain_id: string;
     http: string;
-    ship: string;
     WS_ROUTER_PORT: number;
     WS_ROUTER_HOST: string;
-    control_port: number;
 }
 
 interface RedisConfig {

launcher.ts

@@ -13,7 +13,6 @@ const hyperionWorkers = {
 	reader: 'state-reader',
 	deserializer: 'deserializer',
 	continuous_reader: 'state-reader',
-	repair_reader: 'state-reader',
 	ingestor: 'indexer',
 	router: 'ws-router',
 	delta_updater: 'delta-updater'

modules/action_data/act.js

@@ -1,39 +0,0 @@
-const hyperionModule = {
-    chain: "*",
-    contract: '*',
-    action: 'act',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
-    defineQueryPrefix: 'act',
-
-    mappings: {
-        action: {
-            "@act": {
-                "properties": {
-                    "coopname": {"type": "keyword"},
-                    "username": {"type": "keyword"},
-                    "action": {"type": "keyword"},
-                    "decision_id": {"type": "long"},
-                    "batch_id": {"type": "long"},
-                    "act": {"enabled": false}
-                }
-            }
-        }
-    },
-
-    handler: (action) => {
-        const data = action['act']['data'];
-        
-        action['@act'] = {
-            coopname: data.coopname,
-            username: data.username,
-            action: data.action,
-            decision_id: data.decision_id,
-            batch_id: data.batch_id,
-            act: data.act
-        };
-
-        
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/batch.js

@@ -1,33 +0,0 @@
-const hyperionModule = {
-    chain: "*",
-    contract: '*',
-    action: 'batch',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
-    defineQueryPrefix: 'batch',
-        
-    mappings: {
-        action: {
-            "@batch": {
-                "properties": {
-                    "coopname": {"type": "keyword"},
-                    "action": {"type": "keyword"},
-                    "batch_id": {"type": "long"}
-                }
-            }
-        }
-    },
-
-    handler: (action) => {
-        const data = action['act']['data'];
-        
-        action['@batch'] = {
-            coopname: data.coopname,
-            action: data.action,
-            batch_id: data.batch_id,
-        };
-
-        
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/daobet-eosio-delegatebw.js

@@ -0,0 +1,45 @@
+const hyperionModule = {
+    chain: "74d023a9293d9b68c3c52e2f738ee681c1671cc3dc0f263cf2c533cd5523ff95",
+    contract: 'eosio',
+    action: 'delegatebw',
+    parser_version: ['1.7'],
+    defineQueryPrefix: 'delegatebw',
+    mappings: {
+        action: {
+            "@delegatebw": {
+                "properties": {
+                    "from": {"type": "keyword"},
+                    "receiver": {"type": "keyword"},
+                    "stake_cpu_quantity": {"type": "float"},
+                    "stake_net_quantity": {"type": "float"},
+                    "stake_vote_quantity": {"type": "float"},
+                    "transfer": {"type": "boolean"},
+                    "amount": {"type": "float"}
+                }
+            }
+        }
+    },
+    handler: (action) => {
+        const data = action['act']['data'];
+        let cpu_qtd = null;
+        let net_qtd = null;
+        let vote_qtd = null;
+        if (data['stake_net_quantity'] && data['stake_cpu_quantity'] && data['stake_vote_quantity']) {
+            cpu_qtd = parseFloat(data['stake_cpu_quantity'].split(' ')[0]);
+            net_qtd = parseFloat(data['stake_net_quantity'].split(' ')[0]);
+            vote_qtd = parseFloat(data['stake_vote_quantity'].split(' ')[0]);
+        }
+        action['@delegatebw'] = {
+            amount: cpu_qtd + net_qtd + vote_qtd,
+            stake_cpu_quantity: cpu_qtd,
+            stake_net_quantity: net_qtd,
+            stake_vote_quantity: vote_qtd,
+            from: data['from'],
+            receiver: data['receiver'],
+            transfer: data['transfer']
+        };
+        delete action['act']['data'];
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/daobet-eosio-undelegatebw.js

@@ -0,0 +1,43 @@
+const hyperionModule = {
+    chain: "74d023a9293d9b68c3c52e2f738ee681c1671cc3dc0f263cf2c533cd5523ff95",
+    contract: 'eosio',
+    action: 'undelegatebw',
+    parser_version: ['1.7'],
+    defineQueryPrefix: 'undelegatebw',
+    mappings: {
+        action: {
+            "@undelegatebw": {
+                "properties": {
+                    "from": {"type": "keyword"},
+                    "receiver": {"type": "keyword"},
+                    "unstake_cpu_quantity": {"type": "float"},
+                    "unstake_net_quantity": {"type": "float"},
+                    "unstake_vote_quantity": {"type": "float"},
+                    "amount": {"type": "float"}
+                }
+            }
+        }
+    },
+    handler: (action) => {
+        const data = action['act']['data'];
+        let cpu_qtd = null;
+        let net_qtd = null;
+        let vote_qtd = null;
+        if (data['unstake_net_quantity'] && data['unstake_cpu_quantity'] && data['unstake_vote_quantity']) {
+            cpu_qtd = parseFloat(data['unstake_cpu_quantity'].split(' ')[0]);
+            net_qtd = parseFloat(data['unstake_net_quantity'].split(' ')[0]);
+            vote_qtd = parseFloat(data['unstake_vote_quantity'].split(' ')[0]);
+        }
+        action['@undelegatebw'] = {
+            amount: cpu_qtd + net_qtd + vote_qtd,
+            unstake_cpu_quantity: cpu_qtd,
+            unstake_net_quantity: net_qtd,
+            unstake_vote_quantity: vote_qtd,
+            from: data['from'],
+            receiver: data['receiver']
+        };
+        delete action['act']['data'];
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/decision.js

@@ -1,39 +0,0 @@
-const hyperionModule = {
-    chain: "*",
-    contract: '*',
-    action: 'decision',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
-    defineQueryPrefix: 'decision',
-    
-    mappings: {
-        action: {
-            "@decision": {
-                "properties": {
-                    "coopname": {"type": "keyword"},
-                    "username": {"type": "keyword"},
-                    "action": {"type": "keyword"},
-                    "decision_id": {"type": "long"},
-                    "batch_id": {"type": "long"},
-                    "decision": {"enabled": false}
-                }
-            }
-        }
-    },
-
-    handler: (action) => {
-        const data = action['act']['data'];
-           
-        action['@decision'] = {
-            coopname: data.coopname,
-            username: data.username,
-            action: data.action,
-            decision_id: data.decision_id,
-            batch_id: data.batch_id,
-            decision: data.decision
-        };
-
-        
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/eosio-buyram.js

@@ -1,20 +1,20 @@
 const hyperionModule = {
-    chain: '*',
-    contract: 'eosio',
-    action: 'buyram',
-    defineQueryPrefix: 'buyram',
-    parser_version: ['3.2', '2.1', '1.8', '1.7'],
-    handler: (action) => {
-        const data = action['act']['data'];
-        action['@buyram'] = {
-            payer: data['payer'],
-            receiver: data['receiver'],
-        };
-        if (data['quant']) {
-            action['@buyram']['quant'] = parseFloat(data['quant'].split(' ')[0]);
-        }
-        delete action['act']['data'];
-    },
+  chain: '*',
+  contract: 'eosio',
+  action: 'buyram',
+  defineQueryPrefix: 'buyram',
+  parser_version: ['2.1', '1.8', '1.7'],
+  handler: (action) => {
+    const data = action['act']['data'];
+    action['@buyram'] = {
+      payer: data['payer'],
+      receiver: data['receiver'],
+    };
+    if (data['quant']) {
+      action['@buyram']['quant'] = parseFloat(data['quant'].split(' ')[0]);
+    }
+    delete action['act']['data'];
+  },
 };
 
 module.exports = {hyperionModule};

modules/action_data/eosio-buyrambytes.js

@@ -3,7 +3,7 @@ const hyperionModule = {
     contract: 'eosio',
     action: 'buyrambytes',
     defineQueryPrefix: 'buyrambytes',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
+    parser_version: ['2.1','1.8','1.7'],
     handler: (action) => {
         const data = action['act']['data'];
         action['@buyrambytes'] = {

modules/action_data/eosio-buyrex.js

@@ -3,7 +3,7 @@ const hyperionModule = {
     contract: 'eosio',
     action: 'buyrex',
     defineQueryPrefix: 'buyrex',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
+    parser_version: ['2.1','1.8','1.7'],
     handler: (action) => {
         const data = action['act']['data'];
         let qtd = null;

modules/action_data/eosio-delegatebw.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'delegatebw',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
+    parser_version: ['2.1','1.8','1.7'],
     defineQueryPrefix: 'delegatebw',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-newaccount.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'newaccount',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
+    parser_version: ['2.1','1.8','1.7'],
     defineQueryPrefix: 'newaccount',
     handler: (action) => {
         let name = null;

modules/action_data/eosio-undelegatebw.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'undelegatebw',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
+    parser_version: ['2.1','1.8','1.7'],
     defineQueryPrefix: 'undelegatebw',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-unstaketorex.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'unstaketorex',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
+    parser_version: ['2.1','1.8','1.7'],
     defineQueryPrefix: 'unstaketorex',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-updateauth.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'updateauth',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
+    parser_version: ['2.1','1.8','1.7'],
     defineQueryPrefix: 'updateauth',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-voteproducer.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'voteproducer',
-    parser_version: ['3.2', '2.1','1.8', '1.7'],
+    parser_version: ['2.1','1.8', '1.7'],
     defineQueryPrefix: 'voteproducer',
     mappings: {
         action: {

modules/action_data/exec.js

@@ -1,33 +0,0 @@
-const hyperionModule = {
-    chain: "*",
-    contract: '*',
-    action: 'exec',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
-    defineQueryPrefix: 'exec',
-
-    mappings: {
-        action: {
-            "@exec": {
-                "properties": {
-                    "executer": {"type": "keyword"},
-                    "coopname": {"type": "keyword"},
-                    "decision_id": {"type": "long"}
-                }
-            }
-        }
-    },
-
-    handler: (action) => {
-        const data = action['act']['data'];
-        
-        action['@exec'] = {
-            executer: data.executer,
-            coopname: data.coopname,
-            decision_id: data.decision_id,
-        };
-
-        
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/fio-bprewdupdate.js

@@ -2,7 +2,7 @@ const hyperionModule = {
   chain: '21dcae42c0182200e93f954a074011f9048a7624c6fe81d3c9541a614a88bd1c',
   contract: 'fio.treasury',
   action: 'bprewdupdate',
-  parser_version: ['3.2', '1.8'],
+  parser_version: ['1.8'],
   defineQueryPrefix: 'bprewdupdate',
   mappings: {
     action: {

modules/action_data/fio-fdtnrwdupdat.js

@@ -2,7 +2,7 @@ const hyperionModule = {
   chain: '21dcae42c0182200e93f954a074011f9048a7624c6fe81d3c9541a614a88bd1c',
   contract: 'fio.treasury',
   action: 'fdtnrwdupdat',
-  parser_version: ['3.2', '1.8'],
+  parser_version: ['1.8'],
   defineQueryPrefix: 'fdtnrwdupdat',
   mappings: {
     action: {

modules/action_data/statement.js

@@ -1,39 +0,0 @@
-const hyperionModule = {
-    chain: "*",
-    contract: '*',
-    action: 'statement',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
-    defineQueryPrefix: 'statement',
-
-    mappings: {
-        action: {
-            "@statement": {
-                "properties": {
-                    "coopname": {"type": "keyword"},
-                    "username": {"type": "keyword"},
-                    "action": {"type": "keyword"},
-                    "decision_id": {"type": "long"},
-                    "batch_id": {"type": "long"},
-                    "statement": {"enabled": false}
-                }
-            }
-        }
-    },
-
-    handler: (action) => {
-        const data = action['act']['data'];
-           
-        action['@statement'] = {
-            coopname: data.coopname,
-            username: data.username,
-            action: data.action,
-            decision_id: data.decision_id,
-            batch_id: data.batch_id,
-            statement: data.statement
-        };
-
-        
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/transfer.js

@@ -4,7 +4,7 @@ const hyperionModule = {
   chain: '*',
   contract: '*',
   action: 'transfer',
-  parser_version: ['3.2', '2.1', '1.8', '1.7'],
+  parser_version: ['2.1','1.8', '1.7'],
   defineQueryPrefix: 'transfer',
   handler: (action) => {
     let qtd = null;

modules/action_data/voteagainst.js

@@ -1,33 +0,0 @@
-const hyperionModule = {
-    chain: "*",
-    contract: '*',
-    action: 'voteagainst',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
-    defineQueryPrefix: 'voteagainst',
-
-    mappings: {
-        action: {
-            "@voteagainst": {
-                "properties": {
-                    "coopname": {"type": "keyword"},
-                    "member": {"type": "keyword"},
-                    "decision_id": {"type": "long"}
-                }
-            }
-        }
-    },
-
-    handler: (action) => {
-        const data = action['act']['data'];
-        
-        action['@voteagainst'] = {
-            coopname: data.coopname,
-            member: data.member,
-            decision_id: data.decision_id
-        };
-
-        
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/votefor.js

@@ -1,33 +0,0 @@
-const hyperionModule = {
-    chain: "*",
-    contract: '*',
-    action: 'votefor',
-    parser_version: ['3.2', '2.1','1.8','1.7'],
-    defineQueryPrefix: 'votefor',
-
-    mappings: {
-        action: {
-            "@votefor": {
-                "properties": {
-                    "coopname": {"type": "keyword"},
-                    "member": {"type": "keyword"},
-                    "decision_id": {"type": "long"}
-                }
-            }
-        }
-    },
-
-    handler: (action) => {
-        const data = action['act']['data'];
-        
-        action['@votefor'] = {
-            coopname: data.coopname,
-            member: data.member,
-            decision_id: data.decision_id
-        };
-
-        
-    }
-};
-
-module.exports = {hyperionModule};

modules/index.js

@@ -53,12 +53,12 @@ class HyperionModuleLoader {
                     const key = `${_module.contract}::${_module.action}`;
                     if (this.chainMappings.has(key)) {
                         if (this.chainMappings.get(key) === '*' && _module.chain === chainID) {
-                            console.log('Overwriting module ' + key + ' for ' + _module.chain);
+                            // console.log('Overwriting module ' + key + ' for ' + _module.chain);
                             this.includeModule(_module);
                             this.chainMappings.set(key, _module.chain);
                         }
                     } else {
-                        console.log('Including module ' + key + ' for ' + _module.chain);
+                        // console.log('Including module ' + key + ' for ' + _module.chain);
                         this.includeModule(_module);
                         this.chainMappings.set(key, _module.chain);
                     }

modules/loader.ts

@@ -5,217 +5,189 @@ import {HyperionConnections} from "../interfaces/hyperionConnections";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
 import {BaseParser} from "./parsers/base-parser";
 import {hLog} from "../helpers/common_functions";
-import {HyperionPlugin, HyperionStreamHandler} from "../plugins/hyperion-plugin";
+import {HyperionPlugin} from "../plugins/hyperion-plugin";
 
 export class HyperionModuleLoader {
 
-    private handledActions = new Map();
-    private handledDeltas = new Map();
-    private streamHandlers: HyperionStreamHandler[] = [];
+	private handledActions = new Map();
+	private handledDeltas = new Map();
+	chainMappings = new Map();
+	extendedActions: Set<string> = new Set();
+	extraMappings = [];
+	chainID;
+	private conn: HyperionConnections;
+	private config: HyperionConfig;
+	public parser: BaseParser;
+	public plugins: HyperionPlugin[];
 
-    chainMappings = new Map();
-    extendedActions: Set<string> = new Set();
-    extraMappings = [];
-    chainID;
-    private conn: HyperionConnections;
-    private config: HyperionConfig;
-    public parser: BaseParser;
-    public plugins: HyperionPlugin[];
+	constructor(private cm: ConfigurationModule) {
+		this.plugins = [];
+		this.conn = cm.connections;
+		this.config = cm.config;
+		if (!this.conn.chains[this.config.settings.chain]) {
+			console.log('Chain  ' + this.config.settings.chain + ' not defined on connections.json!');
+			process.exit(0);
+		}
+		this.chainID = this.conn.chains[this.config.settings.chain].chain_id;
+		this.loadActionHandlers();
+	}
 
-    constructor(private cm: ConfigurationModule) {
-        this.plugins = [];
-        this.conn = cm.connections;
-        this.config = cm.config;
-        if (!this.conn.chains[this.config.settings.chain]) {
-            console.log('Chain  ' + this.config.settings.chain + ' not defined on connections.json!');
-            process.exit(0);
-        }
-        this.chainID = this.conn.chains[this.config.settings.chain].chain_id;
-        this.loadActionHandlers();
-    }
+	async loadParser() {
+		const path = join(__dirname, 'parsers', this.config.settings.parser + "-parser");
+		const mod = (await import(path)).default;
+		this.parser = new mod(this.cm) as BaseParser;
+	}
 
-    async loadParser() {
-        const path = join(__dirname, 'parsers', this.config.settings.parser + "-parser");
-        const mod = (await import(path)).default;
-        this.parser = new mod(this.cm) as BaseParser;
-    }
+	processActionData(action) {
+		const wildcard = this.handledActions.get('*');
+		if (wildcard && wildcard.has(action.act.name)) {
+			wildcard.get(action.act.name)(action);
+		}
+		if (this.handledActions && this.handledActions.has(action.act.account)) {
+			const _c = this.handledActions.get(action.act.account);
+			if (_c.has(action.act.name)) {
+				_c.get(action.act.name)(action);
+			}
+		}
+	}
 
-    processActionData(action) {
-        const wildcard = this.handledActions.get('*');
-        if (wildcard && wildcard.has(action.act.name)) {
-            wildcard.get(action.act.name)(action);
-        }
-        if (this.handledActions && this.handledActions.has(action.act.account)) {
-            const _c = this.handledActions.get(action.act.account);
-            if (_c.has(action.act.name)) {
-                _c.get(action.act.name)(action);
-            }
-        }
-    }
+	async processDeltaData(delta): Promise<void> {
+		if (this.handledDeltas.has(delta.code)) {
+			const _c = this.handledDeltas.get(delta.code);
+			if (_c.has(delta.table)) {
+				await _c.get(delta.table)(delta);
+			}
+		}
+	}
 
-    async processDeltaData(delta): Promise<void> {
-        if (this.handledDeltas.has(delta.code)) {
-            const _c = this.handledDeltas.get(delta.code);
-            if (_c.has(delta.table)) {
-                await _c.get(delta.table)(delta);
-            }
-        }
-    }
+	includeActionModule(_module) {
+		if (this.handledActions.has(_module.contract)) {
+			const existing = this.handledActions.get(_module.contract);
+			existing.set(_module.action, _module.handler);
+		} else {
+			const _map = new Map();
+			_map.set(_module.action, _module.handler);
+			this.handledActions.set(_module.contract, _map);
+		}
+		if (_module.mappings) {
+			this.extraMappings.push(_module.mappings);
+		}
+		if (_module.defineQueryPrefix) {
+			this.extendedActions.add(_module.defineQueryPrefix);
+		}
+	}
 
+	includeDeltaModule(deltaModule) {
+		if (this.handledDeltas.has(deltaModule.contract)) {
+			const existing = this.handledDeltas.get(deltaModule.contract);
+			existing.set(deltaModule.table, deltaModule.handler);
+		} else {
+			const _map = new Map();
+			_map.set(deltaModule.table, deltaModule.handler);
+			this.handledDeltas.set(deltaModule.contract, _map);
+		}
+		if (deltaModule.mappings) {
+			this.extraMappings.push(deltaModule.mappings);
+		}
+	}
 
-    includeActionModule(_module) {
-        if (this.handledActions.has(_module.contract)) {
-            const existing = this.handledActions.get(_module.contract);
-            existing.set(_module.action, _module.handler);
-        } else {
-            const _map = new Map();
-            _map.set(_module.action, _module.handler);
-            this.handledActions.set(_module.contract, _map);
-        }
-        if (_module.mappings) {
-            this.extraMappings.push(_module.mappings);
-        }
-        if (_module.defineQueryPrefix) {
-            this.extendedActions.add(_module.defineQueryPrefix);
-        }
-    }
+	loadActionHandlers() {
+		const files = readdirSync('modules/action_data/');
+		for (const plugin of files) {
+			const _module = require(join(__dirname, 'action_data', plugin)).hyperionModule;
+			if (_module.parser_version.includes(this.config.settings.parser)) {
+				if (_module.chain === this.chainID || _module.chain === '*') {
+					const key = `${_module.contract}::${_module.action}`;
+					if (this.chainMappings.has(key)) {
+						if (this.chainMappings.get(key) === '*' && _module.chain === this.chainID) {
+							this.includeActionModule(_module);
+							this.chainMappings.set(key, _module.chain);
+						}
+					} else {
+						this.includeActionModule(_module);
+						this.chainMappings.set(key, _module.chain);
+					}
+				}
+			}
+		}
+	}
 
-    includeDeltaModule(deltaModule) {
-        if (this.handledDeltas.has(deltaModule.contract)) {
-            const existing = this.handledDeltas.get(deltaModule.contract);
-            existing.set(deltaModule.table, deltaModule.handler);
-        } else {
-            const _map = new Map();
-            _map.set(deltaModule.table, deltaModule.handler);
-            this.handledDeltas.set(deltaModule.contract, _map);
-        }
-        if (deltaModule.mappings) {
-            this.extraMappings.push(deltaModule.mappings);
-        }
-    }
+	async init() {
+		try {
+			await this.loadParser();
+			await this.loadPlugins();
+		} catch (e) {
+			console.log(e);
+			process.exit(1);
+		}
+	}
 
-    loadActionHandlers() {
-        const files = readdirSync('modules/action_data/');
-        for (const plugin of files) {
-            const _module = require(join(__dirname, 'action_data', plugin)).hyperionModule;
-            if (_module.parser_version.includes(this.config.settings.parser)) {
-                if (_module.chain === this.chainID || _module.chain === '*') {
-                    const key = `${_module.contract}::${_module.action}`;
-                    if (this.chainMappings.has(key)) {
-                        if (this.chainMappings.get(key) === '*' && _module.chain === this.chainID) {
-                            this.includeActionModule(_module);
-                            this.chainMappings.set(key, _module.chain);
-                        }
-                    } else {
-                        this.includeActionModule(_module);
-                        this.chainMappings.set(key, _module.chain);
-                    }
-                }
-            }
-        }
-    }
+	// main loader function for plugin modules
+	private async loadPlugins() {
+		const base = join(__dirname, '..', 'plugins');
+		if (!existsSync(base)) {
+			return;
+		}
+		const repos = join(base, 'repos');
+		if (!existsSync(repos)) {
+			return;
+		}
+		const state = join(base, '.state.json');
+		if (!existsSync(state)) {
+			return;
+		}
 
-    async init() {
-        try {
-            await this.loadParser();
-            await this.loadPlugins();
-        } catch (e) {
-            console.log(e);
-            process.exit(1);
-        }
-    }
+		let pluginState;
+		try {
+			pluginState = JSON.parse(readFileSync(state).toString());
+		} catch (e) {
+			hLog('Failed to read plugin state');
+			return;
+		}
 
-    // main loader function for plugin modules
-    private async loadPlugins() {
-        const base = join(__dirname, '..', 'plugins');
-        if (!existsSync(base)) {
-            return;
-        }
-        const repos = join(base, 'repos');
-        if (!existsSync(repos)) {
-            return;
-        }
-        const state = join(base, '.state.json');
-        if (!existsSync(state)) {
-            return;
-        }
+		for (const pluginName of Object.keys(pluginState)) {
+			if (pluginState[pluginName] && this.config.plugins[pluginName]) {
+				if (pluginState[pluginName].enabled && this.config.plugins[pluginName].enabled) {
+					try {
+						const pluginModule = (await import(join(repos, pluginName))).default;
+						const pl = new pluginModule(this.config.plugins[pluginName]);
+						if (pl.actionHandlers) {
+							this.loadPluginActionHandlers(pl.actionHandlers);
+						}
+						if (pl.deltaHandlers) {
+							this.loadPluginDeltaHandlers(pl.deltaHandlers);
+						}
+						this.plugins.push(pl);
+					} catch (e) {
+						hLog(`Plugin "${pluginName}" failed to load: ${e.message}`);
+					}
+				}
+			}
+		}
+	}
 
+	appendDynamicContracts(allowedDynamicContracts: Set<string>) {
+		for (const plugin of this.plugins) {
+			if (plugin.dynamicContracts) {
+				if (plugin.dynamicContracts.length > 0) {
+					plugin.dynamicContracts.forEach(value => {
+						allowedDynamicContracts.add(value);
+					});
+				}
+			}
+		}
+	}
 
-        let pState;
-        try {
-            const stateFile = JSON.parse(readFileSync(state).toString());
-            pState = stateFile.plugins;
-        } catch (e) {
-            hLog('Failed to read plugin state');
-            return;
-        }
+	private loadPluginActionHandlers(actionHandlers: any) {
+		for (const handler of actionHandlers) {
+			this.includeActionModule(handler);
+		}
+	}
 
-        for (const key in this.config.plugins) {
-            if (this.config.plugins.hasOwnProperty(key)) {
-                if (pState[key] && pState[key].enabled && this.config.plugins[key].enabled) {
-                    try {
-                        const pMod = (await import(join(repos, key))).default;
-                        const pl = new pMod(this.config.plugins[key]);
-                        if (pl.actionHandlers) {
-                            this.loadPluginActionHandlers(pl.actionHandlers);
-                        }
-                        if (pl.deltaHandlers) {
-                            this.loadPluginDeltaHandlers(pl.deltaHandlers);
-                        }
-                        if (pl.streamHandlers) {
-                            this.loadPluginStreamHandlers(pl.streamHandlers);
-                        }
-                        this.plugins.push(pl);
-                    } catch (e) {
-                        hLog(`Plugin "${key}" failed to load: ${e.message}`);
-                    }
-                }
-            }
-        }
-    }
+	private loadPluginDeltaHandlers(deltaHandlers: any) {
+		for (const handler of deltaHandlers) {
+			this.includeDeltaModule(handler);
+		}
+	}
 
-    appendDynamicContracts(allowedDynamicContracts: Set<string>) {
-        for (const plugin of this.plugins) {
-            if (plugin.dynamicContracts) {
-                if (plugin.dynamicContracts.length > 0) {
-                    plugin.dynamicContracts.forEach(value => {
-                        allowedDynamicContracts.add(value);
-                    });
-                }
-            }
-        }
-    }
-
-
-    private loadPluginActionHandlers(actionHandlers: any) {
-        for (const handler of actionHandlers) {
-            this.includeActionModule(handler);
-        }
-    }
-
-    private loadPluginDeltaHandlers(deltaHandlers: any) {
-        for (const handler of deltaHandlers) {
-            this.includeDeltaModule(handler);
-        }
-    }
-
-    private loadPluginStreamHandlers(streamHandlers: any) {
-        for (const handler of streamHandlers) {
-            this.includeStreamModule(handler);
-        }
-    }
-
-    includeStreamModule(_module) {
-        this.streamHandlers.push(_module);
-    }
-
-    processStreamEvent(msg) {
-        if (this.streamHandlers.length > 0) {
-            this.streamHandlers.forEach(sth => {
-                sth.handler(msg).catch(reason => {
-                    hLog(`Stream processing failed:`, reason);
-                });
-            });
-        }
-    }
 }

modules/parsers/1.7-parser.ts

@@ -1,27 +1,31 @@
-// noinspection JSUnusedGlobalSymbols
-
 import {BaseParser} from "./base-parser";
 import MainDSWorker from "../../workers/deserializer";
 import {Message} from "amqplib";
 import DSPoolWorker from "../../workers/ds-pool";
 import {TrxMetadata} from "../../interfaces/trx-metadata";
 import {ActionTrace} from "../../interfaces/action-trace";
-import {deserialize, hLog} from "../../helpers/common_functions";
+import {hLog} from "../../helpers/common_functions";
+import {unzip} from "zlib";
+import {omit} from "lodash";
+
+async function unzipAsync(data) {
+    return new Promise((resolve, reject) => {
+        const buf = Buffer.from(data, 'hex');
+        unzip(buf, (err, result) => {
+            if (err) {
+                reject(err);
+            } else {
+                resolve(result);
+            }
+        })
+    });
+}
 
 export default class HyperionParser extends BaseParser {
 
-    flatten = false;
+    flatten = true;
 
-    public async parseAction(
-        worker: DSPoolWorker,
-        ts,
-        action: ActionTrace,
-        trx_data: TrxMetadata,
-        _actDataArray,
-        _processedTraces: ActionTrace[],
-        full_trace,
-        usageIncluded
-    ): Promise<boolean> {
+    public async parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded): Promise<boolean> {
 
         // check filters
         if (this.checkBlacklist(action.act)) return false;
@@ -33,32 +37,35 @@ export default class HyperionParser extends BaseParser {
 
         action["@timestamp"] = ts;
         action.block_num = trx_data.block_num;
-        action.block_id = trx_data.block_id;
         action.producer = trx_data.producer;
         action.trx_id = trx_data.trx_id;
 
-        action.account_ram_deltas = action.account_ram_deltas.filter(value => value.delta !== '0')
-
         if (action.account_ram_deltas.length === 0) {
             delete action.account_ram_deltas;
         }
 
         if (action.except === null) {
+
             if (!action.receipt) {
                 console.log(full_trace.status);
                 console.log(action);
             }
+
             action.receipt = action.receipt[1];
             action.global_sequence = parseInt(action.receipt.global_sequence, 10);
+
             delete action.except;
-            if (action.error_code === null) {
-                delete action.error_code;
-            }
-            // add usage data to the first action on the transaction
+            delete action.error_code;
+            delete action.elapsed;
+            delete action.context_free;
+            delete action.level;
+
             if (!usageIncluded.status) {
                 this.extendFirstAction(worker, action, trx_data, full_trace, usageIncluded);
             }
+
             _processedTraces.push(action);
+
         } else {
             hLog(action);
         }
@@ -68,8 +75,7 @@ export default class HyperionParser extends BaseParser {
 
     public async parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void> {
         for (const message of messages) {
-            let allowProcessing = true;
-            const ds_msg = deserialize('result', message.content, this.txEnc, this.txDec, worker.types);
+            const ds_msg = worker.deserializeNative('result', message.content);
             if (!ds_msg) {
                 if (worker.ch_ready) {
                     worker.ch.nack(message);
@@ -77,58 +83,34 @@ export default class HyperionParser extends BaseParser {
                 }
             }
             const res = ds_msg[1];
-            let block = null;
-            let traces = [];
-            let deltas = [];
-
+            let block, traces = [], deltas = [];
             if (res.block && res.block.length) {
-
                 block = worker.deserializeNative('signed_block', res.block);
-
-                if (block === null) {
-                    hLog('incompatible block');
+                if (!block) {
+                    hLog('null block', res);
                     process.exit(1);
                 }
-
-                // verify for whitelisted contracts (root actions only)
-                if (worker.conf.whitelists.root_only) {
-                    try {
-                        if (worker.conf.whitelists && (worker.conf.whitelists.actions.length > 0 || worker.conf.whitelists.deltas.length > 0)) {
-                            allowProcessing = false;
-                            for (const transaction of block.transactions) {
-                                if (transaction.status === 0 && transaction.trx[1] && transaction.trx[1].packed_trx) {
-                                    const unpacked_trx = worker.api.deserializeTransaction(Buffer.from(transaction.trx[1].packed_trx, 'hex'));
-                                    for (const act of unpacked_trx.actions) {
-                                        if (this.checkWhitelist(act)) {
-                                            allowProcessing = true;
-                                            break;
-                                        }
-                                    }
-                                    if (allowProcessing) break;
-                                }
-                            }
-                        }
-                    } catch (e) {
-                        console.log(e);
-                        allowProcessing = true;
-                    }
+            }
+            if (res['traces'] && res['traces'].length) {
+                try {
+                    traces = worker.deserializeNative(
+                        'transaction_trace[]',
+                        await unzipAsync(res['traces'])
+                    );
+                } catch (e) {
+                    hLog(e);
                 }
             }
-
-            if (allowProcessing && res.traces && res.traces.length) {
-                traces = worker.deserializeNative('transaction_trace[]', res.traces);
-                if (!traces) {
-                    hLog(`[WARNING] transaction_trace[] deserialization failed on block ${res['this_block']['block_num']}`);
+            if (res['deltas'] && res['deltas'].length) {
+                try {
+                    deltas = worker.deserializeNative(
+                        'table_delta[]',
+                        await unzipAsync(res['deltas'])
+                    );
+                } catch (e) {
+                    hLog(e);
                 }
             }
-
-            if (allowProcessing && res.deltas && res.deltas.length) {
-                deltas = deserialize('table_delta[]', res.deltas, this.txEnc, this.txDec, worker.types);
-                if (!deltas) {
-                    hLog(`[WARNING] table_delta[] deserialization failed on block ${res['this_block']['block_num']}`);
-                }
-            }
-
             let result;
             try {
                 result = await worker.processBlock(res, block, traces, deltas);
@@ -137,7 +119,6 @@ export default class HyperionParser extends BaseParser {
                         event: 'consumed_block',
                         block_num: result['block_num'],
                         block_id: result['block_id'],
-                        block_ts: result['block_ts'],
                         trx_ids: result['trx_ids'],
                         lib: res.last_irreversible.block_num,
                         live: process.env.live_mode
@@ -163,9 +144,28 @@ export default class HyperionParser extends BaseParser {
         }
     }
 
-    async flattenInlineActions(action_traces: any[]): Promise<any[]> {
-        hLog(`Calling undefined flatten operation!`);
-        return Promise.resolve(undefined);
+    async flattenInlineActions(action_traces: any[], level: number, trace_counter: any, parent_index: number): Promise<any[]> {
+        const arr = [];
+        const nextLevel = [];
+        for (const action_trace of action_traces) {
+            const trace = action_trace[1];
+            trace['creator_action_ordinal'] = parent_index;
+            trace_counter.trace_index++;
+            trace['level'] = level;
+            trace['action_ordinal'] = trace_counter.trace_index;
+            arr.push(["action_trace_v0", omit(trace, "inline_traces")]);
+            if (trace.inline_traces && trace.inline_traces.length > 0) {
+                nextLevel.push({
+                    traces: trace.inline_traces,
+                    parent: trace_counter.trace_index
+                });
+            }
+        }
+        for (const data of nextLevel) {
+            const appendedArray = await this.flattenInlineActions(data.traces, level + 1, trace_counter, data.parent);
+            arr.push(...appendedArray);
+        }
+        return Promise.resolve(arr);
     }
 
 }

modules/parsers/1.8-parser.ts

@@ -215,7 +215,6 @@ export default class HyperionParser extends BaseParser {
 						event: 'consumed_block',
 						block_num: result['block_num'],
 						block_id: result['block_id'],
-						block_ts: result['block_ts'],
 						trx_ids: result['trx_ids'],
 						lib: res.last_irreversible.block_num,
 						live: process.env.live_mode

modules/parsers/2.1-parser.ts

@@ -8,280 +8,244 @@ import {deserialize, hLog} from "../../helpers/common_functions";
 import {appendFileSync} from "fs";
 
 function timedFunction(enable: boolean, method: () => void) {
-    if (enable) {
-        const ref = process.hrtime.bigint();
-        method();
-        return Number(process.hrtime.bigint() - ref) / 1000;
-    } else {
-        method();
-        return null;
-    }
+	if (enable) {
+		const ref = process.hrtime.bigint();
+		method();
+		return Number(process.hrtime.bigint() - ref) / 1000;
+	} else {
+		method();
+		return null;
+	}
 }
 
 export default class HyperionParser extends BaseParser {
 
-    flatten = false;
+	flatten = false;
 
-    public async parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded): Promise<boolean> {
-        // check filters
-        if (this.checkBlacklist(action.act)) return false;
-        if (this.filters.action_whitelist.size > 0) {
-            if (!this.checkWhitelist(action.act)) return false;
-        }
+	public async parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded): Promise<boolean> {
+		// check filters
+		if (this.checkBlacklist(action.act)) return false;
+		if (this.filters.action_whitelist.size > 0) {
+			if (!this.checkWhitelist(action.act)) return false;
+		}
 
-        await this.deserializeActionData(worker, action, trx_data);
+		await this.deserializeActionData(worker, action, trx_data);
 
-        action["@timestamp"] = ts;
-        action.block_num = trx_data.block_num;
-        action.block_id = trx_data.block_id;
-        action.producer = trx_data.producer;
-        action.trx_id = trx_data.trx_id;
+		action["@timestamp"] = ts;
+		action.block_num = trx_data.block_num;
+		action.block_id = trx_data.block_id;
+		action.producer = trx_data.producer;
+		action.trx_id = trx_data.trx_id;
 
-        if (action.account_ram_deltas.length === 0) {
-            delete action.account_ram_deltas;
-        }
+		if (action.account_ram_deltas.length === 0) {
+			delete action.account_ram_deltas;
+		}
 
-        if (action.except === null) {
-            if (!action.receipt) {
-                console.log(full_trace.status);
-                console.log(action);
-            }
+		if (action.except === null) {
+			if (!action.receipt) {
+				console.log(full_trace.status);
+				console.log(action);
+			}
 
-            action.receipt = action.receipt[1];
-            action.global_sequence = parseInt(action.receipt.global_sequence, 10);
+			action.receipt = action.receipt[1];
+			action.global_sequence = parseInt(action.receipt.global_sequence, 10);
 
-            delete action.except;
-            delete action.error_code;
+			delete action.except;
+			delete action.error_code;
 
-            // add usage data to the first action on the transaction
-            if (!usageIncluded.status) {
-                this.extendFirstAction(worker, action, trx_data, full_trace, usageIncluded);
-            }
+			// add usage data to the first action on the transaction
+			if (!usageIncluded.status) {
+				this.extendFirstAction(worker, action, trx_data, full_trace, usageIncluded);
+			}
 
-            _processedTraces.push(action);
-        } else {
-            hLog(action);
-        }
+			_processedTraces.push(action);
+		} else {
+			hLog(action);
+		}
 
-        return true;
-    }
+		return true;
+	}
 
-    public async parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void> {
-        const dsProfiling = worker.conf.settings.ds_profiling;
-        for (const message of messages) {
+	public async parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void> {
+		const dsProfiling = worker.conf.settings.ds_profiling;
+		for (const message of messages) {
 
-            // profile deserialization
-            const ds_times = {
-                size: message.content.length,
-                eosjs: {
-                    result: undefined,
-                    signed_block: undefined,
-                    transaction_trace: undefined,
-                    table_delta: undefined
-                },
-                abieos: {
-                    result: undefined,
-                    signed_block: undefined,
-                    transaction_trace: undefined,
-                    table_delta: undefined
-                }
-            };
+			// profile deserialization
+			const ds_times = {
+				size: message.content.length,
+				eosjs: {
+					result: undefined,
+					signed_block: undefined,
+					transaction_trace: undefined,
+					table_delta: undefined
+				},
+				abieos: {
+					result: undefined,
+					signed_block: undefined,
+					transaction_trace: undefined,
+					table_delta: undefined
+				}
+			};
 
-            let allowProcessing = true;
-            let ds_msg;
+			let allowProcessing = true;
+			let ds_msg;
 
-            // deserialize result using abieos
-            // ds_times.abieos.result = timedFunction(dsProfiling,() => {
-            //     ds_msg = worker.deserializeNative('result', message.content);
-            // });
+			// deserialize result using abieos
+			// ds_times.abieos.result = timedFunction(dsProfiling,() => {
+			//     ds_msg = worker.deserializeNative('result', message.content);
+			// });
 
-            // deserialize result using eosjs (faster)
-            ds_times.eosjs.result = timedFunction(dsProfiling, () => {
-                ds_msg = deserialize('result', message.content, this.txEnc, this.txDec, worker.types);
-            });
+			// deserialize result using eosjs (faster)
+			ds_times.eosjs.result = timedFunction(dsProfiling, () => {
+				ds_msg = deserialize('result', message.content, this.txEnc, this.txDec, worker.types);
+			});
 
-            if (!ds_msg) {
-                if (worker.ch_ready) {
-                    worker.ch.nack(message);
-                    throw new Error('failed to deserialize datatype=result');
-                }
-            }
+			if (!ds_msg) {
+				if (worker.ch_ready) {
+					worker.ch.nack(message);
+					throw new Error('failed to deserialize datatype=result');
+				}
+			}
 
-            const res = ds_msg[1];
+			const res = ds_msg[1];
+			let block = null;
+			let traces = [];
+			let deltas = [];
 
-            let block = null;
-            if (res.block && res.block.length) {
+			if (res.block && res.block.length) {
 
-                if (typeof res.block === 'object' && res.block.length === 2) {
-                    // already deserialized
-                    block = res.block[1];
-                } else {
-                    try {
-                        // deserialize signed_block using abieos (faster)
-                        ds_times.abieos.signed_block = timedFunction(dsProfiling, () => {
-                            block = worker.deserializeNative('signed_block_variant', res.block)[1];
-                        });
-                    } catch (e) {
-                        hLog('signed_block_variant deserialization failed with abieos!');
-                    }
-                    if (!block) {
-                        try {
-                            // deserialize signed_block using eosjs
-                            ds_times.eosjs.signed_block = timedFunction(dsProfiling, () => {
-                                block = deserialize('signed_block_variant', res.block, this.txEnc, this.txDec, worker.types);
-                            });
-                        } catch (e) {
-                            hLog('signed_block_variant deserialization failed with eosjs!');
-                        }
-                    }
-                }
+				// deserialize signed_block using abieos (faster)
+				ds_times.abieos.signed_block = timedFunction(dsProfiling, () => {
+					block = worker.deserializeNative('signed_block_variant', res.block)[1];
+				});
 
-                if (block === null) {
-                    console.log(res);
-                    process.exit(1);
-                }
+				// // deserialize signed_block using eosjs
+				// ds_times.eosjs.signed_block = timedFunction(dsProfiling, () => {
+				// 	console.log(worker.types);
+				// 	block = deserialize('signed_block_variant', res.block, this.txEnc, this.txDec, worker.types);
+				// 	console.log(block);
+				// });
 
-                // verify for whitelisted contracts (root actions only)
-                if (worker.conf.whitelists.root_only) {
-                    try {
+				if (block === null) {
+					console.log(res);
+					process.exit(1);
+				}
 
-                        // get time reference for profiling
-                        if (worker.conf.settings.ds_profiling) ds_times['packed_trx'] = process.hrtime.bigint();
+				// verify for whitelisted contracts (root actions only)
+				if (worker.conf.whitelists.root_only) {
+					try {
 
-                        if (worker.conf.whitelists &&
-                            (worker.conf.whitelists.actions.length > 0 ||
-                                worker.conf.whitelists.deltas.length > 0)) {
+						// get time reference for profiling
+						if (worker.conf.settings.ds_profiling) ds_times['packed_trx'] = process.hrtime.bigint();
 
-                            allowProcessing = false;
+						if (worker.conf.whitelists &&
+							(worker.conf.whitelists.actions.length > 0 ||
+								worker.conf.whitelists.deltas.length > 0)) {
 
-                            for (const transaction of block.transactions) {
-                                if (transaction.status === 0 && transaction.trx[1] && transaction.trx[1].packed_trx) {
-                                    const unpacked_trx = worker.api.deserializeTransaction(Buffer.from(transaction.trx[1].packed_trx, 'hex'));
-                                    for (const act of unpacked_trx.actions) {
-                                        if (this.checkWhitelist(act)) {
-                                            allowProcessing = true;
-                                            break;
-                                        }
-                                    }
-                                    if (allowProcessing) break;
-                                }
-                            }
-                        }
+							allowProcessing = false;
 
-                        // store time diff
-                        if (worker.conf.settings.ds_profiling) ds_times['packed_trx'] = Number(process.hrtime.bigint() - ds_times['packed_trx']) / 1000;
+							for (const transaction of block.transactions) {
+								if (transaction.status === 0 && transaction.trx[1] && transaction.trx[1].packed_trx) {
+									const unpacked_trx = worker.api.deserializeTransaction(Buffer.from(transaction.trx[1].packed_trx, 'hex'));
+									for (const act of unpacked_trx.actions) {
+										if (this.checkWhitelist(act)) {
+											allowProcessing = true;
+											break;
+										}
+									}
+									if (allowProcessing) break;
+								}
+							}
+						}
 
-                    } catch (e) {
-                        console.log(e);
-                        allowProcessing = true;
-                    }
-                }
-            }
+						// store time diff
+						if (worker.conf.settings.ds_profiling) ds_times['packed_trx'] = Number(process.hrtime.bigint() - ds_times['packed_trx']) / 1000;
 
-            // unpack traces
-            let traces = null;
-            if (allowProcessing && res.traces && res.traces.length) {
+					} catch (e) {
+						console.log(e);
+						allowProcessing = true;
+					}
+				}
+			}
 
-                // deserialize transaction_trace using abieos (faster)
-                try {
-                    ds_times.abieos.transaction_trace = timedFunction(dsProfiling, () => {
-                        traces = worker.deserializeNative('transaction_trace[]', res.traces);
-                    });
-                } catch (e) {
-                    hLog('transaction_trace[] deserialization failed with abieos!');
-                }
+			if (allowProcessing && res.traces && res.traces.length) {
 
-                // deserialize transaction_trace using eosjs
-                if (!traces) {
-                    try {
-                        ds_times.eosjs.transaction_trace = timedFunction(dsProfiling, () => {
-                            traces = deserialize('transaction_trace[]', res.traces, this.txEnc, this.txDec, worker.types);
-                        });
-                    } catch (e) {
-                        hLog('transaction_trace[] deserialization failed with eosjs!');
-                    }
-                }
+				// deserialize transaction_trace using abieos (faster)
+				ds_times.abieos.transaction_trace = timedFunction(dsProfiling, () => {
+					traces = worker.deserializeNative('transaction_trace[]', res.traces);
+				});
 
+				// deserialize transaction_trace using eosjs
+				// ds_times.eosjs.transaction_trace = timedFunction(dsProfiling, () => {
+				//     traces = deserialize('transaction_trace[]', res.traces, this.txEnc, this.txDec, worker.types);
+				// });
 
-                if (!traces) {
-                    hLog(`[WARNING] transaction_trace[] deserialization failed on block ${res['this_block']['block_num']}`);
-                }
-            }
+				if (!traces) {
+					hLog(`[WARNING] transaction_trace[] deserialization failed on block ${res['this_block']['block_num']}`);
+				}
+			}
 
-            // unpack deltas
-            let deltas = null;
-            if (allowProcessing && res.deltas && res.deltas.length) {
+			if (allowProcessing && res.deltas && res.deltas.length) {
 
-                // deserialize table_delta using abieos
-                try {
-                    ds_times.abieos.table_delta = timedFunction(dsProfiling, () => {
-                        deltas = worker.deserializeNative('table_delta[]', res.deltas);
-                    });
-                } catch (e) {
-                    hLog('table_delta[] deserialization failed with abieos!');
-                }
+				// deserialize table_delta using abieos
+				// ds_times.abieos.table_delta = timedFunction(dsProfiling, () => {
+				//     worker.deserializeNative('table_delta[]', res.deltas);
+				// });
 
-                // deserialize table_delta using eosjs
-                if (!deltas) {
-                    try {
-                        ds_times.eosjs.table_delta = timedFunction(dsProfiling, () => {
-                            deltas = deserialize('table_delta[]', res.deltas, this.txEnc, this.txDec, worker.types);
-                        });
-                    } catch (e) {
-                        hLog('table_delta[] deserialization failed with eosjs!');
-                    }
-                }
+				// deserialize table_delta using eosjs (faster)
+				ds_times.eosjs.table_delta = timedFunction(dsProfiling, () => {
+					deltas = deserialize('table_delta[]', res.deltas, this.txEnc, this.txDec, worker.types);
+				});
 
-                if (!deltas) {
-                    hLog(`[WARNING] table_delta[] deserialization failed on block ${res['this_block']['block_num']}`);
-                }
-            }
+				if (!deltas) {
+					hLog(`[WARNING] table_delta[] deserialization failed on block ${res['this_block']['block_num']}`);
+				}
+			}
 
-            if (worker.conf.settings.ds_profiling) {
-                hLog(ds_times);
-                const line = [ds_times.size];
-                line.push(...[ds_times.eosjs.result, ds_times.eosjs.signed_block, ds_times.eosjs.transaction_trace, ds_times.eosjs.table_delta]);
-                line.push(...[ds_times.abieos.result, ds_times.abieos.signed_block, ds_times.abieos.transaction_trace, ds_times.abieos.table_delta]);
-                appendFileSync('ds_profiling.csv', line.join(',') + "\n");
-            }
+			if (worker.conf.settings.ds_profiling) {
+				hLog(ds_times);
+				const line = [ds_times.size];
+				line.push(...[ds_times.eosjs.result, ds_times.eosjs.signed_block, ds_times.eosjs.transaction_trace, ds_times.eosjs.table_delta]);
+				line.push(...[ds_times.abieos.result, ds_times.abieos.signed_block, ds_times.abieos.transaction_trace, ds_times.abieos.table_delta]);
+				appendFileSync('ds_profiling.csv', line.join(',') + "\n");
+			}
 
-            let result;
-            try {
-                result = await worker.processBlock(res, block, traces, deltas);
-                if (result) {
-                    const evPayload = {
-                        event: 'consumed_block',
-                        block_num: result['block_num'],
-                        block_id: result['block_id'],
-                        block_ts: result['block_ts'],
-                        trx_ids: result['trx_ids'],
-                        lib: res.last_irreversible.block_num,
-                        live: process.env.live_mode
-                    };
-                    if (block) {
-                        evPayload["producer"] = block['producer'];
-                        evPayload["schedule_version"] = block['schedule_version'];
-                    }
-                    process.send(evPayload);
-                } else {
-                    hLog(`ERROR: Block data not found for #${res['this_block']['block_num']}`);
-                }
-                if (worker.ch_ready) {
-                    worker.ch.ack(message);
-                }
-            } catch (e) {
-                console.log(e);
-                if (worker.ch_ready) {
-                    worker.ch.nack(message);
-                }
-                process.exit(1);
-            }
-        }
-    }
+			let result;
+			try {
+				result = await worker.processBlock(res, block, traces, deltas);
+				if (result) {
+					const evPayload = {
+						event: 'consumed_block',
+						block_num: result['block_num'],
+						block_id: result['block_id'],
+						trx_ids: result['trx_ids'],
+						lib: res.last_irreversible.block_num,
+						live: process.env.live_mode
+					};
+					if (block) {
+						evPayload["producer"] = block['producer'];
+						evPayload["schedule_version"] = block['schedule_version'];
+					}
+					process.send(evPayload);
+				} else {
+					hLog(`ERROR: Block data not found for #${res['this_block']['block_num']}`);
+				}
+				if (worker.ch_ready) {
+					worker.ch.ack(message);
+				}
+			} catch (e) {
+				console.log(e);
+				if (worker.ch_ready) {
+					worker.ch.nack(message);
+				}
+				process.exit(1);
+			}
+		}
+	}
 
-    async flattenInlineActions(action_traces: any[]): Promise<any[]> {
-        hLog(`Calling undefined flatten operation!`);
-        return Promise.resolve(undefined);
-    }
+	async flattenInlineActions(action_traces: any[]): Promise<any[]> {
+		hLog(`Calling undefined flatten operation!`);
+		return Promise.resolve(undefined);
+	}
 
 }

modules/parsers/base-parser.ts

@@ -10,222 +10,211 @@ import {HyperionActionAct} from "../../interfaces/hyperion-action";
 
 export abstract class BaseParser {
 
-    txDec = new TextDecoder();
-    txEnc = new TextEncoder();
-    configModule: ConfigurationModule;
-    filters: Filters;
-    private readonly chain: string;
-    flatten: boolean = false;
-    private actionReinterpretMap: Map<string, (act: HyperionActionAct) => any>;
+	txDec = new TextDecoder();
+	txEnc = new TextEncoder();
+	configModule: ConfigurationModule;
+	filters: Filters;
+	private readonly chain: string;
+	flatten: boolean = false;
+	private actionReinterpretMap: Map<string, (act: HyperionActionAct) => any>;
 
-    protected constructor(cm: ConfigurationModule) {
-        this.configModule = cm;
-        this.filters = this.configModule.filters;
-        this.chain = this.configModule.config.settings.chain;
-        this.actionReinterpretMap = new Map();
-        this.addCustomHandlers();
-    }
+	protected constructor(cm: ConfigurationModule) {
+		this.configModule = cm;
+		this.filters = this.configModule.filters;
+		this.chain = this.configModule.config.settings.chain;
+		this.actionReinterpretMap = new Map();
+		this.addCustomHandlers();
+	}
 
-    private anyFromCode(act) {
-        return this.chain + '::' + act['account'] + '::*';
-    }
+	private anyFromCode(act) {
+		return this.chain + '::' + act['account'] + '::*';
+	}
 
-    private anyFromName(act) {
-        return this.chain + '::*::' + act['name'];
-    }
+	private anyFromName(act) {
+		return this.chain + '::*::' + act['name'];
+	}
 
-    private codeActionPair(act) {
-        return this.chain + '::' + act['account'] + '::' + act['name'];
-    }
+	private codeActionPair(act) {
+		return this.chain + '::' + act['account'] + '::' + act['name'];
+	}
 
-    protected checkBlacklist(act) {
+	protected checkBlacklist(act) {
 
-        // test action blacklist for chain::code::*
-        if (this.filters.action_blacklist.has(this.anyFromCode(act))) {
-            return true;
-        }
+		// test action blacklist for chain::code::*
+		if (this.filters.action_blacklist.has(this.anyFromCode(act))) {
+			return true;
+		}
 
-        // test action blacklist for chain::*::name
-        if (this.filters.action_blacklist.has(this.anyFromName(act))) {
-            return true;
-        }
+		// test action blacklist for chain::*::name
+		if (this.filters.action_blacklist.has(this.anyFromName(act))) {
+			return true;
+		}
 
-        // test action blacklist for chain::code::name
-        return this.filters.action_blacklist.has(this.codeActionPair(act));
-    }
+		// test action blacklist for chain::code::name
+		return this.filters.action_blacklist.has(this.codeActionPair(act));
+	}
 
-    protected checkWhitelist(act) {
+	protected checkWhitelist(act) {
 
-        // test action whitelist for chain::code::*
-        if (this.filters.action_whitelist.has(this.anyFromCode(act))) {
-            return true;
-        }
+		// test action whitelist for chain::code::*
+		if (this.filters.action_whitelist.has(this.anyFromCode(act))) {
+			return true;
+		}
 
-        // test action whitelist for chain::*::name
-        if (this.filters.action_whitelist.has(this.anyFromName(act))) {
-            return true;
-        }
+		// test action whitelist for chain::*::name
+		if (this.filters.action_whitelist.has(this.anyFromName(act))) {
+			return true;
+		}
 
-        // test action whitelist for chain::code::name
-        return this.filters.action_whitelist.has(this.codeActionPair(act));
-    }
+		// test action whitelist for chain::code::name
+		return this.filters.action_whitelist.has(this.codeActionPair(act));
+	}
 
-    protected extendFirstAction(
-        worker: DSPoolWorker,
-        action: ActionTrace,
-        trx_data: TrxMetadata,
-        full_trace: any,
-        usageIncluded: { status: boolean }
-    ) {
-        action.cpu_usage_us = trx_data.cpu_usage_us;
-        action.net_usage_words = trx_data.net_usage_words;
-        action.signatures = trx_data.signatures;
-        if (full_trace.action_traces.length > 1) {
-            action.inline_count = trx_data.inline_count - 1;
-            action.inline_filtered = trx_data.filtered;
-            if (action.inline_filtered) {
-                action.max_inline = worker.conf.indexer.max_inline;
-            }
-        } else {
-            action.inline_count = 0;
-        }
-        usageIncluded.status = true;
-    }
+	protected extendFirstAction(worker: DSPoolWorker, action: ActionTrace, trx_data: TrxMetadata, full_trace: any, usageIncluded) {
+		action.cpu_usage_us = trx_data.cpu_usage_us;
+		action.net_usage_words = trx_data.net_usage_words;
+		action.signatures = trx_data.signatures;
+		if (full_trace.action_traces.length > 1) {
+			action.inline_count = trx_data.inline_count - 1;
+			action.inline_filtered = trx_data.filtered;
+			if (action.inline_filtered) {
+				action.max_inline = worker.conf.indexer.max_inline;
+			}
+		} else {
+			action.inline_count = 0;
+		}
+		usageIncluded.status = true;
+	}
 
-    protected createSerialBuffer(data: string) {
-        return new SerialBuffer({
-            textDecoder: this.txDec,
-            textEncoder: this.txEnc,
-            array: Buffer.from(data, 'hex')
-        });
-    }
+	protected createSerialBuffer(data: string) {
+		return new SerialBuffer({
+			textDecoder: this.txDec,
+			textEncoder: this.txEnc,
+			array: Buffer.from(data, 'hex')
+		});
+	}
 
-    protected addCustomHandlers() {
-        // simple assets
-        this.actionReinterpretMap.set('*::saecreate', (act) => {
-            const _sb = this.createSerialBuffer(act.data);
-            const result = {owner: "", assetid: null};
-            result.owner = _sb.getName();
-            result.assetid = _sb.getUint64AsNumber()
-            return result;
-        });
+	protected addCustomHandlers() {
+		// simple assets
+		this.actionReinterpretMap.set('*::saecreate', (act) => {
+			const _sb = this.createSerialBuffer(act.data);
+			const result = {owner: "", assetid: null};
+			result.owner = _sb.getName();
+			result.assetid = _sb.getUint64AsNumber()
+			return result;
+		});
 
-        this.actionReinterpretMap.set('*::saetransfer', (act) => {
-            const _sb = this.createSerialBuffer(act.data);
-            const result = {from: "", to: "", assetids: [], memo: ""};
-            result.from = _sb.getName();
-            result.to = _sb.getName();
-            const len = _sb.getVaruint32();
-            for (let i = 0; i < len; i++) {
-                result.assetids.push(_sb.getUint64AsNumber());
-            }
-            result.memo = _sb.getString();
-            return result;
-        });
+		this.actionReinterpretMap.set('*::saetransfer', (act) => {
+			const _sb = this.createSerialBuffer(act.data);
+			const result = {from: "", to: "", assetids: [], memo: ""};
+			result.from = _sb.getName();
+			result.to = _sb.getName();
+			const len = _sb.getVaruint32();
+			for (let i = 0; i < len; i++) {
+				result.assetids.push(_sb.getUint64AsNumber());
+			}
+			result.memo = _sb.getString();
+			return result;
+		});
 
-        this.actionReinterpretMap.set('*::saeclaim', (act) => {
-            const _sb = this.createSerialBuffer(act.data);
-            const result = {who: "", assetids: {}};
-            result.who = _sb.getName();
-            const len = _sb.getVaruint32();
-            for (let i = 0; i < len; i++) {
-                result.assetids[_sb.getUint64AsNumber()] = _sb.getName();
-            }
-            return result;
-        });
+		this.actionReinterpretMap.set('*::saeclaim', (act) => {
+			const _sb = this.createSerialBuffer(act.data);
+			const result = {who: "", assetids: {}};
+			result.who = _sb.getName();
+			const len = _sb.getVaruint32();
+			for (let i = 0; i < len; i++) {
+				result.assetids[_sb.getUint64AsNumber()] = _sb.getName();
+			}
+			return result;
+		});
 
-        this.actionReinterpretMap.set('*::saeburn', (act) => {
-            const _sb = this.createSerialBuffer(act.data);
-            const result = {who: "", assetids: [], memo: ""};
-            result.who = _sb.getName();
-            const len = _sb.getVaruint32();
-            for (let i = 0; i < len; i++) {
-                result.assetids.push(_sb.getUint64AsNumber());
-            }
-            result.memo = _sb.getString();
-            return result;
-        });
-    }
+		this.actionReinterpretMap.set('*::saeburn', (act) => {
+			const _sb = this.createSerialBuffer(act.data);
+			const result = {who: "", assetids: [], memo: ""};
+			result.who = _sb.getName();
+			const len = _sb.getVaruint32();
+			for (let i = 0; i < len; i++) {
+				result.assetids.push(_sb.getUint64AsNumber());
+			}
+			result.memo = _sb.getString();
+			return result;
+		});
+	}
 
-    async reinterpretActionData(act: HyperionActionAct) {
-        if (this.actionReinterpretMap.has(`${act.account}::${act.name}`)) {
-            // code and action
-            return await this.actionReinterpretMap.get(`${act.account}::${act.name}`)(act);
+	async reinterpretActionData(act: HyperionActionAct) {
+		if (this.actionReinterpretMap.has(`${act.account}::${act.name}`)) {
+			// code and action
+			return await this.actionReinterpretMap.get(`${act.account}::${act.name}`)(act);
 
-        } else if (this.actionReinterpretMap.has(`*::${act.name}`)) {
-            // wildcard action
-            return await this.actionReinterpretMap.get(`*::${act.name}`)(act);
+		} else if (this.actionReinterpretMap.has(`*::${act.name}`)) {
+			// wildcard action
+			return await this.actionReinterpretMap.get(`*::${act.name}`)(act);
 
-        } else {
-            return;
-        }
-    }
+		} else {
+			return;
+		}
+	}
 
-    async deserializeActionData(worker: DSPoolWorker, action: ActionTrace, trx_data) {
-        let act = action.act;
-        const original_act = Object.assign({}, act);
-        let ds_act, error_message;
-        try {
-            ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num);
-        } catch (e) {
-            console.log(e);
-            error_message = e.message;
-        }
+	async deserializeActionData(worker: DSPoolWorker, action: ActionTrace, trx_data) {
+		let act = action.act;
+		const original_act = Object.assign({}, act);
+		let ds_act, error_message;
+		try {
+			ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num);
+		} catch (e) {
+			console.log(e);
+			error_message = e.message;
+		}
 
-        // retry failed deserialization for custom ABI maps
-        if (!ds_act) {
-            try {
-                ds_act = await this.reinterpretActionData(act);
-            } catch (e) {
-                hLog(`Failed to reinterpret action: ${act.account}::${act.name}`);
-                hLog(act.data);
-            }
-        }
+		// retry failed deserialization for custom ABI maps
+		if (!ds_act) {
+			try {
+				ds_act = await this.reinterpretActionData(act);
+			} catch (e) {
+				hLog(`Failed to reinterpret action: ${act.account}::${act.name}`);
+				hLog(act.data);
+			}
+		}
 
-        // retry with last abi before the given block_num
-        if (!ds_act) {
-            debugLog('DS Failed ->>', original_act);
-            ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num - 1);
-            debugLog('Retry with previous ABI ->>', ds_act);
-        }
+		// retry with last abi before the given block_num
+		if (!ds_act) {
+			debugLog('DS Failed ->>', original_act);
+			ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num - 1);
+			debugLog('Retry with previous ABI ->>', ds_act);
+		}
 
-        if (ds_act) {
+		if (ds_act) {
+			// save serialized data
+			action.act.data = ds_act;
+			try {
+				worker.common.attachActionExtras(worker, action);
+			} catch (e) {
+				hLog('Failed to call attachActionExtras:', e.message);
+				hLog(action?.act?.data);
+			}
+		} else {
+			action['act'] = original_act;
+			if (typeof action.act.data !== 'string') {
+				action.act.data = Buffer.from(action.act.data).toString('hex');
+			}
+			action['ds_error'] = true;
+			process.send({
+				event: 'ds_error',
+				data: {
+					type: 'action_ds_error',
+					block: trx_data.block_num,
+					account: act.account,
+					action: act.name,
+					gs: parseInt(action.receipt[1].global_sequence, 10),
+					message: error_message
+				}
+			});
+		}
+	}
 
-            if (ds_act.account && ds_act.name && ds_act.authorization) {
-                action.act.data = ds_act.data;
-            }
+	abstract parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded: { status: boolean }): Promise<boolean>
 
-            // save serialized data
-            action.act.data = ds_act;
-            try {
-                worker.common.attachActionExtras(worker, action);
-            } catch (e) {
-                hLog('Failed to call attachActionExtras:', e.message);
-                hLog(action?.act?.account, action?.act?.name, action?.act?.data);
-            }
-        } else {
-            action['act'] = original_act;
-            if (typeof action.act.data !== 'string') {
-                action.act.data = Buffer.from(action.act.data).toString('hex');
-            }
-            action['ds_error'] = true;
-            process.send({
-                event: 'ds_error',
-                data: {
-                    type: 'action_ds_error',
-                    block: trx_data.block_num,
-                    account: act.account,
-                    action: act.name,
-                    gs: parseInt(action.receipt[1].global_sequence, 10),
-                    message: error_message
-                }
-            });
-        }
-    }
+	abstract parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void>
 
-    abstract parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded: { status: boolean }): Promise<boolean>
-
-    abstract parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void>
-
-    abstract flattenInlineActions(action_traces: any[], level?: number, trace_counter?: any, parent_index?: number): Promise<any[]>
+	abstract flattenInlineActions(action_traces: any[], level?: number, trace_counter?: any, parent_index?: number): Promise<any[]>
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperion-history",
-  "version": "3.3.9-8",
+  "version": "3.3.4-rc6",
   "description": "Scalable Full History API Solution for EOSIO based blockchains",
   "main": "launcher.js",
   "scripts": {
@@ -9,12 +9,12 @@
     "start:indexer": "pm2 start --only Indexer --update-env",
     "tsc": "tsc",
     "build": "tsc",
-    "build:watch": "tsc --watch",
     "postinstall": "npm run build && npm run fix-permissions",
-    "fix-permissions": "node scripts/fix-permissions.js"
+    "fix-permissions": "node scripts/fix-permissions.js",
+    "plugin-manager": "node plugins/hyperion-plugin-manager.js"
   },
   "engines": {
-    "node": ">=18"
+    "node": "^16"
   },
   "author": {
     "name": "EOS Rio",
@@ -26,55 +26,52 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@elastic/elasticsearch": "^7.17.0",
-    "@eosrio/node-abieos": "2.1.1",
-    "@fastify/autoload": "4.0.1",
-    "@fastify/cors": "7.0.0",
-    "@fastify/formbody": "^6.0.1",
-    "@fastify/rate-limit": "^6.0.1",
-    "@fastify/redis": "^5.0.0",
-    "@fastify/swagger": "6.1.0",
+    "@elastic/elasticsearch": "^7.14.0",
+    "@eosrio/node-abieos": "^2.1.1",
     "@pm2/io": "^5.0.0",
-    "amqplib": "^0.10.3",
-    "async": "^3.2.4",
-    "base-x": "^4.0.0",
-    "cli-progress": "^3.12.0",
-    "commander": "^8.3.0",
-    "cross-fetch": "^3.1.8",
+    "amqplib": "^0.8.0",
+    "async": "^3.2.1",
+    "base-x": "^3.0.8",
+    "cross-fetch": "^3.1.4",
     "eosjs": "^22.1.0",
-    "fast-json-stringify": "2.7.13",
-    "fastify": "3.29.4",
+    "fast-json-stringify": "^2.7.8",
+    "fastify": "^3.20.2",
+    "fastify-autoload": "^3.8.0",
+    "fastify-compress": "^3.6.0",
+    "fastify-cors": "^6.0.2",
     "fastify-elasticsearch": "^2.0.0",
-    "fastify-plugin": "^3.0.1",
+    "fastify-formbody": "^5.1.0",
+    "fastify-plugin": "^3.0.0",
+    "fastify-rate-limit": "^5.6.0",
+    "fastify-redis": "^4.3.1",
+    "fastify-static": "^4.2.3",
+    "fastify-swagger": "^4.8.4",
     "flatstr": "^1.0.12",
-    "global-agent": "^3.0.0",
-    "got": "11.8.5",
-    "ioredis": "^4.28.5",
+    "got": "11.8.2",
+    "ioredis": "^4.27.7",
     "lodash": "^4.17.21",
-    "moment": "^2.29.4",
-    "nodemailer": "^6.9.0",
-    "pino-pretty": "^10.2.0",
-    "portfinder": "^1.0.32",
-    "socket.io": "4.7.2",
-    "socket.io-client": "4.7.2",
+    "moment": "^2.29.1",
+    "nodemailer": "^6.6.3",
+    "portfinder": "^1.0.28",
+    "socket.io": "^4.1.3",
+    "socket.io-client": "^4.1.3",
     "socket.io-redis": "^6.1.1",
-    "telegraf": "^4.12.3-canary.1",
-    "typescript": "^4.5.2",
-    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.31.0",
-    "ws": "^8.14.2"
+    "telegraf": "^4.4.1",
+    "typescript": "^4.3.5",
+    "ws": "^8.1.0",
+    "yargs": "^17.1.1"
   },
   "devDependencies": {
-    "@types/amqplib": "^0.10.2",
-    "@types/async": "^3.2.18",
-    "@types/global-agent": "^2.1.1",
-    "@types/ioredis": "^4.28.10",
-    "@types/lodash": "^4.14.197",
-    "@types/node": "^20.5.7",
-    "@types/nodemailer": "^6.4.9",
-    "@types/ws": "^8.5.5"
+    "@types/amqplib": "^0.8.1",
+    "@types/async": "^3.2.7",
+    "@types/ioredis": "^4.26.7",
+    "@types/lodash": "^4.14.172",
+    "@types/node": "^16.6.1",
+    "@types/nodemailer": "^6.4.4",
+    "@types/ws": "^7.4.7"
   },
   "optionalDependencies": {
-    "bufferutil": "^4.0.7",
-    "utf-8-validate": "^6.0.3"
+    "bufferutil": "^4.0.3",
+    "utf-8-validate": "^5.0.5"
   }
 }

plugins/hyperion-plugin-manager.js

@@ -72,6 +72,7 @@ async function verifyInstallation(data) {
       process.exit(0);
     }
   }
+
 }
 
 function startInstallation(data) {

plugins/hyperion-plugin.js

@@ -3,10 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.HyperionPlugin = void 0;
 class HyperionPlugin {
     constructor(config) {
-        this.internalPluginName = '';
         this.actionHandlers = [];
         this.deltaHandlers = [];
-        this.streamHandlers = [];
         this.dynamicContracts = [];
         this.hasApiRoutes = false;
         this.chainName = '';
@@ -14,13 +12,6 @@ class HyperionPlugin {
             this.baseConfig = config;
         }
     }
-    // abstract processActionData(input: any): Promise<any>;
-    initOnce() {
-        // called only once
-    }
-    initHandlerMap() {
-        return {};
-    }
 }
 exports.HyperionPlugin = HyperionPlugin;
 //# sourceMappingURL=hyperion-plugin.js.map

plugins/hyperion-plugin.js.map

@@ -1 +1 @@
-{"version":3,"file":"hyperion-plugin.js","sourceRoot":"","sources":["hyperion-plugin.ts"],"names":[],"mappings":";;;AAgCA,MAAsB,cAAc;IAYhC,YAAsB,MAAY;QAXlC,uBAAkB,GAAW,EAAE,CAAC;QAGhC,mBAAc,GAA4B,EAAE,CAAC;QAC7C,kBAAa,GAA2B,EAAE,CAAC;QAC3C,mBAAc,GAA4B,EAAE,CAAC;QAC7C,qBAAgB,GAAa,EAAE,CAAC;QAChC,iBAAY,GAAY,KAAK,CAAC;QAE9B,cAAS,GAAW,EAAE,CAAC;QAGnB,IAAI,MAAM,EAAE;YACR,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;SAC5B;IACL,CAAC;IAID,wDAAwD;IACxD,QAAQ;QACJ,mBAAmB;IACvB,CAAC;IAED,cAAc;QACV,OAAO,EAAE,CAAC;IACd,CAAC;CACJ;AA5BD,wCA4BC"}
+{"version":3,"file":"hyperion-plugin.js","sourceRoot":"","sources":["hyperion-plugin.ts"],"names":[],"mappings":";;;AAmBA,MAAsB,cAAc;IAQnC,YAAsB,MAAY;QAPlC,mBAAc,GAA4B,EAAE,CAAC;QAC7C,kBAAa,GAA2B,EAAE,CAAC;QAC3C,qBAAgB,GAAa,EAAE,CAAC;QAChC,iBAAY,GAAY,KAAK,CAAC;QAE9B,cAAS,GAAW,EAAE,CAAC;QAGtB,IAAI,MAAM,EAAE;YACX,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;SACzB;IACF,CAAC;CAKD;AAjBD,wCAiBC"}

plugins/hyperion-plugin.ts

@@ -4,58 +4,34 @@ import {HyperionDelta} from "../interfaces/hyperion-delta";
 
 
 interface HyperionActionHandler {
-    action: string;
-    contract: string;
-    mappings?: any;
-    handler: (action: HyperionAction) => Promise<void>;
+	action: string;
+	contract: string;
+	mappings?: any;
+	handler: (action: HyperionAction) => Promise<void>;
 }
 
 interface HyperionDeltaHandler {
-    table: string;
-    contract: string;
-    mappings?: any;
-    handler: (delta: HyperionDelta) => Promise<void>;
-}
-
-interface HyperionStreamEvent {
-
-}
-
-export interface HyperionStreamHandler {
-    event: string;
-    code?: string;
-    account?: string;
-    name?: string;
-    table?: string;
-    handler: (streamEvent: any) => Promise<void>;
+	table: string;
+	contract: string;
+	mappings?: any;
+	handler: (delta: HyperionDelta) => Promise<void>;
 }
 
 export abstract class HyperionPlugin {
-    internalPluginName: string = '';
-    indexerPlugin: boolean;
-    apiPlugin: boolean;
-    actionHandlers: HyperionActionHandler[] = [];
-    deltaHandlers: HyperionDeltaHandler[] = [];
-    streamHandlers: HyperionStreamHandler[] = [];
-    dynamicContracts: string[] = [];
-    hasApiRoutes: boolean = false;
-    baseConfig: any;
-    chainName: string = '';
+	actionHandlers: HyperionActionHandler[] = [];
+	deltaHandlers: HyperionDeltaHandler[] = [];
+	dynamicContracts: string[] = [];
+	hasApiRoutes: boolean = false;
+	baseConfig: any;
+	chainName: string = '';
 
-    protected constructor(config?: any) {
-        if (config) {
-            this.baseConfig = config;
-        }
-    }
+	protected constructor(config?: any) {
+		if (config) {
+			this.baseConfig = config;
+		}
+	}
 
-    abstract addRoutes(server: FastifyInstance): void;
+	abstract addRoutes(server: FastifyInstance): void;
 
-    // abstract processActionData(input: any): Promise<any>;
-    initOnce() {
-        // called only once
-    }
-
-    initHandlerMap(): any {
-        return {};
-    }
+	// abstract processActionData(input: any): Promise<any>;
 }

scripts/fix-permissions.js

@@ -5,5 +5,5 @@ if (os.type() === 'Windows_NT') {
   console.log('Skipping permission update on windows');
 } else {
   console.log('Updating permissions...');
-  execSync('chmod u+x run.sh stop.sh hpm hyp-config');
+  execSync('chmod u+x run.sh stop.sh');
 }

scripts/fix_missing_blocks/README.MD

@@ -1,36 +0,0 @@
-# Hyperion History API - Script
-
-## Fix Missing Blocks
-
-- Python script that connects to elasticsearch and identifies any missing blocks within indexes, and performs a re-indexing of those ranges to include any missing blocks.
-- Once the script completes, it will automatically re-start the indexer on live mode with re-write disabled.
-
-### Install
-
-```
-pip3 install sh
-pip3 install elasticsearch==7.15.2
-```
-
-### Update the following 3 config lines in the python code.
-
-```
-hyperionfolder="/home/charles/hyperion-history-api"
-indexer_log_file = "/home/charles/.pm2/logs/wax-indexer-out.log"
-# Please set the version of Hyperion to either 3.1 or 3.3 to take into account changes to no blocks processed text in log file. 
-hyperion_version = '3.3'  #'3.1' or '3.3'
-```
-
-### RUN
-
-```
-python3 fix-missing-blocks.py http://elasticusername:elasticpasswordD@127.0.0.1
-```
-
-
-### TODO:
-- Add progress 
-
-
-
-

scripts/hpm.js

@@ -1,409 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const commander_1 = require("commander");
-const path_1 = __importDefault(require("path"));
-const promises_1 = require("fs/promises");
-const fs_1 = __importStar(require("fs"));
-const child_process_1 = require("child_process");
-const crypto_1 = __importDefault(require("crypto"));
-const package_json = JSON.parse((0, fs_1.readFileSync)('./package.json').toString());
-const debug = process.env.HPM_DEBUG;
-const program = new commander_1.Command();
-const pluginDirAbsolutePath = path_1.default.join(path_1.default.resolve(), 'plugins', 'repos');
-const pluginStatePath = path_1.default.join(path_1.default.resolve(), 'plugins', '.state.json');
-let stateJson;
-const ignoredDirsForHash = ['.git', 'node_modules'];
-program.version(package_json.version);
-function checkUrl(value) {
-    if (value.startsWith('https://') || value.startsWith('git@')) {
-        return value;
-    }
-    else {
-        throw new commander_1.InvalidArgumentError('Invalid url.');
-    }
-}
-function throwAndQuit(message) {
-    console.error('⚠ >>', message);
-    process.exit(1);
-}
-function getCurrentBranch(dir) {
-    const gitCmd = `(cd ${dir} && git rev-parse --abbrev-ref HEAD)`;
-    if (debug) {
-        console.log(` >> ${gitCmd}`);
-    }
-    return (0, child_process_1.execSync)(gitCmd).toString().trim();
-}
-function checkoutBranch(dir, branchName) {
-    const gitCmd = `(cd ${dir} && git checkout ${branchName})`;
-    if (debug) {
-        console.log(` >> ${gitCmd}`);
-    }
-    return (0, child_process_1.execSync)(gitCmd).toString();
-}
-function gitPull(dir) {
-    const gitCmd = `(cd ${dir} && git pull)`;
-    if (debug) {
-        console.log(` >> ${gitCmd}`);
-    }
-    return (0, child_process_1.execSync)(gitCmd).toString();
-}
-function gitReset(dir) {
-    const gitCmd = `(cd ${dir} && git reset --hard)`;
-    if (debug) {
-        console.log(` >> ${gitCmd}`);
-    }
-    return (0, child_process_1.execSync)(gitCmd).toString();
-}
-async function buildPlugin(name, flags) {
-    const pluginDir = path_1.default.join(pluginDirAbsolutePath, name);
-    if (!(0, fs_1.existsSync)(pluginDir)) {
-        throwAndQuit(`plugin ${name} not found!`);
-    }
-    const currentBranch = getCurrentBranch(pluginDir);
-    console.log(`-------- ⚒ ${name} @ ${currentBranch} -----------`);
-    if (!flags.skipInstall) {
-        // run npm install
-        await new Promise(resolve => {
-            const npmInstall = (0, child_process_1.spawn)('npm', ['install'], {
-                cwd: pluginDir,
-                stdio: 'inherit',
-            });
-            npmInstall.on('close', (code) => {
-                if (code !== 0) {
-                    console.log(`process exited with code ${code}`);
-                    process.exit(code);
-                }
-                resolve(true);
-            });
-        });
-    }
-    // typescript build
-    await new Promise(resolve => {
-        const npmInstall = (0, child_process_1.spawn)('npm', ['run', 'build'], {
-            cwd: pluginDir,
-            stdio: 'inherit',
-        });
-        npmInstall.on('close', (code) => {
-            if (code !== 0) {
-                console.log(`process exited with code ${code}`);
-                process.exit(code);
-            }
-            resolve(true);
-        });
-    });
-    console.log('-------------------------------');
-    // save build date
-    if (stateJson) {
-        if (!stateJson.plugins[name]) {
-            stateJson.plugins[name] = {};
-        }
-        stateJson.plugins[name].last_build_date = new Date().toISOString();
-        const hash = await hashItem(pluginDir);
-        console.log(`SHA1 Hash: ${hash}`);
-        stateJson.plugins[name].hash = hash;
-        stateJson.plugins[name].branch = currentBranch;
-        saveState();
-    }
-}
-async function verifyInstalledPlugin(pluginName, options) {
-    const dir = path_1.default.join(pluginDirAbsolutePath, pluginName);
-    const dirExists = (0, fs_1.existsSync)(dir);
-    if (dirExists) {
-        if (options.branch && options.branch !== '') {
-            const currentBranch = getCurrentBranch(dir);
-            if (currentBranch === options.branch) {
-                console.log('✅ Plugin already installed, branch check OK');
-                process.exit(0);
-            }
-            else {
-                // checkout new branch
-                const checkoutResults = checkoutBranch(dir, options.branch);
-                console.log(checkoutResults);
-                console.log('done!');
-                process.exit(0);
-            }
-        }
-        throwAndQuit('plugin already installed');
-    }
-}
-async function hashItem(item) {
-    if (fs_1.default.lstatSync(item).isDirectory()) {
-        const dir = fs_1.default.readdirSync(item);
-        const sha = crypto_1.default.createHash('sha1');
-        for (const file of dir) {
-            if (ignoredDirsForHash.includes(file) && fs_1.default.lstatSync(path_1.default.join(item, file)).isDirectory()) {
-                sha.update('');
-            }
-            else {
-                const hash = await hashItem(path_1.default.join(item, file));
-                sha.update(hash.toString());
-            }
-        }
-        return sha.digest('hex');
-    }
-    else {
-        const sha = crypto_1.default.createHash('sha1');
-        const reader = fs_1.default.createReadStream(item);
-        return new Promise(resolve => {
-            reader.on('data', chunk => {
-                sha.update(chunk);
-            });
-            reader.on('close', () => {
-                resolve(sha.digest('hex'));
-            });
-        });
-    }
-}
-async function verifyInstallation(installPath, pluginName, opts) {
-    console.log('Verifying integrity...');
-    const hash = await hashItem(installPath);
-    console.log(`SHA1 Hash: ${hash}`);
-    if (stateJson && stateJson.plugins) {
-        stateJson.plugins[pluginName].hash = hash;
-        saveState();
-    }
-    if (!opts.skipValidation) {
-        // if (hash !== data['integrityHash']) {
-        //     console.log('WARNING! Integrity hash dos not match!');
-        //     console.log(`Expected:\t${data['integrityHash']}`);
-        //     console.log(`Current:\t${hash}`);
-        //     process.exit(0);
-        // }
-        const installedFiles = fs_1.default.readdirSync(installPath);
-        if (!installedFiles.includes('package.json')) {
-            console.log('package.json not found!');
-            process.exit(0);
-        }
-    }
-}
-async function clonePluginRepo(pluginName, options) {
-    const dir = path_1.default.join(pluginDirAbsolutePath, pluginName);
-    await new Promise((resolve, reject) => {
-        const gitClone = (0, child_process_1.spawn)('git', ['clone', options.repository, dir]);
-        gitClone.stdout.on('data', chunk => {
-            console.log(`->> ${chunk}`);
-        });
-        gitClone.stderr.on('data', chunk => {
-            console.log(`->> ${chunk}`);
-        });
-        gitClone.on('close', (code) => {
-            if (code !== 0) {
-                console.log(`process exited with code ${code}`);
-            }
-            verifyInstallation(dir, pluginName, options)
-                .then(() => {
-                resolve();
-            })
-                .catch((err) => {
-                throwAndQuit(err.message);
-                reject();
-            });
-        });
-    });
-}
-function init() {
-    const check = (0, fs_1.existsSync)(pluginStatePath);
-    if (check) {
-        try {
-            stateJson = JSON.parse((0, fs_1.readFileSync)(pluginStatePath).toString());
-        }
-        catch (e) {
-            console.log(e);
-        }
-    }
-    else {
-        console.log('Creating new state file...');
-        stateJson = {
-            update_ts: Date.now(),
-            hyperion_version: package_json.version,
-            plugins: {}
-        };
-        (0, fs_1.writeFileSync)(pluginStatePath, JSON.stringify(stateJson, null, 2));
-    }
-}
-async function installPlugin(plugin, options) {
-    console.log(`Plugin Name: ${plugin}`);
-    console.log(`Repository Url: ${options.repository}`);
-    if (options.branch) {
-        console.log(`Branch Name: ${options.branch}`);
-    }
-    // check installation
-    await verifyInstalledPlugin(plugin, options);
-    if (stateJson && stateJson.plugins) {
-        stateJson.plugins[plugin] = {};
-    }
-    else {
-        throwAndQuit('fatal error on state');
-    }
-    // clone repository
-    await clonePluginRepo(plugin, options);
-    // checkout branch
-    if (options.branch) {
-        const dir = path_1.default.join(pluginDirAbsolutePath, plugin);
-        const results = checkoutBranch(dir, options.branch);
-        stateJson.plugins[plugin].branch = options.branch;
-        console.log(results);
-    }
-    // build plugin
-    await buildPlugin(plugin, {
-        skipInstall: false
-    });
-}
-async function uninstall(plugin) {
-    const pluginDir = path_1.default.join(pluginDirAbsolutePath, plugin);
-    try {
-        if (!(0, fs_1.existsSync)(pluginDir)) {
-            throwAndQuit(`plugin ${plugin} not found!`);
-        }
-        console.log(`Removing ${pluginDir}...`);
-        await (0, promises_1.rm)(pluginDir, {
-            recursive: true
-        });
-        if (stateJson && stateJson.plugins) {
-            delete stateJson.plugins[plugin];
-            saveState();
-        }
-        console.log(`${plugin} removed!`);
-    }
-    catch (e) {
-        throwAndQuit(e.message);
-    }
-}
-async function listPlugins() {
-    const dirs = await (0, promises_1.readdir)(pluginDirAbsolutePath);
-    const results = [];
-    for (const dir of dirs) {
-        try {
-            const packageJsonPath = path_1.default.join(pluginDirAbsolutePath, dir, 'package.json');
-            const package_json = JSON.parse((0, fs_1.readFileSync)(packageJsonPath).toString());
-            const { name, version, description } = package_json;
-            // console.table([dir, name, version, description].join('\t'));
-            let pluginState = false;
-            let branch = '';
-            if (stateJson.plugins && stateJson.plugins[dir]) {
-                if (stateJson.plugins[dir].enabled) {
-                    pluginState = true;
-                }
-                branch = stateJson.plugins[dir].branch;
-            }
-            results.push({
-                alias: dir,
-                enabled: pluginState,
-                plugin_name: name,
-                version,
-                branch,
-                description
-            });
-        }
-        catch (e) {
-            console.log(e.message);
-        }
-    }
-    results.sort((a, b) => b.ative - a.ative);
-    console.table(results);
-}
-function saveState() {
-    stateJson.update_ts = Date.now();
-    (0, fs_1.writeFileSync)(pluginStatePath, JSON.stringify(stateJson, null, 2));
-}
-async function setPluginState(pluginName, newState) {
-    if (stateJson) {
-        if (stateJson.plugins[pluginName]) {
-            stateJson.plugins[pluginName].enabled = newState;
-        }
-        else {
-            stateJson.plugins[pluginName] = {
-                enabled: newState
-            };
-        }
-    }
-    saveState();
-    console.log(`[${pluginName}] plugin ${newState ? 'enabled' : 'disabled'} successfully!`);
-}
-function enablePlugin(pluginName) {
-    return setPluginState(pluginName, true);
-}
-function disablePlugin(pluginName) {
-    return setPluginState(pluginName, false);
-}
-function printState() {
-    console.log(JSON.stringify(stateJson, null, 2));
-}
-async function buildAllPlugins(flags) {
-    const names = (0, fs_1.readdirSync)(pluginDirAbsolutePath);
-    for (const name of names) {
-        await buildPlugin(name, flags);
-    }
-}
-async function updatePlugin(name) {
-    const pluginDir = path_1.default.join(pluginDirAbsolutePath, name);
-    const resetStatus = gitReset(pluginDir);
-    console.log(resetStatus);
-    const pullStatus = gitPull(pluginDir);
-    console.log(pullStatus);
-    await buildPlugin(name, {});
-}
-(() => {
-    init();
-    program.description('Command-line utility to manage Hyperion plugins\nMade with ♥  by EOS Rio');
-    program.command('install <plugin>').alias('i')
-        .description('install new plugin')
-        .option('-r, --repository <url>', 'custom repository url', checkUrl)
-        .option('-b, --branch <branch>', 'checkout specific branch')
-        .option('-s, --skip-validation', 'skip plugin integrity validation')
-        .action(installPlugin);
-    program.command('enable <plugin>')
-        .description('enable plugin')
-        .action(enablePlugin);
-    program.command('disable <plugin>')
-        .description('disable plugin')
-        .action(disablePlugin);
-    program.command('state')
-        .description('print current state file')
-        .action(printState);
-    program.command('list').alias('ls')
-        .description('list installed plugins')
-        .action(listPlugins);
-    program.command('build <plugin>').alias('b')
-        .description('build a single plugin')
-        .option('-s, --skip-install', 'skip "npm install" step')
-        .action(buildPlugin);
-    program.command('update <plugin>')
-        .description('update a single plugin')
-        .action(updatePlugin);
-    program.command('build-all')
-        .description('build all plugins')
-        .option('-s, --skip-install', 'skip "npm install" step')
-        .action(buildAllPlugins);
-    program.command('uninstall <plugin>').alias('rm')
-        .description('uninstall plugin')
-        .action(uninstall);
-    program.parse();
-})();
-//# sourceMappingURL=hpm.js.map

scripts/hpm.ts

@@ -1,432 +0,0 @@
-import {Command, InvalidArgumentError} from 'commander';
-import path from 'path'
-import {readdir, rm} from "fs/promises";
-import fs, {existsSync, readdirSync, readFileSync, writeFileSync} from "fs";
-import {execSync, spawn} from "child_process";
-import crypto from "crypto";
-
-const package_json = JSON.parse(readFileSync('./package.json').toString());
-const debug = process.env.HPM_DEBUG;
-const program = new Command();
-const pluginDirAbsolutePath = path.join(path.resolve(), 'plugins', 'repos');
-const pluginStatePath = path.join(path.resolve(), 'plugins', '.state.json');
-
-let stateJson;
-
-const ignoredDirsForHash = ['.git', 'node_modules'];
-
-program.version(package_json.version);
-
-function checkUrl(value) {
-    if (value.startsWith('https://') || value.startsWith('git@')) {
-        return value;
-    } else {
-        throw new InvalidArgumentError('Invalid url.');
-    }
-}
-
-function throwAndQuit(message: string) {
-    console.error('⚠ >>', message);
-    process.exit(1);
-}
-
-function getCurrentBranch(dir): string {
-    const gitCmd = `(cd ${dir} && git rev-parse --abbrev-ref HEAD)`;
-    if (debug) {
-        console.log(` >> ${gitCmd}`);
-    }
-    return execSync(gitCmd).toString().trim();
-}
-
-function checkoutBranch(dir, branchName): string {
-    const gitCmd = `(cd ${dir} && git checkout ${branchName})`;
-    if (debug) {
-        console.log(` >> ${gitCmd}`);
-    }
-    return execSync(gitCmd).toString();
-}
-
-function gitPull(dir): string {
-    const gitCmd = `(cd ${dir} && git pull)`;
-    if (debug) {
-        console.log(` >> ${gitCmd}`);
-    }
-    return execSync(gitCmd).toString();
-}
-
-function gitReset(dir): string {
-    const gitCmd = `(cd ${dir} && git reset --hard)`;
-    if (debug) {
-        console.log(` >> ${gitCmd}`);
-    }
-    return execSync(gitCmd).toString();
-}
-
-async function buildPlugin(name, flags) {
-
-    const pluginDir = path.join(pluginDirAbsolutePath, name);
-
-    if (!existsSync(pluginDir)) {
-        throwAndQuit(`plugin ${name} not found!`);
-    }
-
-    const currentBranch = getCurrentBranch(pluginDir);
-
-    console.log(`-------- ⚒ ${name} @ ${currentBranch} -----------`);
-
-    if (!flags.skipInstall) {
-        // run npm install
-        await new Promise(resolve => {
-            const npmInstall = spawn('npm', ['install'], {
-                cwd: pluginDir,
-                stdio: 'inherit',
-            });
-            npmInstall.on('close', (code) => {
-                if (code !== 0) {
-                    console.log(`process exited with code ${code}`);
-                    process.exit(code);
-                }
-                resolve(true);
-            });
-        });
-    }
-
-    // typescript build
-    await new Promise(resolve => {
-        const npmInstall = spawn('npm', ['run', 'build'], {
-            cwd: pluginDir,
-            stdio: 'inherit',
-        });
-        npmInstall.on('close', (code) => {
-            if (code !== 0) {
-                console.log(`process exited with code ${code}`);
-                process.exit(code);
-            }
-            resolve(true);
-        });
-    });
-
-    console.log('-------------------------------');
-
-    // save build date
-    if (stateJson) {
-
-        if (!stateJson.plugins[name]) {
-            stateJson.plugins[name] = {};
-        }
-
-        stateJson.plugins[name].last_build_date = new Date().toISOString();
-        const hash = await hashItem(pluginDir);
-        console.log(`SHA1 Hash: ${hash}`);
-        stateJson.plugins[name].hash = hash;
-        stateJson.plugins[name].branch = currentBranch;
-        saveState();
-    }
-}
-
-async function verifyInstalledPlugin(pluginName: string, options: any): Promise<void> {
-    const dir = path.join(pluginDirAbsolutePath, pluginName);
-    const dirExists = existsSync(dir);
-    if (dirExists) {
-        if (options.branch && options.branch !== '') {
-            const currentBranch = getCurrentBranch(dir);
-            if (currentBranch === options.branch) {
-                console.log('✅ Plugin already installed, branch check OK');
-                process.exit(0);
-            } else {
-                // checkout new branch
-                const checkoutResults = checkoutBranch(dir, options.branch);
-                console.log(checkoutResults);
-                console.log('done!');
-                process.exit(0);
-            }
-        }
-        throwAndQuit('plugin already installed');
-    }
-}
-
-async function hashItem(item) {
-    if (fs.lstatSync(item).isDirectory()) {
-        const dir = fs.readdirSync(item);
-        const sha = crypto.createHash('sha1');
-        for (const file of dir) {
-            if (ignoredDirsForHash.includes(file) && fs.lstatSync(path.join(item, file)).isDirectory()) {
-                sha.update('');
-            } else {
-                const hash = await hashItem(path.join(item, file));
-                sha.update(hash.toString());
-            }
-        }
-        return sha.digest('hex');
-    } else {
-        const sha = crypto.createHash('sha1');
-        const reader = fs.createReadStream(item);
-        return new Promise(resolve => {
-            reader.on('data', chunk => {
-                sha.update(chunk);
-            });
-            reader.on('close', () => {
-                resolve(sha.digest('hex'));
-            });
-        });
-    }
-}
-
-async function verifyInstallation(installPath, pluginName, opts) {
-    console.log('Verifying integrity...');
-    const hash = await hashItem(installPath);
-    console.log(`SHA1 Hash: ${hash}`);
-
-    if (stateJson && stateJson.plugins) {
-        stateJson.plugins[pluginName].hash = hash;
-        saveState();
-    }
-
-    if (!opts.skipValidation) {
-
-        // if (hash !== data['integrityHash']) {
-        //     console.log('WARNING! Integrity hash dos not match!');
-        //     console.log(`Expected:\t${data['integrityHash']}`);
-        //     console.log(`Current:\t${hash}`);
-        //     process.exit(0);
-        // }
-
-        const installedFiles = fs.readdirSync(installPath);
-        if (!installedFiles.includes('package.json')) {
-            console.log('package.json not found!');
-            process.exit(0);
-        }
-    }
-}
-
-async function clonePluginRepo(pluginName: string, options: any): Promise<void> {
-    const dir = path.join(pluginDirAbsolutePath, pluginName);
-    await new Promise<void>((resolve, reject) => {
-        const gitClone = spawn('git', ['clone', options.repository, dir]);
-        gitClone.stdout.on('data', chunk => {
-            console.log(`->> ${chunk}`);
-        });
-        gitClone.stderr.on('data', chunk => {
-            console.log(`->> ${chunk}`);
-        });
-        gitClone.on('close', (code) => {
-            if (code !== 0) {
-                console.log(`process exited with code ${code}`);
-            }
-            verifyInstallation(dir, pluginName, options)
-                .then(() => {
-                    resolve();
-                })
-                .catch((err) => {
-                    throwAndQuit(err.message);
-                    reject();
-                });
-        });
-    });
-}
-
-function init() {
-    const check = existsSync(pluginStatePath);
-    if (check) {
-        try {
-            stateJson = JSON.parse(readFileSync(pluginStatePath).toString());
-        } catch (e) {
-            console.log(e);
-        }
-    } else {
-        console.log('Creating new state file...');
-        stateJson = {
-            update_ts: Date.now(),
-            hyperion_version: package_json.version,
-            plugins: {}
-        };
-        writeFileSync(pluginStatePath, JSON.stringify(stateJson, null, 2));
-    }
-}
-
-async function installPlugin(plugin: string, options: any) {
-    console.log(`Plugin Name: ${plugin}`);
-    console.log(`Repository Url: ${options.repository}`);
-
-    if (options.branch) {
-        console.log(`Branch Name: ${options.branch}`);
-    }
-
-    // check installation
-    await verifyInstalledPlugin(plugin, options);
-
-    if (stateJson && stateJson.plugins) {
-        stateJson.plugins[plugin] = {};
-    } else {
-        throwAndQuit('fatal error on state');
-    }
-
-    // clone repository
-    await clonePluginRepo(plugin, options);
-
-    // checkout branch
-    if (options.branch) {
-        const dir = path.join(pluginDirAbsolutePath, plugin);
-        const results = checkoutBranch(dir, options.branch);
-        stateJson.plugins[plugin].branch = options.branch;
-        console.log(results);
-    }
-
-    // build plugin
-    await buildPlugin(plugin, {
-        skipInstall: false
-    });
-}
-
-async function uninstall(plugin) {
-    const pluginDir = path.join(pluginDirAbsolutePath, plugin);
-    try {
-        if (!existsSync(pluginDir)) {
-            throwAndQuit(`plugin ${plugin} not found!`);
-        }
-        console.log(`Removing ${pluginDir}...`);
-        await rm(pluginDir, {
-            recursive: true
-        });
-        if (stateJson && stateJson.plugins) {
-            delete stateJson.plugins[plugin];
-            saveState();
-        }
-        console.log(`${plugin} removed!`);
-    } catch (e) {
-        throwAndQuit(e.message);
-    }
-}
-
-async function listPlugins() {
-    const dirs = await readdir(pluginDirAbsolutePath);
-    const results: any[] = [];
-    for (const dir of dirs) {
-        try {
-            const packageJsonPath = path.join(pluginDirAbsolutePath, dir, 'package.json');
-            const package_json = JSON.parse(readFileSync(packageJsonPath).toString());
-            const {name, version, description} = package_json;
-            // console.table([dir, name, version, description].join('\t'));
-
-            let pluginState = false;
-            let branch = '';
-
-            if (stateJson.plugins && stateJson.plugins[dir]) {
-                if (stateJson.plugins[dir].enabled) {
-                    pluginState = true;
-                }
-                branch = stateJson.plugins[dir].branch;
-            }
-
-            results.push({
-                alias: dir,
-                enabled: pluginState,
-                plugin_name: name,
-                version,
-                branch,
-                description
-            });
-        } catch (e) {
-            console.log(e.message);
-        }
-    }
-    results.sort((a, b) => b.ative - a.ative);
-    console.table(results);
-}
-
-function saveState() {
-    stateJson.update_ts = Date.now();
-    writeFileSync(pluginStatePath, JSON.stringify(stateJson, null, 2));
-}
-
-async function setPluginState(pluginName: string, newState: boolean) {
-    if (stateJson) {
-        if (stateJson.plugins[pluginName]) {
-            stateJson.plugins[pluginName].enabled = newState;
-        } else {
-            stateJson.plugins[pluginName] = {
-                enabled: newState
-            }
-        }
-    }
-    saveState();
-    console.log(`[${pluginName}] plugin ${newState ? 'enabled' : 'disabled'} successfully!`);
-}
-
-function enablePlugin(pluginName: string) {
-    return setPluginState(pluginName, true);
-}
-
-function disablePlugin(pluginName: string) {
-    return setPluginState(pluginName, false);
-}
-
-function printState() {
-    console.log(JSON.stringify(stateJson, null, 2));
-}
-
-async function buildAllPlugins(flags) {
-    const names = readdirSync(pluginDirAbsolutePath);
-    for (const name of names) {
-        await buildPlugin(name, flags)
-    }
-}
-
-async function updatePlugin(name: string) {
-    const pluginDir = path.join(pluginDirAbsolutePath, name);
-    const resetStatus = gitReset(pluginDir);
-    console.log(resetStatus);
-    const pullStatus = gitPull(pluginDir);
-    console.log(pullStatus);
-    await buildPlugin(name, {});
-}
-
-(() => {
-
-    init();
-
-    program.description('Command-line utility to manage Hyperion plugins\nMade with ♥  by EOS Rio');
-
-    program.command('install <plugin>').alias('i')
-        .description('install new plugin')
-        .option('-r, --repository <url>', 'custom repository url', checkUrl)
-        .option('-b, --branch <branch>', 'checkout specific branch')
-        .option('-s, --skip-validation', 'skip plugin integrity validation')
-        .action(installPlugin);
-
-    program.command('enable <plugin>')
-        .description('enable plugin')
-        .action(enablePlugin);
-
-    program.command('disable <plugin>')
-        .description('disable plugin')
-        .action(disablePlugin);
-
-    program.command('state')
-        .description('print current state file')
-        .action(printState);
-
-    program.command('list').alias('ls')
-        .description('list installed plugins')
-        .action(listPlugins);
-
-    program.command('build <plugin>').alias('b')
-        .description('build a single plugin')
-        .option('-s, --skip-install', 'skip "npm install" step')
-        .action(buildPlugin);
-
-    program.command('update <plugin>')
-        .description('update a single plugin')
-        .action(updatePlugin);
-
-    program.command('build-all')
-        .description('build all plugins')
-        .option('-s, --skip-install', 'skip "npm install" step')
-        .action(buildAllPlugins);
-
-    program.command('uninstall <plugin>').alias('rm')
-        .description('uninstall plugin')
-        .action(uninstall);
-
-    program.parse();
-})();

scripts/hyp-config.ts

@@ -1,604 +0,0 @@
-import {Command} from "commander";
-import path from "path";
-import {cp, mkdir, readdir, readFile, rm, writeFile} from "fs/promises";
-import {HyperionConfig} from "../interfaces/hyperionConfig";
-import {HyperionConnections} from "../interfaces/hyperionConnections";
-import {copyFileSync, existsSync, rmSync} from "fs";
-import {JsonRpc} from "eosjs";
-import fetch from "cross-fetch";
-import WebSocket from 'ws';
-import * as readline from "readline";
-import * as amqp from "amqplib";
-import IORedis from "ioredis";
-import {Client} from "@elastic/elasticsearch";
-
-const program = new Command();
-const chainsDir = path.join(path.resolve(), 'chains');
-const connectionsPath = path.join(path.resolve(), 'connections.json');
-const exampleConnectionsPath = path.join(path.resolve(), 'example-connections.json');
-const backupDir = path.join(path.resolve(), 'configuration_backups');
-
-async function getConnections(): Promise<HyperionConnections> {
-    if (existsSync(connectionsPath)) {
-        const connectionsJsonFile = await readFile(connectionsPath);
-        return JSON.parse(connectionsJsonFile.toString());
-    } else {
-        return null;
-    }
-}
-
-async function getExampleConfig(): Promise<HyperionConfig> {
-    const exampleChainData = await readFile(path.join(chainsDir, 'example.config.json'));
-    return JSON.parse(exampleChainData.toString());
-}
-
-
-async function listChains(flags) {
-
-    const dirdata = await readdir(chainsDir);
-    const connections = await getConnections();
-    const exampleChain = await getExampleConfig();
-
-    const configuredTable = [];
-    const pendingTable = [];
-    for (const file of dirdata.filter(f => f.endsWith('.json'))) {
-        if (file === 'example.config.json') {
-            continue;
-        }
-        try {
-            const fileData = await readFile(path.join(chainsDir, file));
-            const jsonData: HyperionConfig = JSON.parse(fileData.toString());
-            const chainName = jsonData.settings.chain;
-            const chainConn = connections.chains[chainName];
-            const fullChainName = jsonData.api.chain_name;
-
-            // validate configs with example
-            let missingSections = 0;
-            let missingFields = 0;
-            for (const key in exampleChain) {
-                if (!jsonData[key]) {
-                    if (flags.fixMissingFields) {
-                        jsonData[key] = exampleChain[key];
-                    } else {
-                        console.log(`⚠ Section ${key} missing on ${file}`);
-                        missingSections++;
-                    }
-                } else {
-                    for (const subKey in exampleChain[key]) {
-                        if (!jsonData[key].hasOwnProperty(subKey)) {
-                            if (flags.fixMissingFields) {
-                                jsonData[key][subKey] = exampleChain[key][subKey];
-                            } else {
-                                console.log(`⚠ Field ${subKey} missing from ${key} on ${file} (default = ${exampleChain[key][subKey]})`);
-                                missingFields++;
-                            }
-                        }
-                    }
-                }
-            }
-
-            if (flags.fixMissingFields) {
-                await writeFile(path.join(chainsDir, file), JSON.stringify(jsonData, null, 2));
-            }
-
-            const common = {
-                full_name: fullChainName,
-                name: chainName,
-                parser: jsonData.settings.parser,
-                abi_scan: jsonData.indexer.abi_scan_mode,
-                live_reader: jsonData.indexer.live_reader,
-                start_on: jsonData.indexer.start_on,
-                stop_on: jsonData.indexer.stop_on
-            };
-
-            if (chainConn) {
-                configuredTable.push({
-                    ...common,
-                    nodeos_http: chainConn?.http,
-                    nodeos_ship: chainConn?.ship,
-                    chain_id: chainConn?.chain_id.substr(0, 16) + '...'
-                });
-            } else {
-                pendingTable.push(common);
-            }
-        } catch (e) {
-            console.log(`Failed to parse ${file} - ${e.message}`);
-        }
-    }
-
-    console.log(' >>>> Fully Configured Chains');
-    console.table(configuredTable);
-
-    if (!flags.valid) {
-        console.log(' >>>> Pending Configuration Chains');
-        console.table(pendingTable);
-    }
-}
-
-async function newChain(shortName, options) {
-    console.log(`Creating new config for ${shortName}...`);
-    const targetPath = path.join(chainsDir, `${shortName}.config.json`);
-
-    if (existsSync(targetPath)) {
-        console.error(`Chain config for ${shortName} already defined!`);
-        process.exit(0);
-    }
-
-    // read example
-    const exampleChain = await getExampleConfig();
-
-    // read connections.json
-    const connections = await getConnections();
-
-    if (!connections) {
-        console.log(`The connections.json file is not present, please run "./hyp-config connections init" to configure it.`);
-        process.exit(1);
-    }
-
-    if (connections.chains[shortName]) {
-        console.log('Connections already defined!');
-        console.log(connections.chains[shortName]);
-        process.exit(0);
-    } else {
-        connections.chains[shortName] = {
-            name: '',
-            ship: '',
-            http: '',
-            chain_id: '',
-            WS_ROUTER_HOST: '127.0.0.1',
-            WS_ROUTER_PORT: 7001,
-            control_port: 7002
-        };
-    }
-
-    const jsonData = exampleChain;
-    jsonData.settings.chain = shortName;
-
-    if (options.http) {
-
-        if (options.http.startsWith('http://') || options.http.startsWith('https://')) {
-            console.log(`Testing connection on ${options.http}`);
-        } else {
-            console.error(`Invalid HTTP Endpoint [${options.http}] - Url must start with either http:// or https://`);
-            process.exit(1);
-        }
-
-        // test nodeos availability
-        try {
-            const jsonRpc = new JsonRpc(options.http, {fetch});
-            const info = await jsonRpc.get_info();
-            jsonData.api.chain_api = options.http;
-            connections.chains[shortName].chain_id = info.chain_id;
-            connections.chains[shortName].http = options.http;
-            if (info.server_version_string.includes('v3')) {
-                jsonData.settings.parser = '3.2';
-            } else if (info.server_version_string.includes('v2.1')) {
-                jsonData.settings.parser = '2.1';
-            } else {
-                jsonData.settings.parser = '1.8';
-            }
-            console.log(`Parser version set to ${jsonData.settings.parser}`);
-            console.log(`Chain ID: ${info.chain_id}`);
-        } catch (e) {
-            console.error(`Invalid HTTP Endpoint [${options.http}] - ${e.message}`);
-            process.exit(1);
-        }
-    }
-
-    if (options.ship) {
-
-        if (options.ship.startsWith('ws://') || options.ship.startsWith('wss://')) {
-            console.log(`Testing connection on ${options.ship}`);
-        } else {
-            console.error(`Invalid WS Endpoint [${options.ship}] - Url must start with either ws:// or wss://`);
-            process.exit(1);
-        }
-
-        // test ship availability
-        const status = await new Promise<boolean>(resolve => {
-            const ws = new WebSocket(options.ship);
-            ws.on("message", (data: Buffer) => {
-                try {
-                    const abi = JSON.parse(data.toString());
-                    if (abi.version) {
-                        console.log(`Received SHIP Abi Version: ${abi.version}`);
-                        resolve(true);
-                    } else {
-                        resolve(false);
-                    }
-                } catch (e) {
-                    console.log(e.message);
-                    resolve(false);
-                }
-                ws.close();
-            });
-            ws.on("error", err => {
-                console.log(err);
-                ws.close();
-                resolve(false);
-            });
-        });
-        if (status) {
-            connections.chains[shortName].ship = options.ship;
-        } else {
-            console.log(`Invalid SHIP Endpoint [${options.ship}]`);
-        }
-    }
-
-    const fullNameArr = [];
-    shortName.split('-').forEach((word: string) => {
-        fullNameArr.push(word[0].toUpperCase() + word.substr(1));
-    });
-    const fullChainName = fullNameArr.join(' ');
-
-    jsonData.api.chain_name = fullChainName;
-    connections.chains[shortName].name = fullChainName;
-
-    console.log(connections.chains[shortName]);
-
-    console.log('Saving connections.json...');
-    await writeFile(connectionsPath, JSON.stringify(connections, null, 2));
-    console.log(`Saving chains/${shortName}.config.json...`);
-    await writeFile(targetPath, JSON.stringify(jsonData, null, 2));
-}
-
-async function rmChain(shortName) {
-    console.log(`Removing config for ${shortName}...`);
-    const targetPath = path.join(chainsDir, `${shortName}.config.json`);
-
-    if (!existsSync(targetPath)) {
-        console.error(`Chain config for ${shortName} not found!`);
-        process.exit(0);
-    }
-
-    // create backups
-    if (!existsSync(backupDir)) {
-        await mkdir(backupDir);
-    }
-
-    const connections = await getConnections();
-    try {
-        const chainConn = connections.chains[shortName];
-        const now = Date.now();
-        if (chainConn) {
-            await writeFile(
-                path.join(backupDir, `${shortName}_${now}_connections.json`),
-                JSON.stringify(chainConn, null, 2)
-            );
-        }
-        await cp(targetPath, path.join(backupDir, `${shortName}_${now}_config.json`));
-    } catch (e) {
-        console.log(e.message);
-        console.log('Failed to create backups! Aborting!');
-        process.exit(1);
-    }
-
-    console.log(`Deleting ${targetPath}`);
-    await rm(targetPath);
-    delete connections.chains[shortName];
-    console.log('Saving connections.json...');
-    await writeFile(connectionsPath, JSON.stringify(connections, null, 2));
-    console.log(`✅ ${shortName} removal completed!`);
-}
-
-async function getExampleConnections(): Promise<HyperionConnections | null> {
-    try {
-        const connectionsJsonFile = await readFile(exampleConnectionsPath);
-        return JSON.parse(connectionsJsonFile.toString());
-    } catch (e: any) {
-        return null;
-    }
-}
-
-async function checkAMQP(conn: HyperionConnections): Promise<boolean> {
-    console.log(`\n[info] [RABBIT:AMQP] - Testing rabbitmq amqp connection...`);
-    let frameMaxValue = '0x10000';
-    if (conn.amqp.frameMax) {
-        frameMaxValue = conn.amqp.frameMax;
-    }
-    const u = encodeURIComponent(conn.amqp.user);
-    const p = encodeURIComponent(conn.amqp.pass);
-    const v = encodeURIComponent(conn.amqp.vhost)
-    const amqp_url = `amqp://${u}:${p}@${conn.amqp.host}/${v}?frameMax=${frameMaxValue}`;
-    try {
-        const connection = await amqp.connect(amqp_url);
-        if (connection) {
-            console.log('[info] [RABBIT:AMQP] - Connection established!');
-            await connection.close();
-            console.log(`\n[info] [RABBIT:HTTP] - Testing rabbitmq http api connection...`);
-            const apiUrl = `${conn.amqp.protocol}://${conn.amqp.api}/api/nodes`;
-
-            let headers = new Headers();
-            headers.set('Authorization', 'Basic ' + btoa(conn.amqp.user + ":" + conn.amqp.pass));
-            const data = await fetch(apiUrl, {
-                method: 'GET',
-                headers: headers
-            });
-            const resp = await data.json();
-            if (resp.length > 0) {
-                for (const node of resp) {
-                    console.log(`[info] [RABBIT:HTTP] - Node name: ${node.name} (running: ${node.running})`);
-                    if (!node.running) {
-                        return false;
-                    }
-                }
-                return true;
-            } else {
-                return false;
-            }
-        } else {
-            return false;
-        }
-    } catch (reason: any) {
-        console.log('[error] [RABBIT] - ' + reason.message);
-        return false;
-    }
-}
-
-async function checkES(conn: HyperionConnections): Promise<boolean> {
-    console.log(`\n[info] [ES] - Testing elasticsearch connection...`);
-    let es_url;
-    const _es = conn.elasticsearch;
-    if (!_es.protocol) {
-        _es.protocol = 'http';
-    }
-    if (_es.user !== '') {
-        es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${_es.host}`;
-    } else {
-        es_url = `${_es.protocol}://${_es.host}`
-    }
-    // console.log(`Prepared client: ${es_url}`);
-    const client = new Client({
-        node: es_url,
-        ssl: {rejectUnauthorized: false}
-    });
-    try {
-        const result = await client.cat.health();
-        console.log(`[info] [ES] - ${result.body.trim()}`);
-        console.log('[info] [ES] - Connection established!');
-        return true;
-    } catch (reason: any) {
-        console.log('[error] [ES] - ' + reason.message);
-        return false;
-    }
-}
-
-async function checkRedis(conn: HyperionConnections) {
-    console.log(`\n[info] [REDIS] - Testing redis connection...`);
-    try {
-        const ioRedisClient = new IORedis(conn.redis);
-        const pingResult = await ioRedisClient.ping();
-        if (pingResult === 'PONG') {
-            console.log('[info] [REDIS] - Connection established!');
-            ioRedisClient.disconnect();
-            return true;
-        } else {
-            console.log('[error] [REDIS] - PONG expected, got: ' + pingResult);
-            return false;
-        }
-    } catch (reason: any) {
-        console.log('[error] [REDIS] - ' + reason.message);
-        return false;
-    }
-}
-
-async function initConfig() {
-    const connections = await getConnections();
-
-    if (connections) {
-        console.log('connections.json already created!\n use "./hyp-config connections reset" to revert to default');
-        return;
-    }
-
-    const exampleConn = await getExampleConnections();
-
-    const rl = readline.createInterface({input: process.stdin, output: process.stdout});
-    const prompt = (query: string) => new Promise((resolve) => rl.question(query, resolve));
-
-    const conn = exampleConn;
-
-    if (!conn) {
-        console.log('No example-connections.json to use as reference!');
-        process.exit(1);
-    }
-    conn.chains = {};
-
-    // check rabbitmq
-    let amqp_state = false;
-    while (!amqp_state) {
-        amqp_state = await checkAMQP(conn);
-        if (!amqp_state) {
-
-            const amqp_user = await prompt('\n > Enter the RabbitMQ user (or press ENTER to use "hyperion_user"): ');
-            if (amqp_user) {
-                conn.amqp.user = amqp_user as string;
-            } else {
-                conn.amqp.user = 'hyperion_user'
-            }
-
-            const amqp_pass = await prompt(' > Enter the RabbitMQ password (or press ENTER to use "hyperion_password"): ');
-            if (amqp_pass) {
-                conn.amqp.pass = amqp_pass as string;
-            } else {
-                conn.amqp.pass = 'hyperion_password'
-            }
-
-            const amqp_vhost = await prompt(' > Enter the RabbitMQ vhost (or press ENTER to use "hyperion"): ');
-            if (amqp_vhost) {
-                conn.amqp.vhost = amqp_vhost as string;
-            } else {
-                conn.amqp.vhost = 'hyperion'
-            }
-
-            console.log('\n------ current RabbitMQ config -----');
-            console.log(conn.amqp);
-            console.log('--------------------------------------');
-        }
-    }
-
-    console.log(`\n RabbitMQ ✅`);
-
-    // check elasticsearch
-    let elastic_state = false;
-    while (!elastic_state) {
-        elastic_state = await checkES(conn);
-        if (!elastic_state) {
-
-            const es_user = await prompt('\n > Enter the elasticsearch user (or press ENTER to use "elastic"): ');
-            if (es_user) {
-                conn.elasticsearch.user = es_user as string;
-            } else {
-                conn.elasticsearch.user = 'elastic'
-            }
-
-            const es_pass = await prompt(' > Enter the elasticsearch password: ');
-            if (es_pass) {
-                conn.elasticsearch.pass = es_pass as string;
-            }
-
-            const es_proto = await prompt(' > Do you want to use http or https?\n1 = http\n2 = https (default)\n');
-            if (es_proto === "1") {
-                conn.elasticsearch.protocol = 'http';
-            } else {
-                conn.elasticsearch.protocol = 'https';
-            }
-
-            console.log('\n------ current elasticsearch config -----');
-            console.log(conn.elasticsearch);
-            console.log('-------------------------------------------');
-        }
-    }
-
-    console.log(`\n Elasticsearch ✅`);
-
-    // check redis connection
-    let redis_state = false;
-    while (!redis_state) {
-        redis_state = await checkRedis(conn);
-        if (!redis_state) {
-            const redis_host = await prompt('\n > Enter the Redis Server address (or press ENTER to use "127.0.0.1"): ');
-            if (redis_host) {
-                conn.redis.host = redis_host as string;
-            } else {
-                conn.redis.host = '127.0.0.1';
-            }
-
-            const redis_port = await prompt(' > Enter the Redis Server port (or press ENTER to use "6379"): ');
-            if (redis_port) {
-                conn.redis.port = parseInt(redis_port as string);
-            } else {
-                conn.redis.port = 6379;
-            }
-
-            console.log('\n------ current redis config -----');
-            console.log(conn.redis);
-            console.log('-----------------------------------');
-        }
-    }
-
-    console.log(`\n Redis ✅`);
-
-    console.log('Init completed! Saving configuration...');
-
-    await writeFile(connectionsPath, JSON.stringify(conn, null, 2));
-
-    console.log('✅ ✅');
-    rl.close();
-}
-
-async function testConnections() {
-    if (existsSync(connectionsPath)) {
-        try {
-            const conn = await getConnections();
-            if (conn) {
-                const results = {
-                    redis: await checkRedis(conn),
-                    elasticsearch: await checkES(conn),
-                    rabbitmq: await checkAMQP(conn)
-                }
-                console.log('\n------ Testing Completed -----');
-                console.table(results);
-            } else {
-                console.log(`The connections.json was not found!.`);
-            }
-        } catch (e: any) {
-            console.log(e.message);
-        }
-    } else {
-        console.log(`The connections.json file is not present, please run "./hyp-config connections init" to configure it.`);
-    }
-}
-
-async function resetConnections() {
-    try {
-
-        // create backups
-        if (!existsSync(backupDir)) {
-            await mkdir(backupDir);
-        }
-
-        if (existsSync(connectionsPath)) {
-            const rl = readline.createInterface({input: process.stdin, output: process.stdout});
-            const prompt = (query: string) => new Promise((resolve) => rl.question(query, resolve));
-            const confirmation = await prompt('Are you sure you want to reset the connection configuration? Type "YES" to confirm.\n');
-            if (confirmation === 'YES') {
-                copyFileSync(connectionsPath, path.join(backupDir, 'connections.json.bak'));
-                rmSync(connectionsPath);
-                console.log('connections.json removed, please use "./hyp-config connections init" to reconfigure');
-            } else {
-                console.log('Operation canceled. No files were removed.');
-            }
-            rl.close();
-        } else {
-            console.log(`The connections.json file is not present, please run "./hyp-config connections init" to configure it.`);
-        }
-    } catch (e: any) {
-        console.log(e);
-    }
-}
-
-
-// main program
-(() => {
-
-    const connections = program.command('connections');
-
-    // ./hyp-config connections init
-    connections
-        .command('init')
-        .description('create and test the connections.json file')
-        .action(initConfig);
-
-    // ./hyp-config connections test
-    connections
-        .command('test')
-        .description('test connections to the hyperion infrastructure')
-        .action(testConnections);
-
-    // ./hyp-config connections reset
-    connections
-        .command('reset')
-        .description('remove the connections.json file')
-        .action(resetConnections);
-
-    const list = program.command('list').alias('ls');
-    list.command('chains')
-        .option('--valid', 'only show valid chains')
-        .option('--fix-missing-fields', 'set defaults on missing fields')
-        .description('list configured chains')
-        .action(listChains);
-
-    const newCmd = program.command('new').alias('n');
-    newCmd.command('chain <shortName>')
-        .description('initialize new chain config based on example')
-        .option('--http <http_endpoint>', 'define chain api http endpoint')
-        .option('--ship <ship_endpoint>', 'define state history ws endpoint')
-        .action(newChain)
-
-    const remove = program.command('remove').alias('rm');
-    remove.command('chain <shortName>')
-        .description('backup and delete chain configuration')
-        .action(rmChain);
-
-    program.parse(process.argv);
-})();

scripts/hyp-repair.ts

@@ -1,954 +0,0 @@
-import { Command } from 'commander';
-import { readFileSync } from 'node:fs';
-import { Client, estypes } from '@elastic/elasticsearch';
-// @ts-ignore
-import cliProgress from 'cli-progress';
-import { JsonRpc } from 'eosjs';
-import fetch from 'cross-fetch';
-import { existsSync, mkdirSync, writeFileSync } from 'fs';
-import { HyperionBlock } from './repair-cli/interfaces.js';
-import {
-    getBlocks,
-    getFirstIndexedBlock,
-    getLastIndexedBlock,
-    initESClient,
-    readChainConfig,
-    readConnectionConfig,
-} from './repair-cli/functions.js';
-import { SearchResponse } from '@elastic/elasticsearch/api/types';
-
-import { WebSocket } from 'ws';
-import { block } from '../definitions/index-templates';
-
-const progressBar = new cliProgress.SingleBar(
-    {},
-    cliProgress.Presets.shades_classic
-);
-const program = new Command();
-const errorRanges: any[] = [];
-let pendingBlock: HyperionBlock | null = null;
-
-const missingBlocks: {
-    start: number;
-    end: number;
-}[] = [];
-
-async function run(
-    client: Client,
-    rpc: JsonRpc,
-    indexName: string,
-    blockInit: number,
-    firstBlock: number,
-    qtdTotal: any,
-    loop: any = 1
-) {
-    let blockInitial = blockInit;
-    let blockFinal = blockInitial - qtdTotal;
-    const tRef = process.hrtime.bigint();
-    progressBar.start(loop, 0);
-    for (let i: any = 1; i <= loop; i++) {
-        if (blockFinal < firstBlock) {
-            blockFinal = firstBlock;
-        }
-        try {
-            let { body } = await getBlocks(
-                client,
-                indexName,
-                blockInitial,
-                blockFinal,
-                qtdTotal
-            );
-            let {
-                hits: { hits },
-            } = body;
-            const blocks = hits.map((obj: any) => obj._source);
-            await findForksOnRange(blocks, rpc);
-            blockInitial = blockFinal;
-            blockFinal = blockInitial - qtdTotal;
-            progressBar.update(i);
-        } catch (e: any) {
-            console.log('Error: ', e);
-            process.exit(1);
-        }
-    }
-    progressBar.stop();
-    console.log('=========== Forked Ranges ===========');
-    console.table(errorRanges);
-    console.log('=========== Missing Ranges ==========');
-    console.table(missingBlocks);
-    const tDiff = Number(process.hrtime.bigint() - tRef) / 1000000;
-    console.log(`Total time: ${Math.round(tDiff / 1000)}s`);
-}
-
-async function findForksOnRange(blocks: HyperionBlock[], rpc: JsonRpc) {
-    const removals: Set<string> = new Set();
-    let start = null;
-    let end = null;
-    if (
-        blocks.length > 0 &&
-        pendingBlock &&
-        blocks[0].block_num !== pendingBlock.block_num
-    ) {
-        blocks.unshift(pendingBlock);
-    }
-
-    let i = 0;
-    for (const currentBlock of blocks) {
-        const currentBlockNumber = currentBlock.block_num;
-        // console.log('current -> ', currentBlockNumber);
-        const previousBlock = blocks[i + 1];
-        if (!previousBlock) {
-            pendingBlock = currentBlock;
-            // console.log('pending -> ', pendingBlock.block_num);
-            return;
-        }
-        i++;
-        // console.log('previous -> ', previousBlock.block_num);
-        if (previousBlock.block_num !== currentBlockNumber - 1) {
-            console.log(
-                `\nBlock number mismatch, expected: ${
-                    currentBlockNumber - 1
-                } got ${previousBlock.block_num}`
-            );
-            missingBlocks.push({
-                start: previousBlock.block_num + 1,
-                end: currentBlockNumber - 1,
-            });
-            continue;
-        }
-
-        if (previousBlock && previousBlock.block_id !== currentBlock.prev_id) {
-            if (start === null) {
-                start = currentBlockNumber - 1;
-                const blockData = await rpc.get_block_info(start);
-                if (blockData && blockData.id !== previousBlock.block_id) {
-                    removals.add(previousBlock.block_id);
-                }
-            }
-        } else {
-            if (start) {
-                const blockData = await rpc.get_block_info(currentBlockNumber);
-                if (blockData) {
-                    if (blockData.id !== currentBlock.block_id) {
-                        removals.add(currentBlock.block_id);
-                    } else {
-                        end = currentBlockNumber + 1;
-                        const range = { start, end, ids: [...removals] };
-                        errorRanges.push(range);
-                        removals.clear();
-                        // console.log(`\n ⚠️⚠️ Forked at ${range.start} to ${range.end}`);
-                        start = null;
-                        end = null;
-                    }
-                }
-            }
-        }
-    }
-}
-
-async function scanChain(chain: string, args: any) {
-    const chainConfig = readChainConfig(chain);
-
-    console.log(chainConfig.settings.index_version);
-
-    const config = readConnectionConfig();
-
-    const client = initESClient(config);
-
-    const jsonRpc = new JsonRpc(config.chains[chain].http, { fetch });
-
-    const ping = await client.ping();
-
-    if (!ping) {
-        console.log('Could not connect to ElasticSearch');
-        process.exit();
-    }
-
-    const blockIndex = `${chain}-block-${chainConfig.settings.index_version}`;
-
-    let firstBlock;
-    let lastBlock;
-
-    if (args.first) {
-        // reduce by 2 to account for the fork end validation
-        firstBlock = args.first - 2;
-    } else {
-        firstBlock = await getFirstIndexedBlock(client, blockIndex);
-    }
-
-    if (args.last) {
-        lastBlock = args.last;
-    } else {
-        lastBlock = await getLastIndexedBlock(client, blockIndex);
-    }
-
-    const totalBlocks = lastBlock - firstBlock;
-
-    let batchSize = 1000;
-    if (args.batch) {
-        batchSize = args.batch;
-    }
-
-    const numberOfBatches = Math.ceil(totalBlocks / batchSize);
-
-    console.log('Range:', firstBlock, lastBlock);
-    console.log('Total Blocks:', totalBlocks);
-    console.log('Batch Size:', batchSize);
-    console.log('Number of Batches:', numberOfBatches);
-
-    await run(
-        client,
-        jsonRpc,
-        blockIndex,
-        lastBlock,
-        firstBlock,
-        batchSize,
-        numberOfBatches
-    );
-    console.log(`Finished checking forked blocks!`);
-
-    if (errorRanges.length > 0 || missingBlocks.length > 0) {
-        if (!existsSync('.repair')) {
-            mkdirSync('.repair');
-        }
-    }
-
-    if (errorRanges.length > 0) {
-        let path = `.repair/${chain}-${
-            firstBlock + 2
-        }-${lastBlock}-forked-blocks.json`;
-        if (args.outFile) {
-            path = args.outFile + '-forked-blocks.json';
-        }
-        writeFileSync(path, JSON.stringify(errorRanges));
-    }
-
-    if (missingBlocks.length > 0) {
-        let path = `.repair/${chain}-${
-            firstBlock + 2
-        }-${lastBlock}-missing-blocks.json`;
-        if (args.outFile) {
-            path = args.outFile + '-missing-blocks.json';
-        }
-        writeFileSync(path, JSON.stringify(missingBlocks));
-    }
-}
-
-async function repairMissing(chain: string, file: string, args: any) {
-    const chainConfig = readChainConfig(chain);
-    const config = readConnectionConfig();
-    const client = initESClient(config);
-    const ping = await client.ping();
-    if (!ping) {
-        console.log('Could not connect to ElasticSearch');
-        process.exit();
-    }
-    await fillMissingBlocksFromFile(args.host, chain, file, args.dry);
-}
-
-async function repairChain(chain: string, file: string, args: any) {
-    const chainConfig = readChainConfig(chain);
-    const config = readConnectionConfig();
-    const client = initESClient(config);
-    const ping = await client.ping();
-    if (!ping) {
-        console.log('Could not connect to ElasticSearch');
-        process.exit();
-    }
-
-    if (args.checkTasks) {
-        const tasks = await client.tasks.list();
-        if (tasks.body && tasks.body.nodes) {
-            for (let node in tasks.body.nodes) {
-                const nodeTasks = tasks.body.nodes[node].tasks;
-                console.log(nodeTasks);
-            }
-        }
-        process.exit(0);
-    }
-
-    const forkedBlocks = JSON.parse(readFileSync(file).toString());
-    const blockIndex = `${chain}-block-${chainConfig.settings.index_version}`;
-    let deleteBlocks = 0;
-    let deleteActions = 0;
-    let deleteDeltas = 0;
-    let deleteAbis = 0;
-    let deleteAccounts = 0;
-    let deleteVoters = 0;
-    let deleteProposals = 0;
-    let deleteLinks = 0;
-    let deletePermissions = 0;
-
-    for (const range of forkedBlocks) {
-        // ACTIONS
-        const searchActions = {
-            index: `${chain}-action-${chainConfig.settings.index_version}-*`,
-            size: 0,
-            track_total_hits: true,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                range: {
-                                    block_num: {
-                                        lte: range.start,
-                                        gte: range.end,
-                                    },
-                                },
-                            },
-                        ],
-                    },
-                },
-            },
-        };
-
-        if (args.dry) {
-            const resultActions = await client.search<any>(searchActions);
-            if (
-                (resultActions.body.hits.total as estypes.SearchTotalHits)
-                    ?.value > 0
-            ) {
-                deleteActions += (
-                    resultActions.body.hits.total as estypes.SearchTotalHits
-                )?.value;
-            }
-        } else {
-            const indexExists = await client.indices.exists({
-                index: searchActions.index,
-            });
-
-            if (indexExists.body) {
-                delete searchActions.size;
-                const deletedActionsResult = await client.deleteByQuery(
-                    searchActions
-                );
-                if (
-                    deletedActionsResult &&
-                    deletedActionsResult.body.deleted &&
-                    deletedActionsResult.body.deleted > 0
-                ) {
-                    deleteActions += deletedActionsResult.body.deleted;
-                }
-            } else {
-                console.log(
-                    `Index ${searchActions.index} doesn't exist. Unable to delete.`
-                );
-            }
-        }
-
-        // DELTAS
-        const searchDeltas = {
-            index: `${chain}-delta-${chainConfig.settings.index_version}-*`,
-            size: 0,
-            track_total_hits: true,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                range: {
-                                    block_num: {
-                                        lte: range.start,
-                                        gte: range.end,
-                                    },
-                                },
-                            },
-                        ],
-                    },
-                },
-            },
-        };
-
-        if (args.dry) {
-            const resultDeltas = await client.search<any>(searchDeltas);
-            if (
-                (resultDeltas.body.hits.total as estypes.SearchTotalHits)
-                    ?.value > 0
-            ) {
-                deleteDeltas += (
-                    resultDeltas.body.hits.total as estypes.SearchTotalHits
-                )?.value;
-            }
-        } else {
-            const indexExists = await client.indices.exists({
-                index: searchDeltas.index,
-            });
-            if (indexExists.body) {
-                delete searchDeltas.size;
-                const deletedDeltasResult = await client.deleteByQuery(
-                    searchDeltas
-                );
-                if (
-                    deletedDeltasResult &&
-                    deletedDeltasResult.body.deleted &&
-                    deletedDeltasResult.body.deleted > 0
-                ) {
-                    deleteDeltas += deletedDeltasResult.body.deleted;
-                }
-            } else {
-                console.log(
-                    `Index ${searchDeltas.index} doesn't exist. Unable to delete.`
-                );
-            }
-        }
-
-        // ABIS
-        const searchAbis = {
-            index: `${chain}-abi-${chainConfig.settings.index_version}`,
-            size: 0,
-            track_total_hits: true,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                range: {
-                                    block: { lte: range.start, gte: range.end },
-                                },
-                            },
-                        ],
-                    },
-                },
-            },
-        };
-
-        if (args.dry) {
-            const resultAbis = await client.search<any>(searchAbis);
-            if (
-                (resultAbis.body.hits.total as estypes.SearchTotalHits)?.value >
-                0
-            ) {
-                deleteAbis += (
-                    resultAbis.body.hits.total as estypes.SearchTotalHits
-                )?.value;
-                console.log('ABIs', { lte: range.start, gte: range.end });
-            }
-        } else {
-            const indexExists = await client.indices.exists({
-                index: searchAbis.index,
-            });
-            if (indexExists.body) {
-                delete searchAbis.size;
-                const deletedAbisResult = await client.deleteByQuery(
-                    searchAbis
-                );
-                if (
-                    deletedAbisResult &&
-                    deletedAbisResult.body.deleted &&
-                    deletedAbisResult.body.deleted > 0
-                ) {
-                    deleteAbis += deletedAbisResult.body.deleted;
-                }
-            } else {
-                console.log(
-                    `Index ${searchAbis.index} doesn't exist. Unable to delete.`
-                );
-            }
-        }
-
-        // ACCOUNTS
-        const searchAccounts = {
-            index: `${chain}-table-accounts-${chainConfig.settings.index_version}`,
-            size: 0,
-            track_total_hits: true,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                range: {
-                                    block_num: {
-                                        lte: range.start,
-                                        gte: range.end,
-                                    },
-                                },
-                            },
-                        ],
-                    },
-                },
-            },
-        };
-
-        if (args.dry) {
-            const resultAccounts = await client.search<any>(searchAccounts);
-            if (
-                (resultAccounts.body.hits.total as estypes.SearchTotalHits)
-                    ?.value > 0
-            ) {
-                console.log(
-                    '[WARNING]',
-                    (resultAccounts.body.hits.total as estypes.SearchTotalHits)
-                        ?.value,
-                    'accounts needs to be updated'
-                );
-                deleteAccounts += (
-                    resultAccounts.body.hits.total as estypes.SearchTotalHits
-                )?.value;
-            }
-        } else {
-
-            const indexExists = await client.indices.exists({
-                index: searchAccounts.index,
-            });
-
-            if (indexExists.body) {
-            delete searchAccounts.size;
-            const deletedAccountsResult = await client.deleteByQuery(
-                searchAccounts
-            );
-            if (
-                deletedAccountsResult &&
-                deletedAccountsResult.body.deleted &&
-                deletedAccountsResult.body.deleted > 0
-            ) {
-                deleteAccounts += deletedAccountsResult.body.deleted;
-            }
-        }else {
-            console.log(
-                `Index ${searchAccounts.index} doesn't exist. Unable to delete.`
-            );
-        }
-    }
-
-        // VOTERS
-        const searchVoters = {
-            index: `${chain}-table-voters-${chainConfig.settings.index_version}`,
-            size: 0,
-            track_total_hits: true,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                range: {
-                                    block_num: {
-                                        lte: range.start,
-                                        gte: range.end,
-                                    },
-                                },
-                            },
-                        ],
-                    },
-                },
-            },
-        };
-
-        if (args.dry) {
-            const resultVoters = await client.search<any>(searchVoters);
-            if (
-                (resultVoters.body.hits.total as estypes.SearchTotalHits)
-                    ?.value > 0
-            ) {
-                console.log(
-                    '[WARNING]',
-                    (resultVoters.body.hits.total as estypes.SearchTotalHits)
-                        ?.value,
-                    'voters needs to be updated'
-                );
-                deleteVoters += (
-                    resultVoters.body.hits.total as estypes.SearchTotalHits
-                )?.value;
-            }
-        } else {
-            const indexExists = await client.indices.exists({
-                index: searchVoters.index,
-            });
-
-            if (indexExists.body) {
-
-            delete searchVoters.size;
-            const deletedVotersResult = await client.deleteByQuery(
-                searchVoters
-            );
-            if (
-                deletedVotersResult &&
-                deletedVotersResult.body.deleted &&
-                deletedVotersResult.body.deleted > 0
-            ) {
-                deleteVoters += deletedVotersResult.body.deleted;
-            }
-        }else {
-            console.log(
-                `Index ${searchVoters.index} doesn't exist. Unable to delete.`
-            );
-        }
-    }
-
-        // PROPOSALS
-        const searchProposals = {
-            index: `${chain}-table-proposals-${chainConfig.settings.index_version}`,
-            size: 0,
-            track_total_hits: true,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                range: {
-                                    block_num: {
-                                        lte: range.start,
-                                        gte: range.end,
-                                    },
-                                },
-                            },
-                        ],
-                    },
-                },
-            },
-        };
-
-        if (args.dry) {
-            const resultProposals = await client.search<any>(searchProposals);
-            if (
-                (resultProposals.body.hits.total as estypes.SearchTotalHits)
-                    ?.value > 0
-            ) {
-                console.log(
-                    '[WARNING]',
-                    (resultProposals.body.hits.total as estypes.SearchTotalHits)
-                        ?.value,
-                    'proposals needs to be updated'
-                );
-                deleteProposals += (
-                    resultProposals.body.hits.total as estypes.SearchTotalHits
-                )?.value;
-            }
-        } else {
-
-
-            const indexExists = await client.indices.exists({index: searchProposals.index});
-
-            if (indexExists.body) {
-            delete searchProposals.size;
-            const deletedProposalsResult = await client.deleteByQuery(
-                searchProposals
-            );
-            if (
-                deletedProposalsResult &&
-                deletedProposalsResult.body.deleted &&
-                deletedProposalsResult.body.deleted > 0
-            ) {
-                deleteProposals += deletedProposalsResult.body.deleted;
-            }
-        }else { console.log(`Index ${searchProposals.index} doesn't exist. Unable to delete.`);}
-    }
-
-        // LINKS
-        const searchLinks = {
-            index: `${chain}-link-${chainConfig.settings.index_version}`,
-            size: 0,
-            track_total_hits: true,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                range: {
-                                    block_num: {
-                                        lte: range.start,
-                                        gte: range.end,
-                                    },
-                                },
-                            },
-                        ],
-                    },
-                },
-            },
-        };
-
-        if (args.dry) {
-            const resultLinks = await client.search<any>(searchLinks);
-            if (
-                (resultLinks.body.hits.total as estypes.SearchTotalHits)
-                    ?.value > 0
-            ) {
-                console.log(
-                    '[WARNING]',
-                    (resultLinks.body.hits.total as estypes.SearchTotalHits)
-                        ?.value,
-                    'links needs to be updated'
-                );
-                deleteLinks += (
-                    resultLinks.body.hits.total as estypes.SearchTotalHits
-                )?.value;
-            }
-        } else {
-            const indexExists = await client.indices.exists({index: searchLinks.index});
-
-            if (indexExists.body) {
-            delete searchLinks.size;
-            const deletedLinksResult = await client.deleteByQuery(searchLinks);
-            if (
-                deletedLinksResult &&
-                deletedLinksResult.body.deleted &&
-                deletedLinksResult.body.deleted > 0
-            ) {
-                deleteLinks += deletedLinksResult.body.deleted;
-            }
-        }else { console.log(`Index ${searchLinks.index} doesn't exist. Unable to delete.`);}
-    }
-
-        // PERMISSIONS
-        const searchPermissions = {
-            index: `${chain}-perm-${chainConfig.settings.index_version}`,
-            size: 0,
-            track_total_hits: true,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {
-                                range: {
-                                    block_num: {
-                                        lte: range.start,
-                                        gte: range.end,
-                                    },
-                                },
-                            },
-                        ],
-                    },
-                },
-            },
-        };
-
-        if (args.dry) {
-            const resultPermissions = await client.search<any>(
-                searchPermissions
-            );
-            if (
-                (resultPermissions.body.hits.total as estypes.SearchTotalHits)
-                    ?.value > 0
-            ) {
-                console.log(
-                    '[WARNING]',
-                    (
-                        resultPermissions.body.hits
-                            .total as estypes.SearchTotalHits
-                    )?.value,
-                    'permissions needs to be updated'
-                );
-                console.log({ lte: range.start, gte: range.end });
-                deletePermissions += (
-                    resultPermissions.body.hits.total as estypes.SearchTotalHits
-                )?.value;
-            }
-        } else {
-
-            const indexExists = await client.indices.exists({index: searchPermissions.index});
-            if (indexExists.body) {
-            delete searchPermissions.size;
-            const deletedPermissionsResult = await client.deleteByQuery(
-                searchPermissions
-            );
-            if (
-                deletedPermissionsResult &&
-                deletedPermissionsResult.body.deleted &&
-                deletedPermissionsResult.body.deleted > 0
-            ) {
-                deletePermissions += deletedPermissionsResult.body.deleted;
-            }
-        }else { console.log(`Index ${searchPermissions.index} doens't exist. Não foi possível realizar a exclusão.`);}
-    }
-
-        for (const id of range.ids) {
-            const searchBlocks = {
-                index: blockIndex,
-                body: {
-                    query: {
-                        bool: { must: [{ term: { block_id: { value: id } } }] },
-                    },
-                },
-            };
-            if (args.dry) {
-                console.log(`Deleting block ${id}...`);
-                const resultBlocks = await client.search<
-                    SearchResponse<HyperionBlock>
-                >(searchBlocks);
-                if (
-                    resultBlocks.body.hits.hits.length > 0 &&
-                    resultBlocks.body.hits.hits[0]._source
-                ) {
-                    deleteBlocks++;
-                }
-            } else {
-                const result = await client.deleteByQuery(searchBlocks);
-                if (result && result.body.deleted && result.body.deleted > 0) {
-                    deleteBlocks += result.body.deleted;
-                }
-            }
-        }
-    }
-
-    if (args.dry) {
-        console.log(
-            `DRY-RUN: Would have deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`
-        );
-        console.dir({
-            blocks: deleteBlocks,
-            actions: deleteActions,
-            deltas: deleteDeltas,
-            abis: deleteAbis,
-            accounts: deleteAccounts,
-            voters: deleteVoters,
-            proposals: deleteProposals,
-            links: deleteLinks,
-            permissions: deletePermissions,
-        });
-    } else {
-        console.log(
-            `Deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`
-        );
-    }
-
-    await fillMissingBlocksFromFile(args.host, chain, file, args.dry);
-}
-
-async function fillMissingBlocksFromFile(host, chain, file, dryRun) {
-    const config = readConnectionConfig();
-    const controlPort = config.chains[chain].control_port;
-    let hyperionIndexer = `ws://localhost:${controlPort}`;
-    if (host) {
-        hyperionIndexer = `ws://${host}:${controlPort}`;
-    }
-    const controller = new WebSocket(hyperionIndexer + '/local');
-
-    // Function to send Chunk
-    async function sendChunk(chunk): Promise<void> {
-        return new Promise((resolve, reject) => {
-            const payload = {
-                event: 'fill_missing_blocks',
-                data: chunk,
-            };
-            controller.send(JSON.stringify(payload));
-
-            // Wait repair_completed confirmation
-            controller.once('message', (data) => {
-                const parsed = JSON.parse(data.toString());
-                if (parsed.event === 'repair_completed') {
-                    // console.log(`Hyperion repair completed for chunk!`);
-                    resolve();
-                }
-            });
-        });
-    }
-
-    controller.on('open', async () => {
-        console.log('Connected to Hyperion Controller');
-        const parsedFile = JSON.parse(readFileSync(file).toString());
-        const chunkSize = 20; // Chunk size
-        const totalLines = parsedFile.length;
-
-        if (!dryRun) {
-            let completedLines = 0;
-
-            for (let i = 0; i < parsedFile.length; i += chunkSize) {
-                const chunk = parsedFile.slice(i, i + chunkSize);
-                await sendChunk(chunk);
-
-                // Atualizar o progresso com base no número total de linhas
-                completedLines += chunk.length;
-                const progress = (completedLines / totalLines) * 100;
-                const progressBar = Array(Math.round(progress / 2))
-                    .fill('#')
-                    .join('');
-                process.stdout.clearLine(0); // Limpar a linha anterior
-                process.stdout.cursorTo(0); // Mover o cursor para o início da linha
-                process.stdout.write(
-                    `Progress: [${progressBar}] ${progress.toFixed(2)}%`
-                );
-            }
-            console.log(); // Pule para a próxima linha após a conclusão
-            controller.close();
-        } else {
-            console.log('Dry run, skipping repair');
-            controller.close();
-        }
-    });
-
-    controller.on('close', () => {
-        console.log('Disconnected from Hyperion Controller');
-    });
-
-    controller.on('error', (err) => {
-        console.log(err);
-    });
-}
-
-function viewFile(file: string) {
-    const data = readFileSync(file, 'utf8');
-    const parsed = JSON.parse(data);
-    console.table(parsed);
-}
-
-// Commander Logic
-
-program
-    .name('Hyperion Repair CLI')
-    .description('CLI to find and repair forked and missing blocks on Hyperion')
-    .version('0.2.2');
-
-program
-    .command('scan <chain>')
-    .description('scan for forked blocks')
-    .option('-d, --dry', 'dry-run, do not delete or repair blocks')
-    .option('-o, --out-file <file>', 'forked-blocks.json output file')
-    .option('-f, --first <number>', 'initial block to start validation')
-    .option('-l, --last <number>', 'last block to validate')
-    .option('-b, --batch <number>', 'batch size to process')
-    .action(scanChain);
-
-program
-    .command('repair <chain> <file>')
-    .description('repair forked blocks')
-    .option('-h, --host <host>', 'Hyperion local control api')
-    .option('-d, --dry', 'dry-run, do not delete or repair blocks')
-    .option('-t, --check-tasks', 'check for running tasks')
-    .action(repairChain);
-
-program
-    .command('fill-missing <chain> <file>')
-    .description('write missing blocks')
-    .option('-h, --host <host>', 'Hyperion local control api')
-    .option('-d, --dry', 'dry-run, do not delete or repair blocks')
-    .action(repairMissing);
-
-program
-    .command('view <file>')
-    .description('view forked blocks')
-    .action(viewFile);
-
-program
-    .command('connect')
-    .option('-h, --host <host>', 'Hyperion local control api')
-    .action(async (args) => {
-        let hyperionIndexer = 'ws://localhost:4321';
-        let valid = false;
-        if (args.host) {
-            hyperionIndexer = args.host;
-        }
-        try {
-            const controller = new WebSocket(hyperionIndexer + '/local');
-            controller.on('open', () => {
-                valid = true;
-                controller.close();
-            });
-            controller.on('close', () => {
-                if (valid) {
-                    console.log(
-                        `✅  Hyperion Indexer Online - ${hyperionIndexer}`
-                    );
-                }
-            });
-            controller.on('error', (err) => {
-                console.log('Error:', err.message);
-                console.log(
-                    `Failed to connect on Hyperion Indexer at ${hyperionIndexer}, please use "--host ws://ADDRESS:PORT" to specify a remote indexer connection`
-                );
-            });
-        } catch (e) {
-            console.log(e);
-        }
-    });
-
-program.parse();

scripts/repair-cli/functions.js

@@ -1,82 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getBlocks = exports.getLastIndexedBlock = exports.getFirstIndexedBlock = exports.readChainConfig = exports.initESClient = exports.readConnectionConfig = void 0;
-const node_fs_1 = require("node:fs");
-const elasticsearch_1 = require("@elastic/elasticsearch");
-function readConnectionConfig() {
-    const file = (0, node_fs_1.readFileSync)('connections.json', 'utf8');
-    return JSON.parse(file);
-}
-exports.readConnectionConfig = readConnectionConfig;
-function initESClient(config) {
-    const node = `${config.elasticsearch.protocol}://${config.elasticsearch.host}`;
-    return new elasticsearch_1.Client({
-        nodes: [node],
-        auth: {
-            username: config.elasticsearch.user,
-            password: config.elasticsearch.pass
-        },
-        ssl: {
-            rejectUnauthorized: false
-        }
-    });
-}
-exports.initESClient = initESClient;
-function readChainConfig(chain) {
-    const file = (0, node_fs_1.readFileSync)(`chains/${chain}.config.json`, 'utf8');
-    return JSON.parse(file);
-}
-exports.readChainConfig = readChainConfig;
-async function getFirstIndexedBlock(client, blockIndex) {
-    const { body } = await client.search({
-        index: blockIndex,
-        size: 1,
-        body: {
-            sort: [{ block_num: { order: 'asc' } }]
-        }
-    });
-    if (body.hits.hits[0]._source) {
-        return body.hits.hits[0]._source.block_num;
-    }
-    else {
-        console.log('No blocks indexed yet');
-        process.exit();
-    }
-}
-exports.getFirstIndexedBlock = getFirstIndexedBlock;
-async function getLastIndexedBlock(client, blockIndex) {
-    const { body } = await client.search({
-        index: blockIndex,
-        size: 1,
-        body: {
-            sort: [{ block_num: { order: 'desc' } }]
-        }
-    });
-    if (body.hits.hits[0]._source) {
-        return body.hits.hits[0]._source.block_num;
-    }
-    else {
-        console.log('No blocks indexed yet');
-        process.exit();
-    }
-}
-exports.getLastIndexedBlock = getLastIndexedBlock;
-async function getBlocks(client, indexName, blocoInicial, blocoFinal, size) {
-    return await client.search({
-        index: indexName,
-        size: size,
-        body: {
-            sort: [{ block_num: { order: "desc" } }],
-            query: {
-                range: {
-                    block_num: {
-                        gte: blocoFinal,
-                        lte: blocoInicial,
-                    },
-                },
-            },
-        }
-    });
-}
-exports.getBlocks = getBlocks;
-//# sourceMappingURL=functions.js.map

scripts/repair-cli/functions.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"functions.js","sourceRoot":"","sources":["functions.ts"],"names":[],"mappings":";;;AAAA,qCAAqC;AACrC,0DAAuD;AAGvD,SAAgB,oBAAoB;IAChC,MAAM,IAAI,GAAG,IAAA,sBAAY,EAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAHD,oDAGC;AAED,SAAgB,YAAY,CAAC,MAAW;IACpC,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,aAAa,CAAC,QAAQ,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IAC/E,OAAO,IAAI,sBAAM,CAAC;QACd,KAAK,EAAE,CAAC,IAAI,CAAC;QACb,IAAI,EAAE;YACF,QAAQ,EAAE,MAAM,CAAC,aAAa,CAAC,IAAI;YACnC,QAAQ,EAAE,MAAM,CAAC,aAAa,CAAC,IAAI;SACtC;QACD,GAAG,EAAE;YACD,kBAAkB,EAAE,KAAK;SAC5B;KACJ,CAAC,CAAC;AACP,CAAC;AAZD,oCAYC;AAED,SAAgB,eAAe,CAAC,KAAa;IACzC,MAAM,IAAI,GAAG,IAAA,sBAAY,EAAC,UAAU,KAAK,cAAc,EAAE,MAAM,CAAC,CAAC;IACjE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAHD,0CAGC;AAEM,KAAK,UAAU,oBAAoB,CAAC,MAAc,EAAE,UAAkB;IACzE,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,MAAM,CAAC,MAAM,CAAwC;QACtE,KAAK,EAAE,UAAU;QACjB,IAAI,EAAE,CAAC;QACP,IAAI,EAAE;YACF,IAAI,EAAE,CAAC,EAAC,SAAS,EAAE,EAAC,KAAK,EAAE,KAAK,EAAC,EAAC,CAAC;SACtC;KACJ,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;KAC9C;SAAM;QACH,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,EAAE,CAAC;KAClB;AACL,CAAC;AAdD,oDAcC;AAEM,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,UAAkB;IACxE,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,MAAM,CAAC,MAAM,CAAwC;QACtE,KAAK,EAAE,UAAU;QACjB,IAAI,EAAE,CAAC;QACP,IAAI,EAAE;YACF,IAAI,EAAE,CAAC,EAAC,SAAS,EAAE,EAAC,KAAK,EAAE,MAAM,EAAC,EAAC,CAAC;SACvC;KACJ,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;KAC9C;SAAM;QACH,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,EAAE,CAAC;KAClB;AACL,CAAC;AAdD,kDAcC;AAEM,KAAK,UAAU,SAAS,CAAC,MAAW,EAAE,SAAiB,EAAE,YAAiB,EAAE,UAAe,EAAE,IAAS;IACzG,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC;QACvB,KAAK,EAAE,SAAS;QAChB,IAAI,EAAE,IAAI;QACV,IAAI,EAAE;YACF,IAAI,EAAE,CAAC,EAAC,SAAS,EAAE,EAAC,KAAK,EAAE,MAAM,EAAC,EAAC,CAAC;YACpC,KAAK,EAAE;gBACH,KAAK,EAAE;oBACH,SAAS,EAAE;wBACP,GAAG,EAAE,UAAU;wBACf,GAAG,EAAE,YAAY;qBACpB;iBACJ;aACJ;SACJ;KACJ,CAAC,CAAC;AACP,CAAC;AAhBD,8BAgBC"}

scripts/repair-cli/functions.ts

@@ -1,78 +0,0 @@
-import {readFileSync} from "node:fs";
-import {Client, estypes} from "@elastic/elasticsearch";
-import {HyperionBlock} from "./interfaces.js";
-
-export function readConnectionConfig() {
-    const file = readFileSync('connections.json', 'utf8');
-    return JSON.parse(file);
-}
-
-export function initESClient(config: any) {
-    const node = `${config.elasticsearch.protocol}://${config.elasticsearch.host}`;
-    return new Client({
-        nodes: [node],
-        auth: {
-            username: config.elasticsearch.user,
-            password: config.elasticsearch.pass
-        },
-        ssl: {
-            rejectUnauthorized: false
-        }
-    });
-}
-
-export function readChainConfig(chain: string) {
-    const file = readFileSync(`chains/${chain}.config.json`, 'utf8');
-    return JSON.parse(file);
-}
-
-export async function getFirstIndexedBlock(client: Client, blockIndex: string) {
-    const {body} = await client.search<estypes.SearchResponse<HyperionBlock>>({
-        index: blockIndex,
-        size: 1,
-        body: {
-            sort: [{block_num: {order: 'asc'}}]
-        }
-    });
-    if (body.hits.hits[0]._source) {
-        return body.hits.hits[0]._source.block_num;
-    } else {
-        console.log('No blocks indexed yet');
-        process.exit();
-    }
-}
-
-export async function getLastIndexedBlock(client: Client, blockIndex: string) {
-    const {body} = await client.search<estypes.SearchResponse<HyperionBlock>>({
-        index: blockIndex,
-        size: 1,
-        body: {
-            sort: [{block_num: {order: 'desc'}}]
-        }
-    });
-    if (body.hits.hits[0]._source) {
-        return body.hits.hits[0]._source.block_num;
-    } else {
-        console.log('No blocks indexed yet');
-        process.exit();
-    }
-}
-
-export async function getBlocks(client: any, indexName: string, blocoInicial: any, blocoFinal: any, size: any) {
-    return await client.search({
-        index: indexName,
-        size: size,
-        body: {
-            sort: [{block_num: {order: "desc"}}],
-            query: {
-                range: {
-                    block_num: {
-                        gte: blocoFinal,
-                        lte: blocoInicial,
-                    },
-                },
-            },
-        }
-    });
-}
-

scripts/repair-cli/interfaces.js

@@ -1,3 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=interfaces.js.map

scripts/repair-cli/interfaces.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["interfaces.ts"],"names":[],"mappings":""}