fix

Large tx-sig fix

.gitignore

@@ -68,8 +68,8 @@ helpers/**/*.js.map
 interfaces/**/*.js
 interfaces/**/*.js.map
 
-modules/**/*.js
-modules/**/*.js.map
+#modules/**/*.js
+#modules/**/*.js.map
 
 addons/**/*.js
 addons/**/*.js.map
@@ -77,9 +77,6 @@ addons/**/*.js.map
 scripts/hpm.js
 scripts/hpm.js.map
 
-scripts/hyp-config.js
-scripts/hyp-config.js.map
-
 plugins/.state.json
 plugins/repos
 plugins/*.js
@@ -91,3 +88,12 @@ docker/rabbitmq/data
 docker/eosio/data
 
 configuration_backups
+
+scripts/hyp-config.js
+scripts/hyp-config.js.map
+
+scripts/hyp-repair.js
+scripts/hyp-repair.js.map
+
+scripts/repair-cli/*.js
+scripts/repair-cli/*.js.map

README.md

@@ -91,7 +91,7 @@ to collect action traces and state deltas. Provides data via websocket to the in
 #### 2.7 Hyperion Stream Client (optional)
 
 Web and Node.js client for real-time streaming on enabled hyperion
-providers. [Documentation](https://hyperion.docs.eosrio.io/stream_client/)
+providers. [Documentation](https://hyperion.docs.eosrio.io/dev/stream_client/)
 
 #### 2.8 Hyperion Plugins (optional)
 

api/helpers/cacheManager.ts

@@ -28,19 +28,19 @@ export class CacheManager {
         setInterval(() => {
             try {
                 // remove expired entries
-                let removeCount = 0;
+                // let removeCount = 0;
                 const now = Date.now();
                 this.v1Caches.forEach((pathCacheMap: Map<string, CachedEntry>, pathKey: string) => {
                     pathCacheMap.forEach((cache: CachedEntry, entryKey: string, map: Map<string, CachedEntry>) => {
                         if (cache.exp < now) {
                             map.delete(entryKey);
-                            removeCount++;
+                            // removeCount++;
                         }
                     });
                 });
-                if (removeCount > 0) {
-                    console.log(`${removeCount} expired cache entries removed`);
-                }
+                // if (removeCount > 0) {
+                //     console.log(`${removeCount} expired cache entries removed`);
+                // }
             } catch (e) {
                 console.log(e);
             }

api/helpers/functions.ts

@@ -2,7 +2,8 @@ import {createHash} from "crypto";
 import * as _ from "lodash";
 import {FastifyInstance, FastifyReply, FastifyRequest, FastifySchema, HTTPMethods} from "fastify";
 import got from "got";
-import {checkFilter, hLog} from "../../helpers/common_functions";
+import {checkDeltaFilter, checkFilter, hLog} from "../../helpers/common_functions";
+import {Socket} from "socket.io";
 
 const deltaQueryFields = ['code', 'table', 'scope', 'payer'];
 
@@ -15,51 +16,141 @@ export async function streamPastDeltas(fastify: FastifyInstance, socket, data) {
     deltaQueryFields.forEach(f => {
         addTermMatch(data, search_body, f);
     });
+
+    const onDemandDeltaFilters = [];
+    if (data.filters.length > 0) {
+        data.filters.forEach(f => {
+            if (f.field && f.value) {
+                if (f.field.startsWith('@') && !f.field.startsWith('data')) {
+                    const _q = {};
+                    _q[f.field] = f.value;
+                    search_body.query.bool.must.push({'term': _q});
+                } else {
+                    onDemandDeltaFilters.push(f);
+                }
+            }
+        });
+    }
+
     const responseQueue = [];
+
     let counter = 0;
+    let total = 0;
+    let totalFiltered = 0;
+    let longScroll = false;
+
+    // console.log(JSON.stringify(search_body, null, 2));
+
     const init_response = await fastify.elastic.search({
         index: fastify.manager.chain + '-delta-*',
         scroll: '30s',
-        size: 20,
+        size: fastify.manager.config.api.stream_scroll_batch || 500,
         body: search_body,
     });
+
+    const scrollLimit = fastify.manager.config.api.stream_scroll_limit;
+    if (scrollLimit && scrollLimit !== -1 && init_response.body.hits.total.value > scrollLimit) {
+        const errorMsg = `Requested ${init_response.body.hits.total.value} deltas, limit is ${scrollLimit}.`;
+        socket.emit('message', {
+            reqUUID: socket.data.reqUUID,
+            type: 'delta_trace', mode: 'history', messages: [],
+            error: errorMsg
+        });
+        return {status: false, error: errorMsg};
+    }
+
+    if (init_response.body.hits.total.value > 10000) {
+        total = init_response.body.hits.total.value;
+        longScroll = true;
+        hLog(`Attention! Long scroll (deltas) is running!`);
+    }
+
+    // emit first block
+    if (init_response.body.hits.hits.length > 0) {
+        emitTraceInit(socket, init_response.body.hits.hits[0]._source.block_num, init_response.body.hits.total.value);
+    }
+
     responseQueue.push(init_response);
+
+    let lastTransmittedBlock = 0;
+    let pendingScrollId = '';
     while (responseQueue.length) {
+        let filterCount = 0;
         const {body} = responseQueue.shift();
+        pendingScrollId = body['_scroll_id'];
+        const enqueuedMessages = [];
         counter += body['hits']['hits'].length;
+
+        for (const doc of body['hits']['hits']) {
+            let allow = false;
+            if (onDemandDeltaFilters.length > 0) {
+                allow = onDemandDeltaFilters.every(filter => {
+                    return checkDeltaFilter(filter, doc._source);
+                });
+            } else {
+                allow = true;
+            }
+            if (allow) {
+                enqueuedMessages.push(doc._source);
+            } else {
+                filterCount++;
+            }
+            // set last block
+            if (doc._source.block_num > lastTransmittedBlock) {
+                lastTransmittedBlock = doc._source.block_num;
+            }
+        }
+
+        totalFiltered += filterCount;
+
         if (socket.connected) {
-            socket.emit('message', {
-                type: 'delta_trace',
-                mode: 'history',
-                messages: body['hits']['hits'].map(doc => doc._source),
-            });
+            if (enqueuedMessages.length > 0) {
+                socket.emit('message', {
+                    reqUUID: socket.data.reqUUID,
+                    type: 'delta_trace',
+                    mode: 'history',
+                    messages: enqueuedMessages,
+                    filtered: filterCount
+                });
+            }
         } else {
             hLog('LOST CLIENT');
             break;
         }
+
+        if (longScroll) {
+            hLog(`Progress: ${counter + totalFiltered}/${total}`);
+        }
+
         if (body['hits'].total.value === counter) {
-            hLog(`${counter} past deltas streamed to ${socket.id}`);
+            hLog(`${counter} past deltas streamed to ${socket.id} (${totalFiltered} filtered)`);
             break;
         }
 
         const next_response = await fastify.elastic.scroll({
-            body: {
-                scroll_id: body['_scroll_id'],
-                scroll: '30s'
-            }
+            body: {scroll_id: body['_scroll_id'], scroll: '30s'}
         });
         responseQueue.push(next_response);
     }
+
+    // destroy scroll context
+    await fastify.elastic.clearScroll({scroll_id: pendingScrollId});
+    return {status: true, lastTransmittedBlock};
+}
+
+export function emitTraceInit(socket: Socket, firstBlock: number, totalResults: number) {
+    socket.emit('message', {
+        reqUUID: socket.data.reqUUID,
+        type: 'trace_init',
+        mode: 'history',
+        first_block: firstBlock,
+        results: totalResults
+    });
 }
 
 export async function streamPastActions(fastify: FastifyInstance, socket, data) {
-    const search_body = {
-        query: {bool: {must: []}},
-        sort: {global_sequence: 'asc'},
-    };
-
+    const search_body = {query: {bool: {must: []}}, sort: {global_sequence: 'asc'}};
     await addBlockRangeOpts(data, search_body, fastify);
-
     if (data.account !== '') {
         search_body.query.bool.must.push({
             bool: {
@@ -96,18 +187,50 @@ export async function streamPastActions(fastify: FastifyInstance, socket, data)
 
     const responseQueue = [];
     let counter = 0;
+    let total = 0;
+    let totalFiltered = 0;
+    let longScroll = false;
 
     const init_response = await fastify.elastic.search({
         index: fastify.manager.chain + '-action-*',
         scroll: '30s',
-        size: 20,
+        size: fastify.manager.config.api.stream_scroll_batch || 500,
         body: search_body,
     });
+
+    const scrollLimit = fastify.manager.config.api.stream_scroll_limit;
+    if (scrollLimit && scrollLimit !== -1 && init_response.body.hits.total.value > scrollLimit) {
+        const errorMsg = `Requested ${init_response.body.hits.total.value} actions, limit is ${scrollLimit}.`;
+        socket.emit('message', {
+            reqUUID: socket.data.reqUUID,
+            type: 'action_trace', mode: 'history', messages: [],
+            error: `Requested ${init_response.body.hits.total.value} actions, limit is ${scrollLimit}.`
+        });
+        return {status: false, error: errorMsg};
+    }
+
+    if (init_response.body.hits.total.value > 10000) {
+        total = init_response.body.hits.total.value;
+        longScroll = true;
+        hLog(`Attention! Long scroll (actions) is running!`);
+    }
+
+    // emit first block
+    if (init_response.body.hits.hits.length > 0) {
+        emitTraceInit(socket, init_response.body.hits.hits[0]._source.block_num, init_response.body.hits.total.value);
+    }
+
     responseQueue.push(init_response);
+
+    let lastTransmittedBlock = 0;
+    let pendingScrollId = '';
     while (responseQueue.length) {
+        let filterCount = 0;
         const {body} = responseQueue.shift();
+        pendingScrollId = body['_scroll_id'];
         const enqueuedMessages = [];
         counter += body['hits']['hits'].length;
+
         for (const doc of body['hits']['hits']) {
             let allow = false;
             if (onDemandFilters.length > 0) {
@@ -119,32 +242,55 @@ export async function streamPastActions(fastify: FastifyInstance, socket, data)
             }
             if (allow) {
                 enqueuedMessages.push(doc._source);
+            } else {
+                filterCount++;
+            }
+            // set last block
+            if (doc._source.block_num > lastTransmittedBlock) {
+                lastTransmittedBlock = doc._source.block_num;
             }
         }
+
+        totalFiltered += filterCount;
+
         if (socket.connected) {
-            socket.emit('message', {type: 'action_trace', mode: 'history', messages: enqueuedMessages});
+            if (enqueuedMessages.length > 0) {
+                socket.emit('message', {
+                    reqUUID: socket.data.reqUUID,
+                    type: 'action_trace',
+                    mode: 'history',
+                    messages: enqueuedMessages,
+                    filtered: filterCount
+                });
+            }
         } else {
             hLog('LOST CLIENT');
             break;
         }
+
+        if (longScroll) {
+            hLog(`Progress: ${counter + totalFiltered}/${total}`);
+        }
+
         if (body['hits'].total.value === counter) {
-            hLog(`${counter} past actions streamed to ${socket.id}`);
+            hLog(`${counter} past actions streamed to ${socket.id} (${totalFiltered} filtered)`);
             break;
         }
-        if (init_response.body.hits.total.value < 1000) {
-            const next_response = await fastify.elastic.scroll({
-                body: {scroll_id: body['_scroll_id'], scroll: '30s'}
-            });
-            responseQueue.push(next_response);
-        } else {
-            hLog('Request too large!');
-            socket.emit('message', {type: 'action_trace', mode: 'history', messages: []});
-        }
+
+        const next_response = await fastify.elastic.scroll({
+            body: {scroll_id: body['_scroll_id'], scroll: '30s'}
+        });
+        responseQueue.push(next_response);
     }
+
+    // destroy scroll context
+    await fastify.elastic.clearScroll({scroll_id: pendingScrollId});
+    return {status: true, lastTransmittedBlock};
+
 }
 
 export function addTermMatch(data, search_body, field) {
-    if (data[field] !== '*' && data[field] !== '') {
+    if (data[field] && data[field] !== '*' && data[field] !== '') {
         const termQuery = {};
         termQuery[field] = data[field];
         search_body.query.bool.must.push({'term': termQuery});
@@ -217,6 +363,8 @@ export function extendResponseSchema(responseProps: any) {
         cached: {type: "boolean"},
         hot_only: {type: "boolean"},
         lib: {type: "number"},
+        last_indexed_block: {type: "number"},
+        last_indexed_block_time: {type: "string"},
         total: {
             type: "object",
             properties: {
@@ -367,7 +515,8 @@ function bigint2Milliseconds(input: bigint) {
 
 const defaultRouteCacheMap = {
     get_resource_usage: 3600,
-    get_creator: 3600 * 24
+    get_creator: 3600 * 24,
+    health: 10
 }
 
 export async function timedQuery(
@@ -384,16 +533,31 @@ export async function timedQuery(
         request.method === 'POST' ? request.body : request.query
     );
 
+    let lastIndexedBlockNumber = null;
+    let lastIndexedBlockTime = null;
+    const lastIndexedBlock = await fastify.redis.get(`${fastify.manager.chain}:last_idx_block`);
+    if (lastIndexedBlock) {
+        const arr = lastIndexedBlock.split("@");
+        if (arr.length === 2) {
+            lastIndexedBlockNumber = parseInt(arr[0], 10);
+            lastIndexedBlockTime = arr[1];
+        }
+    }
+
     if (cachedResponse && !request.query["ignoreCache"]) {
         // add cached query time
         cachedResponse['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
+        if (lastIndexedBlock) {
+            cachedResponse['last_indexed_block'] = lastIndexedBlockNumber;
+            cachedResponse['last_indexed_block_time'] = lastIndexedBlockTime;
+        }
         return cachedResponse;
     }
 
     // call query function
     const response = await queryFunction(fastify, request);
 
-    // save response to cash
+    // save response to cache
     if (hash) {
         let EX = null;
         if (defaultRouteCacheMap[route]) {
@@ -405,6 +569,10 @@ export async function timedQuery(
     // add normal query time
     if (response) {
         response['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
+        if (lastIndexedBlock) {
+            response['last_indexed_block'] = lastIndexedBlockNumber;
+            response['last_indexed_block_time'] = lastIndexedBlockTime;
+        }
         return response;
     } else {
         return {};
@@ -416,7 +584,7 @@ export function chainApiHandler(fastify: FastifyInstance) {
         // check cache
         const [cachedData, hash, path] = fastify.cacheManager.getCachedData(request);
         if (cachedData) {
-            // console.log('cache hit:', path, hash);
+            console.log('cache hit:', path, hash);
             reply.headers({'hyperion-cached': true}).send(cachedData);
         } else {
             // call actual request

api/plugins/fastify-eosjs.ts

@@ -3,17 +3,16 @@ import {Api} from "eosjs/dist";
 import fp from "fastify-plugin";
 
 export default fp(async (fastify: FastifyInstance, options: FastifyPluginOptions): Promise<void> => {
-	const rpc = fastify.manager.nodeosJsonRPC;
-	const chain_data = await rpc.get_info();
-	const api = new Api({
-		rpc,
-		signatureProvider: null,
-		chainId: chain_data.chain_id,
-		textDecoder: new TextDecoder(),
-		textEncoder: new TextEncoder(),
-	});
-	fastify.decorate('eosjs', {api, rpc});
+    const rpc = fastify.manager.nodeosJsonRPC;
+    const api = new Api({
+        rpc,
+        signatureProvider: null,
+        chainId: options.chain_id,
+        textDecoder: new TextDecoder(),
+        textEncoder: new TextEncoder(),
+    });
+    fastify.decorate('eosjs', {api, rpc});
 }, {
-	fastify: '>=2.0.0',
-	name: 'fastify-eosjs'
+    fastify: '>=2.0.0',
+    name: 'fastify-eosjs'
 });

api/routes.ts

@@ -62,7 +62,7 @@ export function registerRoutes(server: FastifyInstance) {
     // chain api redirects
     addRoute(server, 'v1-chain', '/v1/chain');
 
-	// other v1 requests
+    // other v1 requests
     server.route({
         url: '/v1/chain/*',
         method: ["GET", "POST"],
@@ -95,6 +95,25 @@ export function registerRoutes(server: FastifyInstance) {
         }
     });
 
+    // global onRequest hook to collect hourly statistics for each path on redis
+    server.addHook('onResponse', (request: FastifyRequest, reply: FastifyReply, done) => {
+        const path = request.url.split('?')[0].replace(/\/$/, '');
+        const now = new Date();
+        const ts = now.getTime();
+        // normalized time in hours
+        const factor = 1000 * 60 * 60;
+        const ts_hour_unix = Math.floor(ts / factor) * factor;
+        const key = `stats:${server.manager.chain}:H:${ts_hour_unix}`;
+        // console.log(reply.statusCode, key, new Date(ts_hour_unix));
+        server.redis.multi()
+            .incr(`stats:${server.manager.chain}:total:${reply.statusCode}:${path}`)
+            .zincrby(key, 1, `[${reply.statusCode}]${path}`)
+            .expire(key, 60 * 60 * 24 * 7)
+            .exec()
+            .catch(console.log);
+        done();
+    });
+
     server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
         console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
         done();

api/routes/v1-chain/abi_bin_to_json/index.ts

@@ -9,11 +9,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         {
             "code": {$ref: 'AccountName#'},
             "action": {$ref: 'AccountName#'},
-            "binargs": {
-                "type": "string",
-                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
-                "title": "Hex"
-            }
+            "binargs": {"type": "string"}
         },
         ["code", "action", "binargs"]
     );

api/routes/v1-chain/abi_json_to_bin/index.ts

@@ -7,13 +7,14 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Convert JSON object to binary',
         {
-            "binargs": {
-                "type": "string",
-                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
-                "title": "Hex"
+            "code": {$ref: 'AccountName#'},
+            "action": {$ref: 'AccountName#'},
+            "args": {
+                type: 'object',
+                additionalProperties: true
             }
         },
-        ["binargs"]
+        ["code", "action", "args"]
     );
     next();
 }

api/routes/v1-chain/get_table_rows/index.ts

@@ -37,7 +37,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                 "type": "string"
             }
         },
-        ["code", "table"]
+        ["code", "table", "scope"]
     );
     next();
 }

api/routes/v1-history/get_actions/get_actions.ts

@@ -6,6 +6,7 @@ import * as AbiEOS from "@eosrio/node-abieos";
 import {ApiResponse} from "@elastic/elasticsearch";
 import {TextDecoder, TextEncoder} from "util";
 import {JsonRpc} from "eosjs/dist";
+import {terms} from "../../v2-history/get_actions/definitions";
 
 const abi_remapping = {
     "_Bool": "bool"
@@ -144,7 +145,6 @@ async function getContractAtBlock(esClient, rpc, chain, accountName: string, blo
 }
 
 
-const terms = ["notified", "act.authorization.actor"];
 const extendedActions = new Set(["transfer", "newaccount", "updateauth"]);
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
@@ -182,8 +182,8 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
     }
     pos = parseInt(reqBody.pos || 0, 10);
     offset = parseInt(reqBody.offset || 20, 10);
-    let from, size;
-    from = size = 0;
+    let from = 0;
+    let size = 0;
     if (pos === -1) {
         if (offset < 0) {
             from = 0;
@@ -272,6 +272,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
         }
     };
+    // console.log(JSON.stringify(esOpts, null, 2));
     const pResults = await Promise.all([fastify.eosjs.rpc.get_info(), fastify.elastic['search'](esOpts)]);
     const results = pResults[1];
     const response = {

api/routes/v1-history/get_transaction/get_transaction.ts

@@ -1,265 +1,190 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
-import {createHash} from "crypto";
-import flatstr from 'flatstr';
 import {GetInfoResult} from "eosjs/dist/eosjs-rpc-interfaces";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
-	if (typeof request.body === 'string') {
-		request.body = JSON.parse(request.body)
-	}
-	const body: any = request.body;
-	const redis = fastify.redis;
-	const trxId = body.id.toLowerCase();
-	const conf = fastify.manager.config;
-	const cachedData = await redis.hgetall('trx_' + trxId);
 
-	const response: any = {
-		"id": body.id,
-		"trx": {
-			"receipt": {
-				"status": "executed",
-				"cpu_usage_us": 0,
-				"net_usage_words": 0,
-				"trx": [1, {
-					"signatures": "",
-					"compression": "none",
-					"packed_context_free_data": "",
-					"packed_trx": ""
-				}]
-			},
-			"trx": {}
-		},
-		"block_num": 0,
-		"block_time": "",
-		"last_irreversible_block": undefined,
-		"traces": []
-	};
+    if (request.method === 'POST' && !request.body) {
+        throw new Error("missing POST body");
+    }
 
-	let hits;
+    if (typeof request.body === 'string') {
+        request.body = JSON.parse(request.body)
+    }
+    const body: any = request.body;
+    const redis = fastify.redis;
+    const trxId = body.id.toLowerCase();
 
-	// build get_info request with caching
-	const $getInfo = new Promise<GetInfoResult>(resolve => {
-		const key = `${fastify.manager.chain}_get_info`;
-		fastify.redis.get(key).then(value => {
-			if (value) {
-				resolve(JSON.parse(value));
-			} else {
-				fastify.eosjs.rpc.get_info().then(value1 => {
-					fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
-					resolve(value1);
-				}).catch((reason) => {
-					console.log(reason);
-					response.error = 'failed to get last_irreversible_block_num'
-					resolve(null);
-				});
-			}
-		});
-	});
+    const conf = fastify.manager.config;
+    const cachedData = await redis.hgetall('trx_' + trxId);
 
-	// reconstruct hits from cached data
-	if (cachedData && Object.keys(cachedData).length > 0) {
-		const gsArr = [];
-		for (let cachedDataKey in cachedData) {
-			gsArr.push(cachedData[cachedDataKey]);
-		}
-		gsArr.sort((a, b) => {
-			return a.global_sequence - b.global_sequence;
-		});
-		hits = gsArr.map(value => {
-			return {
-				_source: JSON.parse(value)
-			};
-		});
-		const promiseResults = await Promise.all([
-			redis.ttl('trx_' + trxId),
-			$getInfo
-		]);
-		response.cache_expires_in = promiseResults[0];
-		response.last_irreversible_block = promiseResults[1].last_irreversible_block_num;
-	}
+    const response: any = {
+        "id": body.id,
+        "trx": {
+            "receipt": {
+                "status": "executed",
+                "cpu_usage_us": 0,
+                "net_usage_words": 0,
+                "trx": [1, {
+                    "signatures": "",
+                    "compression": "none",
+                    "packed_context_free_data": "",
+                    "packed_trx": ""
+                }]
+            },
+            "trx": {}
+        },
+        "block_num": 0,
+        "block_time": "",
+        "last_irreversible_block": undefined,
+        "traces": []
+    };
 
-	// search on ES if cache is not present
-	if (!hits) {
-		const _size = conf.api.limits.get_trx_actions || 100;
-		const blockHint = parseInt(body.block_num_hint, 10);
-		let indexPattern = '';
-		if (blockHint) {
-			const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
-			indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
-		} else {
-			indexPattern = fastify.manager.chain + '-action-*';
-		}
-		let pResults;
-		try {
+    let hits;
 
-			// build search request
-			const $search = fastify.elastic.search({
-				index: indexPattern,
-				size: _size,
-				body: {
-					query: {bool: {must: [{term: {trx_id: trxId}}]}},
-					sort: {global_sequence: "asc"}
-				}
-			});
+    // build get_info request with caching
+    const $getInfo = new Promise<GetInfoResult>(resolve => {
+        const key = `${fastify.manager.chain}_get_info`;
+        fastify.redis.get(key).then(value => {
+            if (value) {
+                resolve(JSON.parse(value));
+            } else {
+                fastify.eosjs.rpc.get_info().then(value1 => {
+                    fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
+                    resolve(value1);
+                }).catch((reason) => {
+                    console.log(reason);
+                    response.error = 'failed to get last_irreversible_block_num'
+                    resolve(null);
+                });
+            }
+        });
+    });
 
-			// execute in parallel
-			pResults = await Promise.all([$getInfo, $search]);
-		} catch (e) {
-			console.log(e.message);
-			if (e.meta.statusCode === 404) {
-				return response;
-			}
-		}
-		hits = pResults[1]['body']['hits']['hits'];
-		response.last_irreversible_block = pResults[0].last_irreversible_block_num;
-	}
+    // reconstruct hits from cached data
+    if (cachedData && Object.keys(cachedData).length > 0) {
+        const gsArr = [];
+        for (let cachedDataKey in cachedData) {
+            gsArr.push(cachedData[cachedDataKey]);
+        }
+        gsArr.sort((a, b) => {
+            return a.global_sequence - b.global_sequence;
+        });
+        hits = gsArr.map(value => {
+            return {
+                _source: JSON.parse(value)
+            };
+        });
+        const promiseResults = await Promise.all([
+            redis.ttl('trx_' + trxId),
+            $getInfo
+        ]);
+        response.cache_expires_in = promiseResults[0];
+        response.last_irreversible_block = promiseResults[1].last_irreversible_block_num;
+    }
+
+    // search on ES if cache is not present
+    if (!hits) {
+        const _size = conf.api.limits.get_trx_actions || 100;
+        const blockHint = parseInt(body.block_num_hint, 10);
+        let indexPattern = '';
+        if (blockHint) {
+            const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
+            indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
+        } else {
+            indexPattern = fastify.manager.chain + '-action-*';
+        }
+        let pResults;
+        try {
+
+            // build search request
+            const $search = fastify.elastic.search({
+                index: indexPattern,
+                size: _size,
+                body: {
+                    query: {bool: {must: [{term: {trx_id: trxId}}]}},
+                    sort: {global_sequence: "asc"}
+                }
+            });
+
+            // execute in parallel
+            pResults = await Promise.all([$getInfo, $search]);
+        } catch (e) {
+            console.log(e.message);
+            if (e.meta.statusCode === 404) {
+                return response;
+            }
+        }
+        hits = pResults[1]['body']['hits']['hits'];
+        response.last_irreversible_block = pResults[0].last_irreversible_block_num;
+    }
 
 
-	if (hits.length > 0) {
-		const actions = hits;
-		response.trx.trx = {
-			"expiration": "",
-			"ref_block_num": 0,
-			"ref_block_prefix": 0,
-			"max_net_usage_words": 0,
-			"max_cpu_usage_ms": 0,
-			"delay_sec": 0,
-			"context_free_actions": [],
-			"actions": [],
-			"transaction_extensions": [],
-			"signatures": [],
-			"context_free_data": []
-		};
-		let traces = {};
-		let seqNum = 0;
-		for (let action of actions) {
-			action = action._source;
-			mergeActionMeta(action);
-
-			// action.act['hex_data'] = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
-
-			if (action.parent === 0) {
-				response.trx.trx.actions.push(action.act);
-			}
-			response.block_num = action.block_num;
-			response.block_time = action['@timestamp'];
-			seqNum += 1;
-			let trace = {
-				receipt: {
-					receiver: action.act.account,
-					global_sequence: String(action.global_sequence),
-					auth_sequence: [
-						action.act.authorization[0].actor,
-						seqNum
-					],
-					act_digest: '',
-					recv_sequence: seqNum,
-					code_sequence: 1,
-					abi_sequence: 1
-				},
-				act: action.act,
-				account_ram_deltas: action.account_ram_deltas || [],
-				context_free: false,
-				block_num: action.block_num,
-				block_time: action['@timestamp'],
-				console: "",
-				elapsed: 0,
-				except: null,
-				inline_traces: [],
-				producer_block_id: "",
-				trx_id: body.id,
-				notified: action.notified
-			};
-			let hash = createHash('sha256');
-			hash.update(JSON.stringify(action.act));
-			trace.receipt.act_digest = hash.digest('hex');
-			traces[action.global_sequence] = trace;
-		}
-
-		actions.forEach(action => {
-			action = action._source;
-			for (let i = 0; i < traces[action.global_sequence].notified.length; i++) {
-				if (traces[action.global_sequence].notified[i] === action.act.account) {
-					traces[action.global_sequence].notified.splice(i, 1);
-					break;
-				}
-			}
-			if (action.parent !== 0 && action.parent) {
-				if (traces[action.parent]) {
-					for (let i = 0; i < traces[action.parent].notified.length; i++) {
-						if (traces[action.parent].notified[i] === action.act.account) {
-							traces[action.parent].notified.splice(i, 1);
-							break;
-						}
-					}
-					traces[action.parent].inline_traces.push(traces[action.global_sequence]);
-				}
-			}
-		});
-
-		actions.forEach(action => {
-			action = action._source;
-			response.traces.push(traces[action.global_sequence]);
-			if (traces[action.global_sequence] && traces[action.global_sequence].notified) {
-				traces[action.global_sequence].notified.forEach((note, index) => {
-					seqNum += 1;
-					let trace = {
-						receipt: {
-							receiver: note,
-							global_sequence: String(action.global_sequence + index + 1),
-							auth_sequence: [
-								action.act.authorization[0].actor,
-								seqNum
-							],
-							act_digest: traces[action.global_sequence].receipt.act_digest,
-							recv_sequence: seqNum,
-							code_sequence: 1,
-							abi_sequence: 1
-						},
-						account_ram_deltas: action.account_ram_deltas || [],
-						act: action.act,
-						block_num: action.block_num,
-						block_time: action['@timestamp'],
-						console: "",
-						context_free: false,
-						elapsed: 0,
-						except: null,
-						inline_traces: [],
-						producer_block_id: "",
-						trx_id: body.id,
-					};
-					traces[action.global_sequence].inline_traces.unshift(trace);
-					response.traces.push(trace);
-				});
-				delete traces[action.global_sequence].notified;
-			}
-		});
-	} else {
-		const errmsg = "Transaction " + body.id.toLowerCase() + " not found in history and no block hint was given";
-		return {
-			code: 500,
-			message: "Internal Service Error",
-			error: {
-				code: 3040011,
-				name: "tx_not_found",
-				what: "The transaction can not be found",
-				details: [
-					{
-						message: errmsg,
-						file: "",
-						line_number: 1,
-						method: "get_transaction"
-					}
-				]
-			}
-		}
-	}
-	return response;
+    if (hits.length > 0) {
+        const actions = hits;
+        response.trx.trx = {
+            "expiration": "",
+            "ref_block_num": 0,
+            "ref_block_prefix": 0,
+            "max_net_usage_words": 0,
+            "max_cpu_usage_ms": 0,
+            "delay_sec": 0,
+            "context_free_actions": [],
+            "actions": [],
+            "transaction_extensions": [],
+            "signatures": [],
+            "context_free_data": []
+        };
+        for (let action of actions) {
+            action = action._source;
+            mergeActionMeta(action);
+            response.block_num = action.block_num;
+            response.block_time = action['@timestamp'];
+            for (const receipt of action.receipts) {
+                if (action.act_digest) {
+                    receipt.act_digest = action.act_digest.toLowerCase();
+                }
+                response.traces.push({
+                    receipt: receipt,
+                    act: action.act,
+                    account_ram_deltas: action.account_ram_deltas || [],
+                    context_free: false,
+                    block_num: action.block_num,
+                    block_time: action['@timestamp'],
+                    console: "",
+                    elapsed: 0,
+                    except: null,
+                    inline_traces: [],
+                    producer_block_id: "",
+                    trx_id: body.id
+                });
+            }
+        }
+    } else {
+        const errmsg = "Transaction " + body.id.toLowerCase() + " not found in history and no block hint was given";
+        return {
+            code: 500,
+            message: "Internal Service Error",
+            error: {
+                code: 3040011,
+                name: "tx_not_found",
+                what: "The transaction can not be found",
+                details: [
+                    {
+                        message: errmsg,
+                        file: "",
+                        line_number: 1,
+                        method: "get_transaction"
+                    }
+                ]
+            }
+        }
+    }
+    return response;
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply) => {
-		reply.send(await timedQuery(getTransaction, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getTransaction, fastify, request, route));
+    }
 }

api/routes/v2-history/get_actions/definitions.ts

@@ -1,5 +1,6 @@
 export const terms = [
     "notified",
+    "receipts.receiver",
     "act.authorization.actor"
 ];
 
@@ -11,12 +12,19 @@ export const extendedActions = new Set([
     "buyrambytes",
     "delegatebw",
     "undelegatebw",
-    "voteproducer"
+    "voteproducer",
+    'statement',
+    'decision',
+    'batch',
+    'act',
+    'exec',
+    'votefor',
+    'voteagainst'
 ]);
 
 export const primaryTerms = [
-    "notified",
     "block_num",
+    "block_id",
     "global_sequence",
     "producer",
     "@timestamp",
@@ -24,5 +32,10 @@ export const primaryTerms = [
     "action_ordinal",
     "cpu_usage_us",
     "net_usage_words",
-    "trx_id"
+    "trx_id",
+    "receipts.receiver",
+    "receipts.global_sequence",
+    "receipts.recv_sequence",
+    "receipts.auth_sequence.account",
+    "receipts.auth_sequence.sequence"
 ];

api/routes/v2-history/get_actions/functions.ts

@@ -1,236 +1,266 @@
 import {primaryTerms, terms} from "./definitions";
 
 export function addSortedBy(query, queryBody, sort_direction) {
-	if (query['sortedBy']) {
-		const opts = query['sortedBy'].split(":");
-		const sortedByObj = {};
-		sortedByObj[opts[0]] = opts[1];
-		queryBody['sort'] = sortedByObj;
-	} else {
-		queryBody['sort'] = {
-			"global_sequence": sort_direction
-		};
-	}
+    if (query['sortedBy']) {
+        const opts = query['sortedBy'].split(":");
+        const sortedByObj = {};
+        sortedByObj[opts[0]] = opts[1];
+        queryBody['sort'] = sortedByObj;
+    } else {
+        queryBody['sort'] = {
+            "global_sequence": sort_direction
+        };
+    }
 }
 
 export function processMultiVars(queryStruct, parts, field) {
-	const must = [];
-	const mustNot = [];
+    const must = [];
+    const mustNot = [];
 
-	parts.forEach(part => {
-		if (part.startsWith("!")) {
-			mustNot.push(part.replace("!", ""));
-		} else {
-			must.push(part);
-		}
-	});
+    parts.forEach(part => {
+        if (part.startsWith("!")) {
+            mustNot.push(part.replace("!", ""));
+        } else {
+            must.push(part);
+        }
+    });
 
-	if (must.length > 1) {
-		queryStruct.bool.must.push({
-			bool: {
-				should: must.map(elem => {
-					const _q = {};
-					_q[field] = elem;
-					return {term: _q}
-				})
-			}
-		});
-	} else if (must.length === 1) {
-		const mustQuery = {};
-		mustQuery[field] = must[0];
-		queryStruct.bool.must.push({term: mustQuery});
-	}
+    if (must.length > 1) {
+        queryStruct.bool.must.push({
+            bool: {
+                should: must.map(elem => {
+                    const _q = {};
+                    _q[field] = elem;
+                    return {term: _q}
+                })
+            }
+        });
+    } else if (must.length === 1) {
+        const mustQuery = {};
+        mustQuery[field] = must[0];
+        queryStruct.bool.must.push({term: mustQuery});
+    }
 
-	if (mustNot.length > 1) {
-		queryStruct.bool.must_not.push({
-			bool: {
-				should: mustNot.map(elem => {
-					const _q = {};
-					_q[field] = elem;
-					return {term: _q}
-				})
-			}
-		});
-	} else if (mustNot.length === 1) {
-		const mustNotQuery = {};
-		mustNotQuery[field] = mustNot[0].replace("!", "");
-		queryStruct.bool.must_not.push({term: mustNotQuery});
-	}
+    if (mustNot.length > 1) {
+        queryStruct.bool.must_not.push({
+            bool: {
+                should: mustNot.map(elem => {
+                    const _q = {};
+                    _q[field] = elem;
+                    return {term: _q}
+                })
+            }
+        });
+    } else if (mustNot.length === 1) {
+        const mustNotQuery = {};
+        mustNotQuery[field] = mustNot[0].replace("!", "");
+        queryStruct.bool.must_not.push({term: mustNotQuery});
+    }
 }
 
 function addRangeQuery(queryStruct, prop, pkey, query) {
-	const _termQuery = {};
-	const parts = query[prop].split("-");
-	_termQuery[pkey] = {
-		"gte": parts[0],
-		"lte": parts[1]
-	};
-	queryStruct.bool.must.push({range: _termQuery});
+    const _termQuery = {};
+    const parts = query[prop].split("-");
+    _termQuery[pkey] = {
+        "gte": parts[0],
+        "lte": parts[1]
+    };
+    queryStruct.bool.must.push({range: _termQuery});
 }
 
 export function applyTimeFilter(query, queryStruct) {
-	if (query['after'] || query['before']) {
-		let _lte = "now";
-		let _gte = "0";
-		if (query['before']) {
-			try {
-				_lte = new Date(query['before']).toISOString();
-			} catch (e) {
-				throw new Error(e.message + ' [before]');
-			}
-		}
-		if (query['after']) {
-			try {
-				_gte = new Date(query['after']).toISOString();
-			} catch (e) {
-				throw new Error(e.message + ' [after]');
-			}
-		}
-		if (!queryStruct.bool['filter']) {
-			queryStruct.bool['filter'] = [];
-		}
-		queryStruct.bool['filter'].push({
-			range: {
-				"@timestamp": {
-					"gte": _gte,
-					"lte": _lte
-				}
-			}
-		});
-	}
+    if (query['after'] || query['before']) {
+
+        if (query['after']?.includes(' ')) {
+            query['after'] = query['after'].replace(' ', 'T');
+        }
+
+        if (query['before']?.includes(' ')) {
+            query['before'] = query['before'].replace(' ', 'T');
+        }
+
+        if (query['after']?.includes('T') || query['before']?.includes('T')) {
+            let _lte = "now";
+            let _gte = "0";
+            if (query['before']) {
+                try {
+                    _lte = new Date(query['before']).toISOString();
+                } catch (e) {
+                    throw new Error(e.message + ' [before]');
+                }
+            }
+            if (query['after']) {
+                try {
+                    _gte = new Date(query['after']).toISOString();
+                } catch (e) {
+                    throw new Error(e.message + ' [after]');
+                }
+            }
+            if (!queryStruct.bool['filter']) {
+                queryStruct.bool['filter'] = [];
+            }
+            queryStruct.bool['filter'].push({
+                range: {
+                    "@timestamp": {
+                        "gte": _gte,
+                        "lte": _lte
+                    }
+                }
+            });
+        } else {
+            // search by block number
+            const rangeObj = {
+                range: {
+                    block_num: {}
+                }
+            };
+            if (parseInt(query['after']) > 0) {
+                rangeObj.range.block_num['gte'] = query['after'];
+            }
+            if (parseInt(query['before']) > 0) {
+                rangeObj.range.block_num['lte'] = query['before'];
+            }
+            if (Object.keys(rangeObj.range.block_num).length > 0) {
+                if (!queryStruct.bool['filter']) {
+                    queryStruct.bool['filter'] = [];
+                }
+                queryStruct.bool['filter'].push(rangeObj);
+            }
+        }
+    }
 }
 
 export function applyGenericFilters(query, queryStruct, allowedExtraParams: Set<string>) {
-	for (const prop in query) {
-		if (Object.prototype.hasOwnProperty.call(query, prop)) {
-			const pair = prop.split(".");
-			if (pair.length > 1 || primaryTerms.includes(pair[0])) {
-				let pkey;
-				if (pair.length > 1 && allowedExtraParams) {
-					pkey = allowedExtraParams.has(pair[0]) ? "@" + prop : prop;
-				} else {
-					pkey = prop;
-				}
-				if (query[prop].indexOf("-") !== -1) {
-					addRangeQuery(queryStruct, prop, pkey, query);
-				} else {
-					const _qObj = {};
-					const parts = query[prop].split(",");
-					if (parts.length > 1) {
-						processMultiVars(queryStruct, parts, prop);
-					} else if (parts.length === 1) {
+    for (const prop in query) {
+        if (Object.prototype.hasOwnProperty.call(query, prop)) {
+            const pair = prop.split(".");
+            if (pair.length > 1 || primaryTerms.includes(pair[0])) {
+                let pkey;
+                if (pair.length > 1 && allowedExtraParams) {
+                    pkey = allowedExtraParams.has(pair[0]) ? "@" + prop : prop;
+                } else {
+                    pkey = prop;
+                }
+                if (query[prop].indexOf("-") !== -1) {
+                    addRangeQuery(queryStruct, prop, pkey, query);
+                } else {
+                    const _qObj = {};
+                    const parts = query[prop].split(",");
+                    if (parts.length > 1) {
+                        processMultiVars(queryStruct, parts, prop);
+                    } else if (parts.length === 1) {
 
-						// @transfer.memo special case
-						if (pkey === '@transfer.memo') {
-							_qObj[pkey] = {
-								query: parts[0]
-							};
+                        // @transfer.memo special case
+                        if (pkey === '@transfer.memo') {
+                            _qObj[pkey] = {
+                                query: parts[0]
+                            };
 
-							if (query.match_fuzziness) {
-								_qObj[pkey].fuzziness = query.match_fuzziness;
-							}
+                            if (query.match_fuzziness) {
+                                _qObj[pkey].fuzziness = query.match_fuzziness;
+                            }
 
-							if (query.match_operator) {
-								_qObj[pkey].operator = query.match_operator;
-							}
+                            if (query.match_operator) {
+                                _qObj[pkey].operator = query.match_operator;
+                            }
 
-							queryStruct.bool.must.push({
-								match: _qObj
-							});
-						} else {
-							const andParts = parts[0].split(" ");
-							if (andParts.length > 1) {
-								andParts.forEach(value => {
-									const _q = {};
-									_q[pkey] = value;
-									queryStruct.bool.must.push({term: _q});
-								});
-							} else {
-								if (parts[0].startsWith("!")) {
-									_qObj[pkey] = parts[0].replace("!", "");
-									queryStruct.bool.must_not.push({term: _qObj});
-								} else {
-									_qObj[pkey] = parts[0];
-									queryStruct.bool.must.push({term: _qObj});
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-	}
+                            queryStruct.bool.must.push({
+                                match: _qObj
+                            });
+                        } else {
+                            const andParts = parts[0].split(" ");
+                            if (andParts.length > 1) {
+                                andParts.forEach(value => {
+                                    const _q = {};
+                                    _q[pkey] = value;
+                                    queryStruct.bool.must.push({term: _q});
+                                });
+                            } else {
+                                if (parts[0].startsWith("!")) {
+                                    _qObj[pkey] = parts[0].replace("!", "");
+                                    queryStruct.bool.must_not.push({term: _qObj});
+                                } else {
+                                    _qObj[pkey] = parts[0];
+                                    queryStruct.bool.must.push({term: _qObj});
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
 }
 
 export function makeShouldArray(query) {
-	const should_array = [];
-	for (const entry of terms) {
-		const tObj = {term: {}};
-		tObj.term[entry] = query.account;
-		should_array.push(tObj);
-	}
-	return should_array;
+    const should_array = [];
+    for (const entry of terms) {
+        const tObj = {term: {}};
+        tObj.term[entry] = query.account;
+        should_array.push(tObj);
+    }
+    return should_array;
 }
 
 export function applyCodeActionFilters(query, queryStruct) {
-	let filterObj = [];
-	if (query.filter) {
-		for (const filter of query.filter.split(',')) {
-			if (filter !== '*:*') {
-				const _arr = [];
-				const parts = filter.split(':');
-				if (parts.length === 2) {
-					const [code, method] = parts;
-					if (code && code !== "*") {
-						_arr.push({'term': {'act.account': code}});
-					}
-					if (method && method !== "*") {
-						_arr.push({'term': {'act.name': method}});
-					}
-				}
-				if (_arr.length > 0) {
-					filterObj.push({bool: {must: _arr}});
-				}
-			}
-		}
-		if (filterObj.length > 0) {
-			queryStruct.bool['should'] = filterObj;
-			queryStruct.bool['minimum_should_match'] = 1;
-		}
-	}
+    let filterObj = [];
+    if (query.filter) {
+        for (const filter of query.filter.split(',')) {
+            if (filter !== '*:*') {
+                const _arr = [];
+                const parts = filter.split(':');
+                if (parts.length === 2) {
+                    const [code, method] = parts;
+                    if (code && code !== "*") {
+                        _arr.push({'term': {'act.account': code}});
+                    }
+                    if (method && method !== "*") {
+                        _arr.push({'term': {'act.name': method}});
+                    }
+                }
+                if (_arr.length > 0) {
+                    filterObj.push({bool: {must: _arr}});
+                }
+            }
+        }
+        if (filterObj.length > 0) {
+            queryStruct.bool['should'] = filterObj;
+            queryStruct.bool['minimum_should_match'] = 1;
+        }
+    }
 }
 
 export function getSkipLimit(query, max?: number) {
-	let skip, limit;
-	skip = parseInt(query.skip, 10);
-	if (skip < 0) {
-		throw new Error('invalid skip parameter');
-	}
-	limit = parseInt(query.limit, 10);
-	if (limit < 1) {
-		throw new Error('invalid limit parameter');
-	} else if (limit > max) {
-		throw new Error(`limit too big, maximum: ${max}`);
-	}
-	return {skip, limit};
+    let skip, limit;
+    skip = parseInt(query.skip, 10);
+    if (skip < 0) {
+        throw new Error('invalid skip parameter');
+    }
+    limit = parseInt(query.limit, 10);
+    if (limit < 1) {
+        throw new Error('invalid limit parameter');
+    } else if (limit > max) {
+        throw new Error(`limit too big, maximum: ${max}`);
+    }
+    return {skip, limit};
 }
 
 export function getSortDir(query) {
-	let sort_direction = 'desc';
-	if (query.sort) {
-		if (query.sort === 'asc' || query.sort === '1') {
-			sort_direction = 'asc';
-		} else if (query.sort === 'desc' || query.sort === '-1') {
-			sort_direction = 'desc'
-		} else {
-			throw new Error('invalid sort direction');
-		}
-	}
-	return sort_direction;
+    let sort_direction = 'desc';
+    if (query.sort) {
+        if (query.sort === 'asc' || query.sort === '1') {
+            sort_direction = 'asc';
+        } else if (query.sort === 'desc' || query.sort === '-1') {
+            sort_direction = 'desc'
+        } else {
+            throw new Error('invalid sort direction');
+        }
+    }
+    return sort_direction;
 }
 
 export function applyAccountFilters(query, queryStruct) {
-	if (query.account) {
-		queryStruct.bool.must.push({"bool": {should: makeShouldArray(query)}});
-	}
+    if (query.account) {
+        queryStruct.bool.must.push({"bool": {should: makeShouldArray(query)}});
+    }
 }

api/routes/v2-history/get_actions/get_actions.ts

@@ -22,11 +22,17 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
     };
 
     const {skip, limit} = getSkipLimit(query, maxActions);
+
     const sort_direction = getSortDir(query);
+
     applyAccountFilters(query, queryStruct);
+
     applyGenericFilters(query, queryStruct, fastify.allowedActionQueryParamSet);
+
     applyTimeFilter(query, queryStruct);
+
     applyCodeActionFilters(query, queryStruct);
+
     // allow precise counting of total hits
     const trackTotalHits = getTrackTotalHits(query);
 
@@ -46,12 +52,15 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
         indexPattern = fastify.manager.chain + '-action';
     }
 
-    const esResults = await fastify.elastic.search({
+    const esOpts = {
         "index": indexPattern,
         "from": skip || 0,
         "size": (limit > maxActions ? maxActions : limit) || 10,
         "body": query_body
-    });
+    };
+
+    // console.log(JSON.stringify(esOpts, null, 2));
+    const esResults = await fastify.elastic.search(esOpts);
 
     const results = esResults['body']['hits'];
     const response: any = {
@@ -101,13 +110,14 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
 
             if (query.simple) {
+                let notified = new Set(action.receipts.map(r => r.receiver));
                 response.simple_actions.push({
                     block: action['block_num'],
                     irreversible: response.lib !== 0 ? action['block_num'] < response.lib : undefined,
                     timestamp: action['@timestamp'],
                     transaction_id: action['trx_id'],
                     actors: action['act']['authorization'].map(a => `${a.actor}@${a.permission}`).join(","),
-                    notified: action['notified'].join(','),
+                    notified: [...notified].join(','),
                     contract: action['act']['account'],
                     action: action['act']['name'],
                     data: action['act']['data']

api/routes/v2-history/get_actions/index.ts

@@ -3,130 +3,158 @@ import {getActionsHandler} from "./get_actions";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export const getActionResponseSchema = {
-	"@timestamp": {type: "string"},
-	"timestamp": {type: "string"},
-	"block_num": {type: "number"},
-	"trx_id": {type: "string"},
-	"act": {
-		type: 'object',
-		properties: {
-			"account": {type: "string"},
-			"name": {type: "string"}
-		},
-		additionalProperties: true
-	},
-	"notified": {
-		type: "array", items: {type: "string"}
-	},
-	"cpu_usage_us": {type: "number"},
-	"net_usage_words": {type: "number"},
-	"account_ram_deltas": {
-		type: "array",
-		items: {
-			type: "object",
-			properties: {
-				"account": {type: "string"},
-				"delta": {type: "number"}
-			},
-			additionalProperties: true
-		}
-	},
-	"global_sequence": {type: "number"},
-	"receiver": {type: 'string'},
-	"producer": {type: "string"},
-	"parent": {type: "number"},
-	"action_ordinal": {type: 'number'},
-	"creator_action_ordinal": {type: 'number'},
-	"signatures": {type: "array", items: {type: 'string'}}
+    "@timestamp": {type: "string"},
+    "timestamp": {type: "string"},
+    "block_num": {type: "number"},
+    "block_id": {type: "string"},
+    "trx_id": {type: "string"},
+    "act": {
+        type: 'object',
+        properties: {
+            "account": {type: "string"},
+            "name": {type: "string"},
+            "authorization": {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        "actor": {type: "string"},
+                        "permission": {type: "string"},
+                    }
+                }
+            }
+        },
+        additionalProperties: true
+    },
+    "receipts": {
+        type: "array",
+        items: {
+            type: "object",
+            properties: {
+                "receiver": {type: "string"},
+                "global_sequence": {type: "number"},
+                "recv_sequence": {type: "number"},
+                "auth_sequence": {
+                    type: "array",
+                    items: {
+                        type: "object",
+                        properties: {
+                            "account": {type: "string"},
+                            "sequence": {type: "number"},
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "cpu_usage_us": {type: "number"},
+    "net_usage_words": {type: "number"},
+    "account_ram_deltas": {
+        type: "array",
+        items: {
+            type: "object",
+            properties: {
+                "account": {type: "string"},
+                "delta": {type: "number"}
+            },
+            additionalProperties: true
+        }
+    },
+    "global_sequence": {type: "number"},
+    "producer": {type: "string"},
+    "parent": {type: "number"},
+    "action_ordinal": {type: 'number'},
+    "creator_action_ordinal": {type: 'number'},
+    "signatures": {type: "array", items: {type: 'string'}}
 };
 
 export default function (fastify: FastifyInstance, opts: any, next) {
-	const schema: FastifySchema = {
-		description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
-			' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
-		summary: 'get root actions',
-		tags: ['history'],
-		querystring: extendQueryStringSchema({
-			"account": {
-				description: 'notified account',
-				type: 'string',
-				minLength: 1,
-				maxLength: 12
-			},
-			"track": {
-				description: 'total results to track (count) [number or true]',
-				type: 'string'
-			},
-			"filter": {
-				description: 'code:name filter',
-				type: 'string',
-				minLength: 3
-			},
-			"sort": {
-				description: 'sort direction',
-				enum: ['desc', 'asc', '1', '-1'],
-				type: 'string'
-			},
-			"after": {
-				description: 'filter after specified date (ISO8601)',
-				type: 'string'
-			},
-			"before": {
-				description: 'filter before specified date (ISO8601)',
-				type: 'string'
-			},
-			"simple": {
-				description: 'simplified output mode',
-				type: 'boolean'
-			},
-			"hot_only": {
-				description: 'search only the latest hot index',
-				type: 'boolean'
-			},
-			"noBinary": {
-				description: "exclude large binary data",
-				type: 'boolean'
-			},
-			"checkLib": {
-				description: "perform reversibility check",
-				type: 'boolean'
-			},
-		}),
-		response: extendResponseSchema({
-			"simple_actions": {
-				type: "array",
-				items: {
-					type: "object",
-					properties: {
-						"block": {type: "number"},
-						"timestamp": {type: "string"},
-						"irreversible": {type: "boolean"},
-						"contract": {type: "string"},
-						"action": {type: "string"},
-						"actors": {type: "string"},
-						"notified": {type: "string"},
-						"transaction_id": {type: "string"},
-						"data": {
-							additionalProperties: true
-						}
-					}
-				}
-			},
-			"actions": {
-				type: "array",
-				items: {
-					type: 'object',
-					properties: getActionResponseSchema
-				}
-			}
-		})
-	};
-	addApiRoute(
-		fastify,
-		'GET',
-		getRouteName(__filename),
-		getActionsHandler,
-		schema
-	);
-	next();
+    const schema: FastifySchema = {
+        description: 'get actions based on notified account. this endpoint also accepts generic filters based on indexed fields' +
+            ' (e.g. act.authorization.actor=eosio or act.name=delegatebw), if included they will be combined with a AND operator',
+        summary: 'get root actions',
+        tags: ['history'],
+        querystring: extendQueryStringSchema({
+            "account": {
+                description: 'notified account',
+                type: 'string',
+                minLength: 1,
+                maxLength: 12
+            },
+            "track": {
+                description: 'total results to track (count) [number or true]',
+                type: 'string'
+            },
+            "filter": {
+                description: 'code:name filter',
+                type: 'string',
+                minLength: 3
+            },
+            "sort": {
+                description: 'sort direction',
+                enum: ['desc', 'asc', '1', '-1'],
+                type: 'string'
+            },
+            "after": {
+                description: 'filter after specified date (ISO8601)',
+                type: 'string'
+            },
+            "before": {
+                description: 'filter before specified date (ISO8601)',
+                type: 'string'
+            },
+            "simple": {
+                description: 'simplified output mode',
+                type: 'boolean'
+            },
+            "hot_only": {
+                description: 'search only the latest hot index',
+                type: 'boolean'
+            },
+            "noBinary": {
+                description: "exclude large binary data",
+                type: 'boolean'
+            },
+            "checkLib": {
+                description: "perform reversibility check",
+                type: 'boolean'
+            },
+        }),
+        response: extendResponseSchema({
+            "simple_actions": {
+                type: "array",
+                items: {
+                    type: "object",
+                    properties: {
+                        "block": {type: "number"},
+                        "timestamp": {type: "string"},
+                        "irreversible": {type: "boolean"},
+                        "contract": {type: "string"},
+                        "action": {type: "string"},
+                        "actors": {type: "string"},
+                        "notified": {type: "string"},
+                        "transaction_id": {type: "string"},
+                        "data": {
+                            additionalProperties: true
+                        }
+                    }
+                }
+            },
+            "actions": {
+                type: "array",
+                items: {
+                    type: 'object',
+                    properties: getActionResponseSchema
+                }
+            }
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getActionsHandler,
+        schema
+    );
+    next();
 }

api/routes/v2-history/get_schedule/get_schedule.ts

@@ -5,69 +5,101 @@ import {base58ToBinary, binaryToBase58} from "eosjs/dist/eosjs-numeric";
 import {Search} from "@elastic/elasticsearch/api/requestParams";
 
 function convertToLegacyKey(block_signing_key: string) {
-	if (block_signing_key.startsWith("PUB_K1_")) {
-		const buf = base58ToBinary(37, block_signing_key.substr(7));
-		const data = buf.slice(0, buf.length - 4);
-		const merged = Buffer.concat([
-			data,
-			createHash('ripemd160')
-				.update(data)
-				.digest()
-				.slice(0, 4)
-		]);
-		return "EOS" + binaryToBase58(merged);
-	} else {
-		return block_signing_key;
-	}
+    if (block_signing_key.startsWith("PUB_K1_")) {
+        const buf = base58ToBinary(37, block_signing_key.substr(7));
+        const data = buf.slice(0, buf.length - 4);
+        const merged = Buffer.concat([
+            data,
+            createHash('ripemd160')
+                .update(data)
+                .digest()
+                .slice(0, 4)
+        ]);
+        return "EOS" + binaryToBase58(merged);
+    } else {
+        return block_signing_key;
+    }
 }
 
 async function getSchedule(fastify: FastifyInstance, request: FastifyRequest) {
-
-	const query: any = request.query;
-
-	const response: any = {
-		producers: []
-	};
-	const searchParams: Search<any> = {
-		track_total_hits: true,
-		index: fastify.manager.chain + "-block-*",
-		size: 1,
-		body: {
-			query: {
-				bool: {
-					must: []
-				}
-			},
-			sort: {block_num: "desc"}
-		}
-	};
-	if (query.version) {
-		searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": query.version}}});
-	} else {
-		searchParams.body.query.bool.must.push({
-			"exists": {
-				"field": "new_producers.version"
-			}
-		});
-	}
-	const apiResponse = await fastify.elastic.search(searchParams);
-	const results = apiResponse.body.hits.hits;
-	if (results) {
-		response.timestamp = results[0]._source["@timestamp"];
-		response.block_num = results[0]._source.block_num;
-		response.version = results[0]._source.new_producers.version;
-		response.producers = results[0]._source.new_producers.producers.map(prod => {
-			return {
-				...prod,
-				legacy_key: convertToLegacyKey(prod.block_signing_key)
-			}
-		});
-	}
-	return response;
+    const query: any = request.query;
+    const response: any = {
+        producers: []
+    };
+    if (query.mode === 'activated') {
+        const searchParams: Search<any> = {
+            track_total_hits: true,
+            index: fastify.manager.chain + "-block-*",
+            size: 1,
+            body: {
+                query: {bool: {must: []}},
+                sort: {block_num: query.sort || "desc"}
+            }
+        };
+        if (query.version) {
+            searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": query.version}}});
+        } else {
+            searchParams.body.query.bool.must.push({"exists": {"field": "new_producers.version"}});
+        }
+        const apiResponse = await fastify.elastic.search(searchParams);
+        const results = apiResponse.body.hits.hits;
+        if (results && results.length > 0) {
+            response.timestamp = results[0]._source["@timestamp"];
+            response.block_num = results[0]._source.block_num;
+            response.version = results[0]._source.new_producers.version;
+            response.producers = results[0]._source.new_producers.producers.map(prod => {
+                return {
+                    ...prod,
+                    legacy_key: convertToLegacyKey(prod.block_signing_key)
+                }
+            });
+        }
+    } else {
+        // default "proposed" mode
+        const searchParams: Search<any> = {
+            track_total_hits: true,
+            index: fastify.manager.chain + "-schedule-*",
+            size: 1,
+            body: {
+                query: {bool: {must: []}},
+                sort: {version: query.sort || "desc"}
+            }
+        };
+        if (query.version) {
+            searchParams.body.query.bool.must.push({"term": {"version": {"value": query.version}}});
+        } else {
+            searchParams.body.query.bool.must.push({"exists": {"field": "version"}});
+        }
+        if (query.producer) {
+            searchParams.body.query.bool.must.push({"term": {"producers.name": {"value": query.producer}}});
+        }
+        if (query.key) {
+            searchParams.body.query.bool.must.push({"term": {"producers.keys": {"value": query.key}}});
+        }
+        const apiResponse = await fastify.elastic.search(searchParams);
+        const results = apiResponse.body.hits.hits;
+        if (results && results.length > 0) {
+            response.timestamp = results[0]._source["@timestamp"];
+            response.proposal_block_num = results[0]._source.block_num;
+            response.version = results[0]._source.version;
+            response.producers = results[0]._source.producers;
+            // sort producers by name
+            // response.producers.sort((a, b) => {
+            //     if (a.name < b.name) {
+            //         return -1;
+            //     }
+            //     if (a.name > b.name) {
+            //         return 1;
+            //     }
+            //     return 0;
+            // });
+        }
+    }
+    return response;
 }
 
 export function getScheduleHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply) => {
-		reply.send(await timedQuery(getSchedule, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getSchedule, fastify, request, route));
+    }
 }

api/routes/v2-history/get_schedule/index.ts

@@ -15,7 +15,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: 'string'
                 },
                 "key": {
-                    description: 'search by key',
+                    description: 'search by signing key',
                     type: 'string'
                 },
                 "after": {
@@ -26,6 +26,10 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     description: 'filter before specified date (ISO8601)',
                     type: 'string'
                 },
+                "mode": {
+                    description: 'search mode (activated or proposed)',
+                    type: 'string',
+                },
                 "version": {
                     description: 'schedule version',
                     type: 'integer',
@@ -36,6 +40,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         response: extendResponseSchema({
             "timestamp": {type: "string"},
             "block_num": {type: "number"},
+            "proposed_block_num": {type: "number"},
             "version": {type: "number"},
             "producers": {
                 type: "array",
@@ -43,8 +48,10 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: "object",
                     properties: {
                         "producer_name": {type: "string"},
+                        "name": {type: "string"},
                         "block_signing_key": {type: "string"},
-                        "legacy_key": {type: "string"}
+                        "legacy_key": {type: "string"},
+                        "keys": {type: "array", items: {type: "string"}}
                     }
                 }
             },

api/routes/v2-stats/get_api_usage/get_api_usage.ts

@@ -0,0 +1,53 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {timedQuery} from "../../../helpers/functions";
+import {parseInt} from "lodash";
+
+async function getApiUsage(fastify: FastifyInstance, request: FastifyRequest) {
+    const response = {
+        total: {responses: {}},
+        buckets: []
+    } as any;
+    const data = await fastify.redis.keys(`stats:${fastify.manager.chain}:*`);
+    const buckets = [];
+    for (const key of data) {
+        const parts = key.split(':');
+        if (parts[2] === 'H') {
+            const bucket = {
+                timestamp: new Date(parseInt(parts[3])),
+                responses: {}
+            };
+            const sortedSet = await fastify.redis.zrange(key, 0, -1, 'WITHSCORES');
+            for (let i = 0; i < sortedSet.length; i += 2) {
+                const [status, path] = sortedSet[i].split(']');
+                if (status) {
+                    const statusCode = status.slice(1);
+                    if (!bucket.responses[statusCode]) {
+                        bucket.responses[statusCode] = {};
+                    }
+                    if (statusCode) {
+                        bucket.responses[statusCode][path] = parseInt(sortedSet[i + 1]);
+                    }
+                }
+            }
+            response.buckets.push(bucket);
+        } else {
+            if (parts[2] === 'total' && parts[4]) {
+                const statusCode = parts[3];
+                if (!response.total.responses[statusCode]) {
+                    response.total.responses[statusCode] = {};
+                }
+                const value = await fastify.redis.get(key);
+                response.total.responses[statusCode][parts[4]] = parseInt(value);
+            }
+        }
+    }
+    // reverse sorte by date
+    response.buckets.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+    return response;
+}
+
+export function getApiUsageHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getApiUsage, fastify, request, route));
+    }
+}

api/routes/v2-stats/get_api_usage/index.ts

@@ -0,0 +1,41 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getApiUsageHandler} from "./get_api_usage";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get hyperion api usage statistics',
+        summary: 'get hyperion api usage statistics',
+        tags: ['stats'],
+        response: extendResponseSchema({
+            total: {
+                type: 'object',
+                properties: {
+                    responses: {
+                        additionalProperties: true
+                    }
+                }
+            },
+            buckets: {
+                type: 'array',
+                items: {
+                    type: 'object',
+                    properties: {
+                        timestamp: {type: 'string'},
+                        responses: {
+                            additionalProperties: true
+                        }
+                    }
+                }
+            }
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getApiUsageHandler,
+        schema
+    );
+    next();
+}

api/routes/v2/health/health.ts

@@ -63,7 +63,14 @@ interface ServiceResponse<T> {
 
 async function checkElastic(fastify: FastifyInstance): Promise<ServiceResponse<ESService>> {
     try {
-        let esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
+        const esStatusCache = await fastify.redis.get(`${fastify.manager.chain}::es_status`);
+        let esStatus;
+        if (esStatusCache) {
+            esStatus = JSON.parse(esStatusCache);
+        } else {
+            esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
+            await fastify.redis.set(`${fastify.manager.chain}::es_status`, JSON.stringify(esStatus), 'EX', 30 * 60);
+        }
         let firstIndexedBlock: number;
         const fib = await fastify.redis.get(`${fastify.manager.chain}::fib`);
         if (fib) {

api/server.ts

@@ -1,4 +1,4 @@
-import {hLog} from "../helpers/common_functions";
+import {hLog, waitUntilReady} from "../helpers/common_functions";
 import {ConfigurationModule} from "../modules/config";
 import {ConnectionManager} from "../connections/manager.class";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
@@ -16,7 +16,6 @@ import {io, Socket} from "socket.io-client";
 import {CacheManager} from "./helpers/cacheManager";
 
 import {bootstrap} from 'global-agent';
-bootstrap();
 
 class HyperionApiServer {
 
@@ -29,6 +28,7 @@ class HyperionApiServer {
 
     socketManager: SocketManager;
     mLoader: HyperionModuleLoader;
+    private pluginParams: any;
 
     constructor() {
 
@@ -37,6 +37,11 @@ class HyperionApiServer {
 
         const cm = new ConfigurationModule();
         this.conf = cm.config;
+
+        if (this.conf.settings.use_global_agent) {
+            bootstrap();
+        }
+
         this.chain = this.conf.settings.chain;
         process.title = `hyp-${this.chain}-api`;
         this.manager = new ConnectionManager(cm);
@@ -111,6 +116,7 @@ class HyperionApiServer {
             },
             fastify_redis: this.manager.conn.redis,
             fastify_eosjs: this.manager,
+            chain_id: '',
         } as any;
 
         if (!this.conf.api.disable_rate_limit) {
@@ -140,26 +146,25 @@ class HyperionApiServer {
             pluginParams.fastify_swagger = docsConfig;
         }
 
-        registerPlugins(this.fastify, pluginParams);
-
-        this.addGenericTypeParsing();
+        this.pluginParams = pluginParams;
     }
 
     activateStreaming() {
-        console.log('Importing stream module');
+        console.log('Importing stream module...');
         import('./socketManager').then((mod) => {
             const connOpts = this.manager.conn.chains[this.chain];
-
             let _port = 57200;
             if (connOpts.WS_ROUTER_PORT) {
                 _port = connOpts.WS_ROUTER_PORT;
             }
-
             let _host = "127.0.0.1";
             if (connOpts.WS_ROUTER_HOST) {
                 _host = connOpts.WS_ROUTER_HOST;
             }
-
+            if (_host === "0.0.0.0") {
+                hLog(`[ERROR] WS Router Host is set to 0.0.0.0, please use a fixed IP address instead. Can't start streaming.`);
+                return;
+            }
             this.socketManager = new mod.SocketManager(
                 this.fastify,
                 `http://${_host}:${_port}`,
@@ -187,6 +192,47 @@ class HyperionApiServer {
 
     async init() {
 
+        const rpc = this.manager.nodeosJsonRPC;
+        await waitUntilReady(async () => {
+            try {
+                const chain_data = await rpc.get_info();
+                if (chain_data && chain_data.chain_id) {
+                    this.pluginParams.chain_id = chain_data.chain_id;
+                    return true;
+                } else {
+                    return false;
+                }
+            } catch (e) {
+                hLog(e.message);
+                return false;
+            }
+        }, 20, 5000, () => {
+            hLog('Failed to validate chain api!');
+            process.exit(1);
+        });
+        hLog('Chain API validated!');
+
+        // Wait for Elasticsearch availability
+        await waitUntilReady(async () => {
+            try {
+                const esInfo = await this.manager.elasticsearchClient.info();
+                hLog(`Elasticsearch: ${esInfo.body.version.number} | Lucene: ${esInfo.body.version.lucene_version}`);
+                return true;
+            } catch (e) {
+                console.log(e.message);
+                return false;
+            }
+        }, 10, 5000, () => {
+            hLog('Failed to check elasticsearch version!');
+            process.exit();
+        });
+
+        hLog('Elasticsearch validated!');
+        hLog('Registering plugins...');
+
+        registerPlugins(this.fastify, this.pluginParams);
+        this.addGenericTypeParsing();
+
         await this.mLoader.init();
 
         // add custom plugin routes

api/socketManager.ts

@@ -1,4 +1,4 @@
-import {hLog} from '../helpers/common_functions';
+import {hLog, sleep} from '../helpers/common_functions';
 import {Server, Socket} from 'socket.io';
 import {createAdapter} from 'socket.io-redis';
 import {io} from 'socket.io-client';
@@ -6,6 +6,7 @@ import {FastifyInstance} from "fastify";
 import IORedis from "ioredis";
 import {App, TemplatedApp} from 'uWebSockets.js';
 import {streamPastActions, streamPastDeltas} from "./helpers/functions";
+import {randomUUID} from "crypto";
 
 export interface StreamDeltasRequest {
     code: string;
@@ -40,17 +41,14 @@ export class SocketManager {
     private readonly url;
     private readonly server: FastifyInstance;
     private readonly uwsApp: TemplatedApp;
+    private readonly chainId: string;
+    private currentBlockNum: number;
 
     constructor(fastify: FastifyInstance, url, redisOptions) {
         this.server = fastify;
         this.url = url;
         this.uwsApp = App({});
 
-        // this.io = new Server(fastify.server, {
-        // 	allowEIO3: true,
-        // 	transports: ['websocket', 'polling'],
-        // });
-
         // WS Server for public access
         this.io = new Server({
             transports: ['websocket'],
@@ -59,9 +57,11 @@ export class SocketManager {
 
         this.io.attachApp(this.uwsApp);
 
+        this.chainId = this.server.manager.conn.chains[this.server.manager.chain].chain_id
+        hLog(`[SocketManager] chain_id: ${this.chainId}`);
         const pubClient = new IORedis(redisOptions);
         const subClient = pubClient.duplicate();
-        this.io.adapter(createAdapter({pubClient, subClient}));
+        this.io.adapter(createAdapter({pubClient, subClient, key: this.chainId}));
 
         this.io.on('connection', (socket: Socket) => {
 
@@ -84,10 +84,42 @@ export class SocketManager {
             socket.on('delta_stream_request', async (data: StreamDeltasRequest, callback) => {
                 if (typeof callback === 'function' && data) {
                     try {
+                        // generate random uuid
+                        socket.data.reqUUID = randomUUID();
+                        const lastHistoryBlock = await new Promise<number>((resolve) => {
+                            // start sending realtime data
+                            this.emitToRelay(data, 'delta_request', socket, (emissionResult) => {
+                                callback(emissionResult);
+                                resolve(emissionResult.currentBlockNum);
+                            });
+                        });
+                        // push history data
                         if (data.start_from) {
-                            await streamPastDeltas(this.server, socket, data);
+                            data.read_until = lastHistoryBlock;
+                            console.log('Performing primary scroll request...');
+                            let ltb = 0;
+                            const hStreamResult = await streamPastDeltas(this.server, socket, data);
+                            if (hStreamResult.status === false) {
+                                return;
+                            } else {
+                                ltb = hStreamResult.lastTransmittedBlock;
+                                let attempts = 0;
+                                await sleep(500);
+                                while (ltb > 0 && lastHistoryBlock > ltb && attempts < 3) {
+                                    attempts++;
+                                    console.log(`Performing fill request from ${ltb}...`);
+                                    data.start_from = hStreamResult.lastTransmittedBlock + 1;
+                                    data.read_until = lastHistoryBlock;
+                                    const r = await streamPastDeltas(this.server, socket, data);
+                                    if (r.status === false) {
+                                        console.log(r);
+                                        return;
+                                    } else {
+                                        ltb = r.lastTransmittedBlock;
+                                    }
+                                }
+                            }
                         }
-                        this.emitToRelay(data, 'delta_request', socket, callback);
                     } catch (e) {
                         console.log(e);
                     }
@@ -97,10 +129,42 @@ export class SocketManager {
             socket.on('action_stream_request', async (data: StreamActionsRequest, callback) => {
                 if (typeof callback === 'function' && data) {
                     try {
+                        // generate random uuid
+                        socket.data.reqUUID = randomUUID();
+                        const lastHistoryBlock = await new Promise<number>((resolve) => {
+                            // start sending realtime data
+                            this.emitToRelay(data, 'action_request', socket, (emissionResult) => {
+                                callback(emissionResult);
+                                resolve(emissionResult.currentBlockNum);
+                            });
+                        });
+                        // push history data
                         if (data.start_from) {
-                            await streamPastActions(this.server, socket, data);
+                            data.read_until = lastHistoryBlock;
+                            console.log('Performing primary scroll request...');
+                            let ltb = 0;
+                            const hStreamResult = await streamPastActions(this.server, socket, data);
+                            if (hStreamResult.status === false) {
+                                return;
+                            } else {
+                                ltb = hStreamResult.lastTransmittedBlock;
+                                let attempts = 0;
+                                await sleep(500);
+                                while (ltb > 0 && lastHistoryBlock > ltb && attempts < 3) {
+                                    attempts++;
+                                    console.log(`Performing fill request from ${hStreamResult.lastTransmittedBlock}...`);
+                                    data.start_from = hStreamResult.lastTransmittedBlock + 1;
+                                    data.read_until = lastHistoryBlock;
+                                    const r = await streamPastActions(this.server, socket, data);
+                                    if (r.status === false) {
+                                        console.log(r);
+                                        return;
+                                    } else {
+                                        ltb = r.lastTransmittedBlock;
+                                    }
+                                }
+                            }
                         }
-                        this.emitToRelay(data, 'action_request', socket, callback);
                     } catch (e) {
                         console.log(e);
                     }
@@ -160,18 +224,40 @@ export class SocketManager {
             this.emitToClient(traceData, 'action_trace');
         });
 
-        // Relay LIB info to clients;
-        this.relay.on('lib_update', (data) => {
-            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-                this.io.emit('lib_update', data);
+        this.relay.on('block', (blockData) => {
+            try {
+                // const decodedBlock = JSON.parse(blockData.content.toString());
+                // console.log(blockData.serverTime, blockData.blockNum, decodedBlock);
+                this.currentBlockNum = blockData.blockNum;
+            } catch (e) {
+                hLog(`Failed to decode incoming live block ${blockData.blockNum}: ${e.message}`);
             }
         });
 
-        // Relay LIB info to clients;
-        this.relay.on('fork_event', (data) => {
-            hLog(data);
-            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-                this.io.emit('fork_event', data);
+        this.addRelayForwarding('lib_update');
+
+        this.addRelayForwarding('fork_event');
+
+        // // Relay LIB info to clients;
+        // this.relay.on('lib_update', (data) => {
+        //     if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+        //         this.io.emit('lib_update', data);
+        //     }
+        // });
+        //
+        // // Relay fork info to clients;
+        // this.relay.on('fork_event', (data) => {
+        //     if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+        //         this.io.emit('fork_event', data);
+        //     }
+        // });
+    }
+
+    // Relay events to clients
+    addRelayForwarding(event: string) {
+        this.relay.on(event, (data: any) => {
+            if (data.chain_id && this.chainId === data.chain_id) {
+                this.io.emit(event, data);
             }
         });
     }
@@ -180,6 +266,7 @@ export class SocketManager {
         if (this.io.sockets.sockets.has(traceData.client)) {
             this.io.sockets.sockets.get(traceData.client).emit('message', {
                 type: type,
+                reqUUID: traceData.req,
                 mode: 'live',
                 message: traceData.message,
             });
@@ -189,10 +276,13 @@ export class SocketManager {
     emitToRelay(data, type, socket, callback) {
         if (this.relay.connected) {
             this.relay.emit('event', {
+                reqUUID: socket.data.reqUUID,
                 type: type,
                 client_socket: socket.id,
                 request: data,
             }, (response) => {
+                response['reqUUID'] = socket.data.reqUUID;
+                response['currentBlockNum'] = this.currentBlockNum;
                 callback(response);
             });
         } else {

chains/example.config.json

@@ -7,6 +7,8 @@
     "server_addr": "127.0.0.1",
     "server_port": 7000,
     "stream_port": 1234,
+    "stream_scroll_limit": -1,
+    "stream_scroll_batch": 500,
     "server_name": "127.0.0.1:7000",
     "provider_name": "Example Provider",
     "provider_url": "https://example.com",
@@ -63,7 +65,7 @@
     "preview": false,
     "chain": "eos",
     "eosio_alias": "eosio",
-    "parser": "2.1",
+    "parser": "3.2",
     "auto_stop": 0,
     "index_version": "v1",
     "debug": false,
@@ -77,7 +79,7 @@
     "auto_mode_switch": false,
     "hot_warm_policy": false,
     "custom_policy": "",
-    "bypass_index_map": false,
+    "use_global_agent": false,
     "index_partition_size": 10000000,
     "es_replicas": 0
   },
@@ -120,7 +122,7 @@
       "voters": true
     },
     "index_deltas": true,
-    "index_transfer_memo": false,
+    "index_transfer_memo": true,
     "index_all_deltas": true,
     "deferred_trx": false,
     "failed_trx": false,

connections/amqp.ts

@@ -17,17 +17,16 @@ export async function createConnection(config): Promise<Connection> {
 }
 
 export function getAmpqUrl(config): string {
-	let frameMaxValue = '0x10000';
-	if (config.frameMax) {
-		frameMaxValue = config.frameMax;
-	}
-	const u = encodeURIComponent(config.user);
-	const p = encodeURIComponent(config.pass);
-	const v = encodeURIComponent(config.vhost)
-	return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
+    let frameMaxValue = '0x10000';
+    if (config.frameMax) {
+        frameMaxValue = config.frameMax;
+    }
+    const u = encodeURIComponent(config.user);
+    const p = encodeURIComponent(config.pass);
+    const v = encodeURIComponent(config.vhost);
+    return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
 }
 
-
 async function createChannels(connection) {
 	try {
 		const channel = await connection.createChannel();

connections/manager.class.ts

@@ -12,167 +12,163 @@ import {hLog} from "../helpers/common_functions";
 
 export class ConnectionManager {
 
-	config: HyperionConfig;
-	conn: HyperionConnections;
+    config: HyperionConfig;
+    conn: HyperionConnections;
 
-	chain: string;
-	last_commit_hash: string;
-	current_version: string;
+    chain: string;
+    last_commit_hash: string;
+    current_version: string;
 
-	private readonly esIngestClients: Client[];
-	private esIngestClient: Client;
+    esIngestClients: Client[];
+    esIngestClient: Client;
 
-	constructor(private cm: ConfigurationModule) {
-		this.config = cm.config;
-		this.conn = cm.connections;
-		if (!this.conn.amqp.protocol) {
-			this.conn.amqp.protocol = 'http';
-		}
-		this.chain = this.config.settings.chain;
-		this.esIngestClients = [];
-		this.prepareESClient();
-		this.prepareIngestClients();
-	}
+    constructor(private cm: ConfigurationModule) {
+        this.config = cm.config;
+        this.conn = cm.connections;
+        if (!this.conn.amqp.protocol) {
+            this.conn.amqp.protocol = 'http';
+        }
+        this.chain = this.config.settings.chain;
+        this.esIngestClients = [];
+        this.prepareESClient();
+        this.prepareIngestClients();
+    }
 
-	get nodeosJsonRPC() {
-		// @ts-ignore
-		return new JsonRpc(this.conn.chains[this.chain].http, {fetch});
-	}
+    get nodeosJsonRPC() {
+        // @ts-ignore
+        return new JsonRpc(this.conn.chains[this.chain].http, {fetch});
+    }
 
-	async purgeQueues() {
-		hLog(`Purging all ${this.chain} queues!`);
-		const apiUrl = `http://${this.conn.amqp.api}`;
-		const vHost = encodeURIComponent(this.conn.amqp.vhost);
-		const getAllQueuesFromVHost = apiUrl + `/api/queues/${vHost}`;
-		const opts = {
-			username: this.conn.amqp.user,
-			password: this.conn.amqp.pass
-		};
-		let result;
-		try {
-			const data = await got(getAllQueuesFromVHost, opts);
-			if (data) {
-				result = JSON.parse(data.body);
-			}
-		} catch (e) {
-			console.log(e.message);
-			console.error('failed to connect to rabbitmq http api');
-			process.exit(1);
-		}
-		if (result) {
-			for (const queue of result) {
-				if (queue.name.startsWith(this.chain + ":")) {
-					const msg_count = parseInt(queue.messages);
-					if (msg_count > 0) {
-						try {
-							await got.delete(apiUrl + `/api/queues/${vHost}/${queue.name}/contents`, opts);
-							hLog(`${queue.messages} messages deleted on queue ${queue.name}`);
-						} catch (e) {
-							console.log(e.message);
-							console.error('failed to connect to rabbitmq http api');
-							process.exit(1);
-						}
-					}
-				}
-			}
-		}
-	}
+    async purgeQueues() {
+        hLog(`Purging all ${this.chain} queues!`);
+        const apiUrl = `http://${this.conn.amqp.api}`;
+        const vHost = encodeURIComponent(this.conn.amqp.vhost);
+        const getAllQueuesFromVHost = apiUrl + `/api/queues/${vHost}`;
+        const opts = {
+            username: this.conn.amqp.user,
+            password: this.conn.amqp.pass
+        };
+        let result;
+        try {
+            const data = await got(getAllQueuesFromVHost, opts);
+            if (data) {
+                result = JSON.parse(data.body);
+            }
+        } catch (e) {
+            console.log(e.message);
+            console.error('failed to connect to rabbitmq http api');
+            process.exit(1);
+        }
+        if (result) {
+            for (const queue of result) {
+                if (queue.name.startsWith(this.chain + ":")) {
+                    const msg_count = parseInt(queue.messages);
+                    if (msg_count > 0) {
+                        try {
+                            await got.delete(apiUrl + `/api/queues/${vHost}/${queue.name}/contents`, opts);
+                            hLog(`${queue.messages} messages deleted on queue ${queue.name}`);
+                        } catch (e) {
+                            console.log(e.message);
+                            console.error('failed to connect to rabbitmq http api');
+                            process.exit(1);
+                        }
+                    }
+                }
+            }
+        }
+    }
 
-	prepareESClient() {
-		let es_url;
-		const _es = this.conn.elasticsearch;
-		if (!_es.protocol) {
-			_es.protocol = 'http';
-		}
-		if (_es.user !== '') {
-			es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${_es.host}`;
-		} else {
-			es_url = `${_es.protocol}://${_es.host}`
-		}
-		this.esIngestClient = new Client({
-			node: es_url,
-			ssl: {
-				rejectUnauthorized: false
-			}
-		});
-	}
+    prepareESClient() {
+        const _es = this.conn.elasticsearch;
+        if (!_es.protocol) {
+            _es.protocol = 'http';
+        }
+        this.esIngestClient = new Client({
+            node: `${_es.protocol}://${_es.host}`,
+            auth: {
+                username: _es.user,
+                password: _es.pass
+            },
+            ssl: _es.protocol === 'https' ? {
+                rejectUnauthorized: false
+            } : undefined
+        });
+    }
 
-	get elasticsearchClient() {
-		return this.esIngestClient;
-	}
+    get elasticsearchClient() {
+        return this.esIngestClient;
+    }
 
-	prepareIngestClients() {
-		const _es = this.conn.elasticsearch;
-		if (!_es.protocol) {
-			_es.protocol = 'http';
-		}
-		if (_es.ingest_nodes) {
-			if (_es.ingest_nodes.length > 0) {
-				for (const node of _es.ingest_nodes) {
-					let es_url;
-					if (_es.user !== '') {
-						es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${node}`;
-					} else {
-						es_url = `${_es.protocol}://${node}`
-					}
-					this.esIngestClients.push(new Client({
-						node: es_url,
-						pingTimeout: 100,
-						ssl: {
-							rejectUnauthorized: false
-						}
-					}));
-				}
-			}
-		}
-	}
+    prepareIngestClients() {
+        const _es = this.conn.elasticsearch;
+        if (!_es.protocol) {
+            _es.protocol = 'http';
+        }
+        if (_es.ingest_nodes) {
+            if (_es.ingest_nodes.length > 0) {
+                for (const node of _es.ingest_nodes) {
+                    this.esIngestClient = new Client({
+                        node: `${_es.protocol}://${_es.host}`,
+                        auth: {
+                            username: _es.user,
+                            password: _es.pass
+                        },
+                        pingTimeout: 100,
+                        ssl: _es.protocol === 'https' ? {
+                            rejectUnauthorized: false
+                        } : undefined
+                    });
+                }
+            }
+        }
+    }
 
-	get ingestClients() {
-		if (this.esIngestClients.length > 0) {
-			return this.esIngestClients;
-		} else {
-			return [this.esIngestClient];
-		}
-	}
+    get ingestClients() {
+        if (this.esIngestClients.length > 0) {
+            return this.esIngestClients;
+        } else {
+            return [this.esIngestClient];
+        }
+    }
 
-	async createAMQPChannels(onReconnect, onClose) {
-		return await amqpConnect(onReconnect, this.conn.amqp, onClose);
-	}
+    async createAMQPChannels(onReconnect, onClose) {
+        return await amqpConnect(onReconnect, this.conn.amqp, onClose);
+    }
 
-	async checkQueueSize(queue) {
-		return await checkQueueSize(queue, this.conn.amqp);
-	}
+    async checkQueueSize(queue) {
+        return await checkQueueSize(queue, this.conn.amqp);
+    }
 
-	get shipClient(): StateHistorySocket {
-		return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_mb);
-	}
+    get shipClient(): StateHistorySocket {
+        return new StateHistorySocket(this.conn.chains[this.config.settings.chain]['ship'], this.config.settings.max_ws_payload_mb);
+    }
 
-	get ampqUrl() {
-		return getAmpqUrl(this.conn.amqp);
-	}
+    get ampqUrl() {
+        return getAmpqUrl(this.conn.amqp);
+    }
 
-	calculateServerHash() {
-		exec('git rev-parse HEAD', (err, stdout) => {
-			if (err) {
-				// hLog(`\n ${err.message}\n >>> Failed to check last commit hash. Version hash will be "custom"`);
-				hLog(`Failed to check last commit hash. Version hash will be "custom"`);
-				this.last_commit_hash = 'custom';
-			} else {
-				hLog('Last commit hash on this branch is:', stdout);
-				this.last_commit_hash = stdout.trim();
-			}
-			this.getHyperionVersion();
-		});
-	}
+    calculateServerHash() {
+        exec('git rev-parse HEAD', (err, stdout) => {
+            if (err) {
+                // hLog(`\n ${err.message}\n >>> Failed to check last commit hash. Version hash will be "custom"`);
+                hLog(`Failed to check last commit hash. Version hash will be "custom"`);
+                this.last_commit_hash = 'custom';
+            } else {
+                hLog('Last commit hash on this branch is:', stdout);
+                this.last_commit_hash = stdout.trim();
+            }
+            this.getHyperionVersion();
+        });
+    }
 
-	getServerHash() {
-		return this.last_commit_hash;
-	}
+    getServerHash() {
+        return this.last_commit_hash;
+    }
 
-	getHyperionVersion() {
-		this.current_version = require('../package.json').version;
-		if (this.last_commit_hash === 'custom') {
-			this.current_version = this.current_version + '-dirty';
-		}
-	}
+    getHyperionVersion() {
+        this.current_version = require('../package.json').version;
+        if (this.last_commit_hash === 'custom') {
+            this.current_version = this.current_version + '-dirty';
+        }
+    }
 }

connections/state-history.ts

@@ -6,6 +6,8 @@ export class StateHistorySocket {
 	private ws;
 	private readonly shipUrl;
 	private readonly max_payload_mb;
+	retryOnDisconnect = true;
+	connected = false;
 
 	constructor(ship_url, max_payload_mb) {
 		this.shipUrl = ship_url;
@@ -17,13 +19,13 @@ export class StateHistorySocket {
 	}
 
 	connect(onMessage, onDisconnect, onError, onConnected) {
-
 		debugLog(`Connecting to ${this.shipUrl}...`);
 		this.ws = new WebSocket(this.shipUrl, {
 			perMessageDeflate: false,
 			maxPayload: this.max_payload_mb * 1024 * 1024,
 		});
 		this.ws.on('open', () => {
+			this.connected = true;
 			hLog('Websocket connected!');
 			if (onConnected) {
 				onConnected();
@@ -31,15 +33,21 @@ export class StateHistorySocket {
 		});
 		this.ws.on('message', (data) => onMessage(data));
 		this.ws.on('close', () => {
+			this.connected = false;
 			hLog('Websocket disconnected!');
-			onDisconnect();
+			if(this.retryOnDisconnect) {
+				onDisconnect();
+			}
 		});
 		this.ws.on('error', (err) => {
 			hLog(`${this.shipUrl} :: ${err.message}`);
 		});
 	}
 
-	close() {
+	close(graceful: boolean) {
+		if(graceful) {
+			this.retryOnDisconnect = false;
+		}
 		this.ws.close();
 	}
 

ecosystem.config.js

@@ -5,7 +5,7 @@ const path = require('path');
 const apps = [];
 const chainsRoot = path.join(path.resolve(), 'chains');
 readdirSync(chainsRoot)
-    .filter(f => f.endsWith('.config.json'))
+    .filter(f => f.endsWith('.config.json') && !f.startsWith('example'))
     .forEach(value => {
         const configFile = readFileSync(path.join(chainsRoot, value))
         const config = JSON.parse(configFile.toString());
@@ -20,4 +20,11 @@ readdirSync(chainsRoot)
         }
     });
 
+// apps.push({
+//     name: 'hyperion-governor',
+//     namespace: 'hyperion',
+//     script: 'governor/server/index.js',
+//     watch: false,
+// });
+
 module.exports = {apps};

example-connections.json

@@ -27,8 +27,9 @@
       "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
       "http": "http://127.0.0.1:8888",
       "ship": "ws://127.0.0.1:8080",
+      "WS_ROUTER_HOST": "127.0.0.1",
       "WS_ROUTER_PORT": 7001,
-      "WS_ROUTER_HOST": "127.0.0.1"
+      "control_port": 7002
     }
   }
 }

helpers/common_functions.ts

@@ -6,245 +6,288 @@ import {join} from "path";
 let config;
 const conf_path = join(__dirname, `../${process.env.CONFIG_JSON}`);
 if (existsSync(conf_path)) {
-	try {
-		config = require(conf_path);
-	} catch (e) {
-		console.log(e.message);
-		process.exit(1);
-	}
+    try {
+        config = require(conf_path);
+    } catch (e) {
+        console.log(e.message);
+        process.exit(1);
+    }
 }
 
 if (!config) {
-	console.log(`Configuration not found: ${conf_path}`);
-	process.exit(1);
+    console.log(`Configuration not found: ${conf_path}`);
+    process.exit(1);
 }
 
 function getLastResult(results: ApiResponse) {
-	if (results.body.hits?.hits?.length > 0) {
-		return parseInt(results.body.hits.hits[0].sort[0], 10);
-	} else {
-		return 1;
-	}
+    if (results.body.hits?.hits?.length > 0) {
+        return parseInt(results.body.hits.hits[0].sort[0], 10);
+    } else {
+        return 1;
+    }
 }
 
 export async function getLastIndexedBlockByDelta(es_client: Client, chain: string) {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-delta-*',
-		size: 1,
-		body: {
-			query: {bool: {filter: {match_all: {}}}},
-			sort: [{block_num: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-delta-*',
+        size: 1,
+        body: {
+            query: {bool: {filter: {match_all: {}}}},
+            sort: [{block_num: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export async function getLastIndexedBlock(es_client: Client, chain: string) {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-block-*',
-		size: 1,
-		body: {
-			query: {bool: {filter: {match_all: {}}}},
-			sort: [{block_num: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-block-*',
+        size: 1,
+        body: {
+            query: {bool: {filter: {match_all: {}}}},
+            sort: [{block_num: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export async function getLastIndexedBlockWithTotalBlocks(es_client: Client, chain: string): Promise<[number, number]> {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-block-*',
-		size: 1,
-		body: {
-			query: {bool: {filter: {match_all: {}}}},
-			sort: [{block_num: {order: "desc"}}],
-			track_total_hits: true
-		}
-	});
-	let lastBlock = getLastResult(results);
-	let totalBlocks = results.body.hits.total.value || 1;
-	return [lastBlock, totalBlocks];
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-block-*',
+        size: 1,
+        body: {
+            query: {bool: {filter: {match_all: {}}}},
+            sort: [{block_num: {order: "desc"}}],
+            track_total_hits: true
+        }
+    });
+    let lastBlock = getLastResult(results);
+    let totalBlocks = results.body.hits.total.value || 1;
+    return [lastBlock, totalBlocks];
 }
 
 export async function getFirstIndexedBlock(es_client: Client, chain: string): Promise<number> {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-block-*',
-		size: 1,
-		body: {
-			query: {bool: {filter: {match_all: {}}}},
-			sort: [{block_num: {order: "asc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-block-*',
+        size: 1,
+        body: {
+            query: {bool: {filter: {match_all: {}}}},
+            sort: [{block_num: {order: "asc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 
 export async function getLastIndexedABI(es_client: Client, chain: string) {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-abi-*',
-		size: 1,
-		body: {
-			query: {
-				match_all: {}
-			},
-			sort: [{block: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-abi-*',
+        size: 1,
+        body: {
+            query: {
+                match_all: {}
+            },
+            sort: [{block: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export async function getLastIndexedBlockByDeltaFromRange(es_client: Client, chain: string, first: number, last: number) {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-delta-*',
-		size: 1,
-		body: {
-			query: {
-				range: {
-					block_num: {
-						"gte": first,
-						"lt": last,
-						"boost": 2
-					}
-				}
-			},
-			sort: [{block_num: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-delta-*',
+        size: 1,
+        body: {
+            query: {
+                range: {
+                    block_num: {
+                        "gte": first,
+                        "lt": last,
+                        "boost": 2
+                    }
+                }
+            },
+            sort: [{block_num: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export async function getLastIndexedBlockFromRange(es_client: Client, chain: string, first: number, last: number) {
-	const results = await es_client.search({
-		index: chain + '-block-*',
-		size: 1,
-		body: {
-			query: {
-				range: {
-					block_num: {
-						"gte": first,
-						"lt": last,
-						"boost": 2
-					}
-				}
-			},
-			sort: [{block_num: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results = await es_client.search({
+        index: chain + '-block-*',
+        size: 1,
+        body: {
+            query: {
+                range: {
+                    block_num: {
+                        "gte": first,
+                        "lt": last,
+                        "boost": 2
+                    }
+                }
+            },
+            sort: [{block_num: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export function messageAllWorkers(cl, payload) {
-	for (const c in cl.workers) {
-		if (cl.workers.hasOwnProperty(c)) {
-			const _w = cl.workers[c];
-			if (_w) {
-				try {
-					if (_w.isConnected()) {
-						_w.send(payload);
-					} else {
-						hLog('Worker is not connected!');
-					}
-				} catch (e) {
-					hLog('Failed to message worker!');
-					hLog(e);
-				}
-			} else {
-				hLog('Worker not found!');
-			}
-		}
-	}
+    for (const c in cl.workers) {
+        if (cl.workers.hasOwnProperty(c)) {
+            const _w = cl.workers[c];
+            if (_w) {
+                try {
+                    if (_w.isConnected()) {
+                        _w.send(payload);
+                    } else {
+                        hLog('Worker is not connected!');
+                    }
+                } catch (e) {
+                    hLog('Failed to message worker!');
+                    hLog(e);
+                }
+            } else {
+                hLog('Worker not found!');
+            }
+        }
+    }
 }
 
 export function serialize(type, value, txtEnc, txtDec, types) {
-	const buffer = new Serialize.SerialBuffer({
-		textEncoder: txtEnc,
-		textDecoder: txtDec
-	});
-	Serialize.getType(types, type).serialize(buffer, value);
-	return buffer.asUint8Array();
+    const buffer = new Serialize.SerialBuffer({
+        textEncoder: txtEnc,
+        textDecoder: txtDec
+    });
+    Serialize.getType(types, type).serialize(buffer, value);
+    return buffer.asUint8Array();
 }
 
 export function deserialize(type, array, txtEnc, txtDec, types) {
-	const buffer = new Serialize.SerialBuffer({
-		textEncoder: txtEnc,
-		textDecoder: txtDec,
-		array
-	});
-	return Serialize.getType(types, type).deserialize(buffer, new Serialize.SerializerState({bytesAsUint8Array: true}));
+    const buffer = new Serialize.SerialBuffer({
+        textEncoder: txtEnc,
+        textDecoder: txtDec,
+        array
+    });
+    return Serialize.getType(types, type).deserialize(buffer, new Serialize.SerializerState({bytesAsUint8Array: true}));
 }
 
 function getNested(path_array, jsonObj) {
-	const nextPath = path_array.shift();
-	const nextValue = jsonObj[nextPath];
-	if (!nextValue) {
-		return null;
-	} else {
-		if (typeof nextValue !== 'object') {
-			return nextValue;
-		} else {
-			if (Array.isArray(nextValue)) {
-				return nextValue;
-			} else {
-				return getNested(path_array, nextValue);
-			}
-		}
-	}
+    const nextPath = path_array.shift();
+    const nextValue = jsonObj[nextPath];
+    if (!nextValue) {
+        return null;
+    } else {
+        if (typeof nextValue !== 'object') {
+            return nextValue;
+        } else {
+            if (Array.isArray(nextValue)) {
+                return nextValue;
+            } else {
+                return getNested(path_array, nextValue);
+            }
+        }
+    }
+}
+
+export function checkDeltaFilter(filter, _source) {
+    if (filter.field && filter.value) {
+        let fieldValue = getNested(filter.field.split("."), _source);
+        if (!fieldValue) {
+            const fArray = filter.field.split(".");
+            if (fArray[0].startsWith('@')) {
+                const tableName = fArray[0].replace('@', '');
+                if (_source.table === tableName) {
+                    fArray[0] = 'data';
+                    fieldValue = getNested(fArray, _source);
+                }
+            }
+        }
+        if (fieldValue) {
+            if (Array.isArray(fieldValue)) {
+                return fieldValue.indexOf(filter.value) !== -1;
+            } else {
+                return fieldValue === filter.value;
+            }
+        } else {
+            return !filter.value;
+        }
+    } else {
+        return false;
+    }
 }
 
 export function checkFilter(filter, _source) {
-	if (filter.field && filter.value) {
-		let fieldValue = getNested(filter.field.split("."), _source);
-		if (!fieldValue) {
-			const fArray = filter.field.split(".");
-			if (fArray[0].startsWith('@')) {
-				const actName = fArray[0].replace('@', '');
-				if (_source.act.name === actName) {
-					fArray[0] = 'data';
-					fArray.unshift('act');
-					fieldValue = getNested(fArray, _source);
-				}
-			}
-		}
-		if (fieldValue) {
-			if (Array.isArray(fieldValue)) {
-				return fieldValue.indexOf(filter.value) !== -1;
-			} else {
-				return fieldValue === filter.value;
-			}
-		} else {
-			return !filter.value;
-		}
-	} else {
-		return false;
-	}
+    if (filter.field && filter.value) {
+        let fieldValue = getNested(filter.field.split("."), _source);
+        if (!fieldValue) {
+            const fArray = filter.field.split(".");
+            if (fArray[0].startsWith('@')) {
+                const actName = fArray[0].replace('@', '');
+                if (_source.act.name === actName) {
+                    fArray[0] = 'data';
+                    fArray.unshift('act');
+                    fieldValue = getNested(fArray, _source);
+                }
+            }
+        }
+        if (fieldValue) {
+            if (Array.isArray(fieldValue)) {
+                return fieldValue.indexOf(filter.value) !== -1;
+            } else {
+                return fieldValue === filter.value;
+            }
+        } else {
+            return !filter.value;
+        }
+    } else {
+        return false;
+    }
 }
 
 export function hLog(input: any, ...extra: any[]) {
-	let role;
-	if (process.env.worker_role) {
-		const id = parseInt(process.env.worker_id);
-		role = `[${process.pid} - ${(id < 10 ? '0' : '') + id.toString()}_${process.env.worker_role}]`;
-	} else {
-		if (process.env.script && process.env.script === './api/server.js') {
-			role = `[${process.pid} - api]`;
-		} else {
-			role = `[${process.pid} - 00_master]`;
-		}
-	}
-	if (process.env.TRACE_LOGS === 'true') {
-		const e = new Error();
-		const frame = e.stack.split("\n")[2];
-		const where = frame.split(" ")[6].split(/[:()]/);
-		const arr = where[1].split("/");
-		const fileName = arr[arr.length - 1];
-		const lineNumber = where[2];
-		role += ` ${fileName}:${lineNumber}`;
-	}
-	console.log(role, input, ...extra);
+    let role;
+    if (process.env.worker_role) {
+        const id = parseInt(process.env.worker_id);
+        role = `[${process.pid} - ${(id < 10 ? '0' : '') + id.toString()}_${process.env.worker_role}]`;
+    } else {
+        if (process.env.script && process.env.script === './api/server.js') {
+            role = `[${process.pid} - api]`;
+        } else {
+            role = `[${process.pid} - 00_master]`;
+        }
+    }
+    if (process.env.TRACE_LOGS === 'true') {
+        const e = new Error();
+        const frame = e.stack.split("\n")[2];
+        const where = frame.split(" ")[6].split(/[:()]/);
+        const arr = where[1].split("/");
+        const fileName = arr[arr.length - 1];
+        const lineNumber = where[2];
+        role += ` ${fileName}:${lineNumber}`;
+    }
+    console.log(role, input, ...extra);
 }
 
 export function debugLog(text: any, ...extra: any[]) {
-	if (config.settings.debug) {
-		hLog(text, ...extra);
-	}
+    if (config.settings.debug) {
+        hLog(text, ...extra);
+    }
+}
+
+export async function sleep(ms): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+export async function waitUntilReady(executor: () => Promise<boolean>, attempts: number, interval: number, onError: () => void): Promise<void> {
+    let i = 0;
+    while (i < attempts) {
+        if (await executor()) {
+            return;
+        }
+        await sleep(interval);
+        i++;
+    }
+    onError();
 }

helpers/elastic-routes.ts

@@ -145,6 +145,14 @@ function buildResUsageBulk(payloads, messageMap: MMap) {
     });
 }
 
+function buildScheduleBulk(payloads, messageMap: MMap) {
+    return flatMap(payloads, (payload, body) => {
+        const id = `${body.version}`;
+        messageMap.set(id, _.omit(payload, ['content']));
+        return makeScriptedOp(id, body);
+    });
+}
+
 function buildGenTrxBulk(payloads, messageMap: MMap) {
     return flatMap(payloads, (payload, body) => {
         const hash = createHash('sha256')
@@ -183,6 +191,10 @@ const generatorsMap = {
         index_name: 'userres',
         func: buildResUsageBulk
     },
+    schedule: {
+        index_name: 'schedule',
+        func: buildScheduleBulk
+    },
     generated_transaction: {
         index_name: 'gentrx',
         func: buildGenTrxBulk
@@ -193,21 +205,13 @@ const generatorsMap = {
     },
 };
 
-interface IndexDist {
-    index: string;
-    first_block: number;
-    last_block: number;
-}
-
 export class ElasticRoutes {
     public routes: any;
     cm: ConnectionManager;
     chain: string;
     ingestNodeCounters = {};
-    distributionMap: IndexDist[];
 
-    constructor(connectionManager: ConnectionManager, distributionMap: IndexDist[]) {
-        this.distributionMap = distributionMap;
+    constructor(connectionManager: ConnectionManager) {
         this.routes = {generic: this.handleGenericRoute.bind(this)};
         this.cm = connectionManager;
         this.chain = this.cm.chain;
@@ -382,27 +386,27 @@ export class ElasticRoutes {
         return Math.ceil(blockNum / this.cm.config.settings.index_partition_size).toString().padStart(6, '0');
     }
 
-    getIndexNameByBlock(block_num) {
-        if (!this.distributionMap) {
-            return null;
-        }
-        for (let i = this.distributionMap.length - 1; i >= 0; i--) {
-            const test = this.distributionMap[i].first_block <= block_num && this.distributionMap[i].last_block >= block_num;
-            if (test) {
-                return this.distributionMap[i].index;
-            }
-        }
-        return null;
-    }
-
-    addToIndexMap(map, idx, payload) {
-        if (idx) {
-            if (!map[idx]) {
-                map[idx] = [];
-            }
-            map[idx].push(payload);
-        }
-    }
+    // getIndexNameByBlock(block_num) {
+    //     if (!this.distributionMap) {
+    //         return null;
+    //     }
+    //     for (let i = this.distributionMap.length - 1; i >= 0; i--) {
+    //         const test = this.distributionMap[i].first_block <= block_num && this.distributionMap[i].last_block >= block_num;
+    //         if (test) {
+    //             return this.distributionMap[i].index;
+    //         }
+    //     }
+    //     return null;
+    // }
+    //
+    // addToIndexMap(map, idx, payload) {
+    //     if (idx) {
+    //         if (!map[idx]) {
+    //             map[idx] = [];
+    //         }
+    //         map[idx].push(payload);
+    //     }
+    // }
 
     private routeFactory(indexName: string, bulkGenerator, routerFunction) {
         return async (payloads, channel, cb) => {

hyp-repair

@@ -0,0 +1,2 @@
+#!/bin/bash
+node ./scripts/hyp-repair.js "$@"

interfaces/hyperionConfig.ts

@@ -20,6 +20,7 @@ export interface ScalingConfigs {
 }
 
 export interface MainSettings {
+    use_global_agent?: boolean;
     process_prefix?: string;
     ignore_snapshot?: boolean;
     ship_request_rev: string;
@@ -88,6 +89,8 @@ interface CachedRouteConfig {
 }
 
 interface ApiConfigs {
+    stream_scroll_batch?: number;
+    stream_scroll_limit?: number;
     enabled?: boolean;
     pm2_scaling?: number;
     node_max_old_space_size?: number;

interfaces/hyperionConnections.ts

@@ -23,6 +23,7 @@ interface HyperionChainData {
     ship: string;
     WS_ROUTER_PORT: number;
     WS_ROUTER_HOST: string;
+    control_port: number;
 }
 
 interface RedisConfig {

launcher.ts

@@ -13,6 +13,7 @@ const hyperionWorkers = {
 	reader: 'state-reader',
 	deserializer: 'deserializer',
 	continuous_reader: 'state-reader',
+	repair_reader: 'state-reader',
 	ingestor: 'indexer',
 	router: 'ws-router',
 	delta_updater: 'delta-updater'

modules/action_data/act.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'act',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'act',
+
+    mappings: {
+        action: {
+            "@act": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "act": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@act'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            act: data.act
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/batch.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'batch',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'batch',
+        
+    mappings: {
+        action: {
+            "@batch": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "batch_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@batch'] = {
+            coopname: data.coopname,
+            action: data.action,
+            batch_id: data.batch_id,
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/daobet-eosio-delegatebw.js

@@ -1,45 +0,0 @@
-const hyperionModule = {
-    chain: "74d023a9293d9b68c3c52e2f738ee681c1671cc3dc0f263cf2c533cd5523ff95",
-    contract: 'eosio',
-    action: 'delegatebw',
-    parser_version: ['1.7'],
-    defineQueryPrefix: 'delegatebw',
-    mappings: {
-        action: {
-            "@delegatebw": {
-                "properties": {
-                    "from": {"type": "keyword"},
-                    "receiver": {"type": "keyword"},
-                    "stake_cpu_quantity": {"type": "float"},
-                    "stake_net_quantity": {"type": "float"},
-                    "stake_vote_quantity": {"type": "float"},
-                    "transfer": {"type": "boolean"},
-                    "amount": {"type": "float"}
-                }
-            }
-        }
-    },
-    handler: (action) => {
-        const data = action['act']['data'];
-        let cpu_qtd = null;
-        let net_qtd = null;
-        let vote_qtd = null;
-        if (data['stake_net_quantity'] && data['stake_cpu_quantity'] && data['stake_vote_quantity']) {
-            cpu_qtd = parseFloat(data['stake_cpu_quantity'].split(' ')[0]);
-            net_qtd = parseFloat(data['stake_net_quantity'].split(' ')[0]);
-            vote_qtd = parseFloat(data['stake_vote_quantity'].split(' ')[0]);
-        }
-        action['@delegatebw'] = {
-            amount: cpu_qtd + net_qtd + vote_qtd,
-            stake_cpu_quantity: cpu_qtd,
-            stake_net_quantity: net_qtd,
-            stake_vote_quantity: vote_qtd,
-            from: data['from'],
-            receiver: data['receiver'],
-            transfer: data['transfer']
-        };
-        delete action['act']['data'];
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/daobet-eosio-undelegatebw.js

@@ -1,43 +0,0 @@
-const hyperionModule = {
-    chain: "74d023a9293d9b68c3c52e2f738ee681c1671cc3dc0f263cf2c533cd5523ff95",
-    contract: 'eosio',
-    action: 'undelegatebw',
-    parser_version: ['1.7'],
-    defineQueryPrefix: 'undelegatebw',
-    mappings: {
-        action: {
-            "@undelegatebw": {
-                "properties": {
-                    "from": {"type": "keyword"},
-                    "receiver": {"type": "keyword"},
-                    "unstake_cpu_quantity": {"type": "float"},
-                    "unstake_net_quantity": {"type": "float"},
-                    "unstake_vote_quantity": {"type": "float"},
-                    "amount": {"type": "float"}
-                }
-            }
-        }
-    },
-    handler: (action) => {
-        const data = action['act']['data'];
-        let cpu_qtd = null;
-        let net_qtd = null;
-        let vote_qtd = null;
-        if (data['unstake_net_quantity'] && data['unstake_cpu_quantity'] && data['unstake_vote_quantity']) {
-            cpu_qtd = parseFloat(data['unstake_cpu_quantity'].split(' ')[0]);
-            net_qtd = parseFloat(data['unstake_net_quantity'].split(' ')[0]);
-            vote_qtd = parseFloat(data['unstake_vote_quantity'].split(' ')[0]);
-        }
-        action['@undelegatebw'] = {
-            amount: cpu_qtd + net_qtd + vote_qtd,
-            unstake_cpu_quantity: cpu_qtd,
-            unstake_net_quantity: net_qtd,
-            unstake_vote_quantity: vote_qtd,
-            from: data['from'],
-            receiver: data['receiver']
-        };
-        delete action['act']['data'];
-    }
-};
-
-module.exports = {hyperionModule};

modules/action_data/decision.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'decision',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'decision',
+    
+    mappings: {
+        action: {
+            "@decision": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "decision": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+           
+        action['@decision'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            decision: data.decision
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/eosio-buyram.js

@@ -1,20 +1,20 @@
 const hyperionModule = {
-  chain: '*',
-  contract: 'eosio',
-  action: 'buyram',
-  defineQueryPrefix: 'buyram',
-  parser_version: ['2.1', '1.8', '1.7'],
-  handler: (action) => {
-    const data = action['act']['data'];
-    action['@buyram'] = {
-      payer: data['payer'],
-      receiver: data['receiver'],
-    };
-    if (data['quant']) {
-      action['@buyram']['quant'] = parseFloat(data['quant'].split(' ')[0]);
-    }
-    delete action['act']['data'];
-  },
+    chain: '*',
+    contract: 'eosio',
+    action: 'buyram',
+    defineQueryPrefix: 'buyram',
+    parser_version: ['3.2', '2.1', '1.8', '1.7'],
+    handler: (action) => {
+        const data = action['act']['data'];
+        action['@buyram'] = {
+            payer: data['payer'],
+            receiver: data['receiver'],
+        };
+        if (data['quant']) {
+            action['@buyram']['quant'] = parseFloat(data['quant'].split(' ')[0]);
+        }
+        delete action['act']['data'];
+    },
 };
 
 module.exports = {hyperionModule};

modules/action_data/eosio-buyrambytes.js

@@ -3,7 +3,7 @@ const hyperionModule = {
     contract: 'eosio',
     action: 'buyrambytes',
     defineQueryPrefix: 'buyrambytes',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     handler: (action) => {
         const data = action['act']['data'];
         action['@buyrambytes'] = {

modules/action_data/eosio-buyrex.js

@@ -3,7 +3,7 @@ const hyperionModule = {
     contract: 'eosio',
     action: 'buyrex',
     defineQueryPrefix: 'buyrex',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     handler: (action) => {
         const data = action['act']['data'];
         let qtd = null;

modules/action_data/eosio-delegatebw.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'delegatebw',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'delegatebw',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-newaccount.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'newaccount',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'newaccount',
     handler: (action) => {
         let name = null;

modules/action_data/eosio-undelegatebw.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'undelegatebw',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'undelegatebw',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-unstaketorex.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'unstaketorex',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'unstaketorex',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-updateauth.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'updateauth',
-    parser_version: ['2.1','1.8','1.7'],
+    parser_version: ['3.2', '2.1','1.8','1.7'],
     defineQueryPrefix: 'updateauth',
     handler: (action) => {
         const data = action['act']['data'];

modules/action_data/eosio-voteproducer.js

@@ -2,7 +2,7 @@ const hyperionModule = {
     chain: "*",
     contract: 'eosio',
     action: 'voteproducer',
-    parser_version: ['2.1','1.8', '1.7'],
+    parser_version: ['3.2', '2.1','1.8', '1.7'],
     defineQueryPrefix: 'voteproducer',
     mappings: {
         action: {

modules/action_data/exec.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'exec',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'exec',
+
+    mappings: {
+        action: {
+            "@exec": {
+                "properties": {
+                    "executer": {"type": "keyword"},
+                    "coopname": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@exec'] = {
+            executer: data.executer,
+            coopname: data.coopname,
+            decision_id: data.decision_id,
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/fio-bprewdupdate.js

@@ -2,7 +2,7 @@ const hyperionModule = {
   chain: '21dcae42c0182200e93f954a074011f9048a7624c6fe81d3c9541a614a88bd1c',
   contract: 'fio.treasury',
   action: 'bprewdupdate',
-  parser_version: ['1.8'],
+  parser_version: ['3.2', '1.8'],
   defineQueryPrefix: 'bprewdupdate',
   mappings: {
     action: {

modules/action_data/fio-fdtnrwdupdat.js

@@ -2,7 +2,7 @@ const hyperionModule = {
   chain: '21dcae42c0182200e93f954a074011f9048a7624c6fe81d3c9541a614a88bd1c',
   contract: 'fio.treasury',
   action: 'fdtnrwdupdat',
-  parser_version: ['1.8'],
+  parser_version: ['3.2', '1.8'],
   defineQueryPrefix: 'fdtnrwdupdat',
   mappings: {
     action: {

modules/action_data/statement.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'statement',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'statement',
+
+    mappings: {
+        action: {
+            "@statement": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "statement": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+           
+        action['@statement'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            statement: data.statement
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/transfer.js

@@ -4,7 +4,7 @@ const hyperionModule = {
   chain: '*',
   contract: '*',
   action: 'transfer',
-  parser_version: ['2.1','1.8', '1.7'],
+  parser_version: ['3.2', '2.1', '1.8', '1.7'],
   defineQueryPrefix: 'transfer',
   handler: (action) => {
     let qtd = null;

modules/action_data/voteagainst.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'voteagainst',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'voteagainst',
+
+    mappings: {
+        action: {
+            "@voteagainst": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "member": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@voteagainst'] = {
+            coopname: data.coopname,
+            member: data.member,
+            decision_id: data.decision_id
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/votefor.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'votefor',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'votefor',
+
+    mappings: {
+        action: {
+            "@votefor": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "member": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@votefor'] = {
+            coopname: data.coopname,
+            member: data.member,
+            decision_id: data.decision_id
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/index.js

@@ -53,12 +53,12 @@ class HyperionModuleLoader {
                     const key = `${_module.contract}::${_module.action}`;
                     if (this.chainMappings.has(key)) {
                         if (this.chainMappings.get(key) === '*' && _module.chain === chainID) {
-                            // console.log('Overwriting module ' + key + ' for ' + _module.chain);
+                            console.log('Overwriting module ' + key + ' for ' + _module.chain);
                             this.includeModule(_module);
                             this.chainMappings.set(key, _module.chain);
                         }
                     } else {
-                        // console.log('Including module ' + key + ' for ' + _module.chain);
+                        console.log('Including module ' + key + ' for ' + _module.chain);
                         this.includeModule(_module);
                         this.chainMappings.set(key, _module.chain);
                     }

modules/parsers/1.8-parser.ts

@@ -215,6 +215,7 @@ export default class HyperionParser extends BaseParser {
 						event: 'consumed_block',
 						block_num: result['block_num'],
 						block_id: result['block_id'],
+						block_ts: result['block_ts'],
 						trx_ids: result['trx_ids'],
 						lib: res.last_irreversible.block_num,
 						live: process.env.live_mode

modules/parsers/2.1-parser.ts

@@ -253,6 +253,7 @@ export default class HyperionParser extends BaseParser {
                         event: 'consumed_block',
                         block_num: result['block_num'],
                         block_id: result['block_id'],
+                        block_ts: result['block_ts'],
                         trx_ids: result['trx_ids'],
                         lib: res.last_irreversible.block_num,
                         live: process.env.live_mode

modules/parsers/3.2-parser.ts

@@ -1,31 +1,27 @@
+// noinspection JSUnusedGlobalSymbols
+
 import {BaseParser} from "./base-parser";
 import MainDSWorker from "../../workers/deserializer";
 import {Message} from "amqplib";
 import DSPoolWorker from "../../workers/ds-pool";
 import {TrxMetadata} from "../../interfaces/trx-metadata";
 import {ActionTrace} from "../../interfaces/action-trace";
-import {hLog} from "../../helpers/common_functions";
-import {unzip} from "zlib";
-import {omit} from "lodash";
-
-async function unzipAsync(data) {
-    return new Promise((resolve, reject) => {
-        const buf = Buffer.from(data, 'hex');
-        unzip(buf, (err, result) => {
-            if (err) {
-                reject(err);
-            } else {
-                resolve(result);
-            }
-        })
-    });
-}
+import {deserialize, hLog} from "../../helpers/common_functions";
 
 export default class HyperionParser extends BaseParser {
 
-    flatten = true;
+    flatten = false;
 
-    public async parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded): Promise<boolean> {
+    public async parseAction(
+        worker: DSPoolWorker,
+        ts,
+        action: ActionTrace,
+        trx_data: TrxMetadata,
+        _actDataArray,
+        _processedTraces: ActionTrace[],
+        full_trace,
+        usageIncluded
+    ): Promise<boolean> {
 
         // check filters
         if (this.checkBlacklist(action.act)) return false;
@@ -37,35 +33,32 @@ export default class HyperionParser extends BaseParser {
 
         action["@timestamp"] = ts;
         action.block_num = trx_data.block_num;
+        action.block_id = trx_data.block_id;
         action.producer = trx_data.producer;
         action.trx_id = trx_data.trx_id;
 
+        action.account_ram_deltas = action.account_ram_deltas.filter(value => value.delta !== '0')
+
         if (action.account_ram_deltas.length === 0) {
             delete action.account_ram_deltas;
         }
 
         if (action.except === null) {
-
             if (!action.receipt) {
                 console.log(full_trace.status);
                 console.log(action);
             }
-
             action.receipt = action.receipt[1];
             action.global_sequence = parseInt(action.receipt.global_sequence, 10);
-
             delete action.except;
-            delete action.error_code;
-            delete action.elapsed;
-            delete action.context_free;
-            delete action.level;
-
+            if (action.error_code === null) {
+                delete action.error_code;
+            }
+            // add usage data to the first action on the transaction
             if (!usageIncluded.status) {
                 this.extendFirstAction(worker, action, trx_data, full_trace, usageIncluded);
             }
-
             _processedTraces.push(action);
-
         } else {
             hLog(action);
         }
@@ -75,7 +68,8 @@ export default class HyperionParser extends BaseParser {
 
     public async parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void> {
         for (const message of messages) {
-            const ds_msg = worker.deserializeNative('result', message.content);
+            let allowProcessing = true;
+            const ds_msg = deserialize('result', message.content, this.txEnc, this.txDec, worker.types);
             if (!ds_msg) {
                 if (worker.ch_ready) {
                     worker.ch.nack(message);
@@ -83,34 +77,58 @@ export default class HyperionParser extends BaseParser {
                 }
             }
             const res = ds_msg[1];
-            let block, traces = [], deltas = [];
+            let block = null;
+            let traces = [];
+            let deltas = [];
+
             if (res.block && res.block.length) {
+
                 block = worker.deserializeNative('signed_block', res.block);
-                if (!block) {
-                    hLog('null block', res);
+
+                if (block === null) {
+                    hLog('incompatible block');
                     process.exit(1);
                 }
-            }
-            if (res['traces'] && res['traces'].length) {
-                try {
-                    traces = worker.deserializeNative(
-                        'transaction_trace[]',
-                        await unzipAsync(res['traces'])
-                    );
-                } catch (e) {
-                    hLog(e);
+
+                // verify for whitelisted contracts (root actions only)
+                if (worker.conf.whitelists.root_only) {
+                    try {
+                        if (worker.conf.whitelists && (worker.conf.whitelists.actions.length > 0 || worker.conf.whitelists.deltas.length > 0)) {
+                            allowProcessing = false;
+                            for (const transaction of block.transactions) {
+                                if (transaction.status === 0 && transaction.trx[1] && transaction.trx[1].packed_trx) {
+                                    const unpacked_trx = worker.api.deserializeTransaction(Buffer.from(transaction.trx[1].packed_trx, 'hex'));
+                                    for (const act of unpacked_trx.actions) {
+                                        if (this.checkWhitelist(act)) {
+                                            allowProcessing = true;
+                                            break;
+                                        }
+                                    }
+                                    if (allowProcessing) break;
+                                }
+                            }
+                        }
+                    } catch (e) {
+                        console.log(e);
+                        allowProcessing = true;
+                    }
                 }
             }
-            if (res['deltas'] && res['deltas'].length) {
-                try {
-                    deltas = worker.deserializeNative(
-                        'table_delta[]',
-                        await unzipAsync(res['deltas'])
-                    );
-                } catch (e) {
-                    hLog(e);
+
+            if (allowProcessing && res.traces && res.traces.length) {
+                traces = worker.deserializeNative('transaction_trace[]', res.traces);
+                if (!traces) {
+                    hLog(`[WARNING] transaction_trace[] deserialization failed on block ${res['this_block']['block_num']}`);
                 }
             }
+
+            if (allowProcessing && res.deltas && res.deltas.length) {
+                deltas = deserialize('table_delta[]', res.deltas, this.txEnc, this.txDec, worker.types);
+                if (!deltas) {
+                    hLog(`[WARNING] table_delta[] deserialization failed on block ${res['this_block']['block_num']}`);
+                }
+            }
+
             let result;
             try {
                 result = await worker.processBlock(res, block, traces, deltas);
@@ -119,6 +137,7 @@ export default class HyperionParser extends BaseParser {
                         event: 'consumed_block',
                         block_num: result['block_num'],
                         block_id: result['block_id'],
+                        block_ts: result['block_ts'],
                         trx_ids: result['trx_ids'],
                         lib: res.last_irreversible.block_num,
                         live: process.env.live_mode
@@ -144,28 +163,9 @@ export default class HyperionParser extends BaseParser {
         }
     }
 
-    async flattenInlineActions(action_traces: any[], level: number, trace_counter: any, parent_index: number): Promise<any[]> {
-        const arr = [];
-        const nextLevel = [];
-        for (const action_trace of action_traces) {
-            const trace = action_trace[1];
-            trace['creator_action_ordinal'] = parent_index;
-            trace_counter.trace_index++;
-            trace['level'] = level;
-            trace['action_ordinal'] = trace_counter.trace_index;
-            arr.push(["action_trace_v0", omit(trace, "inline_traces")]);
-            if (trace.inline_traces && trace.inline_traces.length > 0) {
-                nextLevel.push({
-                    traces: trace.inline_traces,
-                    parent: trace_counter.trace_index
-                });
-            }
-        }
-        for (const data of nextLevel) {
-            const appendedArray = await this.flattenInlineActions(data.traces, level + 1, trace_counter, data.parent);
-            arr.push(...appendedArray);
-        }
-        return Promise.resolve(arr);
+    async flattenInlineActions(action_traces: any[]): Promise<any[]> {
+        hLog(`Calling undefined flatten operation!`);
+        return Promise.resolve(undefined);
     }
 
 }

modules/parsers/base-parser.ts

@@ -70,7 +70,13 @@ export abstract class BaseParser {
         return this.filters.action_whitelist.has(this.codeActionPair(act));
     }
 
-    protected extendFirstAction(worker: DSPoolWorker, action: ActionTrace, trx_data: TrxMetadata, full_trace: any, usageIncluded) {
+    protected extendFirstAction(
+        worker: DSPoolWorker,
+        action: ActionTrace,
+        trx_data: TrxMetadata,
+        full_trace: any,
+        usageIncluded: { status: boolean }
+    ) {
         action.cpu_usage_us = trx_data.cpu_usage_us;
         action.net_usage_words = trx_data.net_usage_words;
         action.signatures = trx_data.signatures;
@@ -186,7 +192,6 @@ export abstract class BaseParser {
         if (ds_act) {
 
             if (ds_act.account && ds_act.name && ds_act.authorization) {
-                console.log(ds_act);
                 action.act.data = ds_act.data;
             }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperion-history",
-  "version": "3.3.7",
+  "version": "3.3.9-8",
   "description": "Scalable Full History API Solution for EOSIO based blockchains",
   "main": "launcher.js",
   "scripts": {
@@ -9,11 +9,12 @@
     "start:indexer": "pm2 start --only Indexer --update-env",
     "tsc": "tsc",
     "build": "tsc",
+    "build:watch": "tsc --watch",
     "postinstall": "npm run build && npm run fix-permissions",
     "fix-permissions": "node scripts/fix-permissions.js"
   },
   "engines": {
-    "node": "^18"
+    "node": ">=18"
   },
   "author": {
     "name": "EOS Rio",
@@ -26,22 +27,23 @@
   "license": "MIT",
   "dependencies": {
     "@elastic/elasticsearch": "^7.17.0",
-    "@eosrio/node-abieos": "^2.1.1",
+    "@eosrio/node-abieos": "2.1.1",
     "@fastify/autoload": "4.0.1",
     "@fastify/cors": "7.0.0",
-    "@fastify/formbody": "^6.0.0",
-    "@fastify/rate-limit": "^6.0.0",
+    "@fastify/formbody": "^6.0.1",
+    "@fastify/rate-limit": "^6.0.1",
     "@fastify/redis": "^5.0.0",
     "@fastify/swagger": "6.1.0",
     "@pm2/io": "^5.0.0",
     "amqplib": "^0.10.3",
     "async": "^3.2.4",
     "base-x": "^4.0.0",
+    "cli-progress": "^3.12.0",
     "commander": "^8.3.0",
-    "cross-fetch": "^3.1.5",
+    "cross-fetch": "^3.1.8",
     "eosjs": "^22.1.0",
-    "fast-json-stringify": "^2.7.9",
-    "fastify": "^3.29.3",
+    "fast-json-stringify": "2.7.13",
+    "fastify": "3.29.4",
     "fastify-elasticsearch": "^2.0.0",
     "fastify-plugin": "^3.0.1",
     "flatstr": "^1.0.12",
@@ -50,28 +52,29 @@
     "ioredis": "^4.28.5",
     "lodash": "^4.17.21",
     "moment": "^2.29.4",
-    "nodemailer": "^6.7.8",
+    "nodemailer": "^6.9.0",
+    "pino-pretty": "^10.2.0",
     "portfinder": "^1.0.32",
-    "socket.io": "4.5.3",
-    "socket.io-client": "4.5.3",
+    "socket.io": "4.7.2",
+    "socket.io-client": "4.7.2",
     "socket.io-redis": "^6.1.1",
-    "telegraf": "^4.9.1",
+    "telegraf": "^4.12.3-canary.1",
     "typescript": "^4.5.2",
-    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.15.0",
-    "ws": "^8.11.0"
+    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.31.0",
+    "ws": "^8.14.2"
   },
   "devDependencies": {
-    "@types/amqplib": "^0.8.2",
-    "@types/async": "^3.2.10",
+    "@types/amqplib": "^0.10.2",
+    "@types/async": "^3.2.18",
     "@types/global-agent": "^2.1.1",
-    "@types/ioredis": "^4.28.1",
-    "@types/lodash": "^4.14.177",
-    "@types/node": "^18.11.9",
-    "@types/nodemailer": "^6.4.4",
-    "@types/ws": "^8.5.3"
+    "@types/ioredis": "^4.28.10",
+    "@types/lodash": "^4.14.197",
+    "@types/node": "^20.5.7",
+    "@types/nodemailer": "^6.4.9",
+    "@types/ws": "^8.5.5"
   },
   "optionalDependencies": {
     "bufferutil": "^4.0.7",
-    "utf-8-validate": "^5.0.7"
+    "utf-8-validate": "^6.0.3"
   }
 }

scripts/fix_missing_blocks/fix-missing-blocks.py

@@ -5,6 +5,7 @@ from subprocess import Popen, PIPE, STDOUT
 from sh import tail
 import time
 import sys
+import shutil
 
 
 # Import that you stop the indexer before you start running this script
@@ -41,28 +42,71 @@ live_reader = '.live_reader.*'
 rewrite = '.rewrite.*'
 
 
-shutting_down = r'.*Shutting down master.*'
+#shutting_down = r'.*Shutting down master.*'
+
 offline = r'.*(offline or unexistent)'
 lastblock_indexed = r'.*Last Indexed Block:.* (\d*)'  #group the last block indexed for reference.
 
 
 # Config
-hyperionfolder="/home/charles/hyperion-history-api/"
+hyperionfolder="/home/charles/hyperion/"
 indexer_log_file = "/home/charles/.pm2/logs/wax-indexer-out.log"
+ecosystem_file_location = f'{hyperionfolder}ecosystem.config.js'
 
 chain = "wax"  #set the Chain name eos or wax
 filepath=hyperionfolder+"chains/wax.config.json"
 indexer_stop = ["pm2", "trigger", "wax-indexer", "stop"]
-indexer_start = [hyperionfolder+"./run.sh", "wax-indexer"]
+indexer_start = [f"{hyperionfolder}./run.sh", f'{ecosystem_file_location}', "wax-indexer"]
+#indexer_start = ['/usr/bin/pm2', 'start', {ecosystem_file_location}, '--only', 'wax-indexer', '--update-env', '--silent']
 tail_indexer_logs = ["tail","-f", indexer_log_file]
-hyperion_version = '3.3'  #'3.1' or '3.3'
+hyperion_version = '3.7'  #'3.1' or '3.3'
 
 
 # No blocks being processed differs between 3.3 and 3.1
-if hyperion_version == '3.3':
-    no_blocks_processed = r'.*No blocks are being processed.*'
-else:
+if hyperion_version == '3.7':
+    no_blocks_processed = r'.*-------- BLOCK RANGE COMPLETED -------------.*'
+    shutting_down = r'.*No more active workers, stopping now...*'
+    
+elif hyperion_version == '3.1':
     no_blocks_processed = r'.*No blocks processed.*'
+    shutting_down = r'.*Shutting down master.*'
+else:
+    no_blocks_processed = r'.*-------- BLOCK RANGE COMPLETED -------------.*'
+    shutting_down = r'.*No more active workers, stopping now...*'
+    
+
+def update_run_script():
+    # Copy the original run.sh to run.bak
+    shutil.copy2(f"{hyperionfolder}/run.sh", f"{hyperionfolder}/run.bak")
+    
+    # Update the contents of run.sh with the new command
+    with open("run.sh", "w") as f:
+        f.write(f'''#!/usr/bin/env bash
+if [ $# -lt 2 ]; then
+  echo "Please inform the app name and ecosystem file location. ex: './run.sh indexer /path/to/ecosystem.config.js'"
+  exit 1
+fi
+echo -e "\n-->> Starting $1..."
+(
+  set -x
+  pm2 start "$2" --only "$1" --update-env --silent
+)
+echo -e "\n-->> Saving pm2 state..."
+(
+  set -x
+  pm2 save
+)
+echo -e "\n-->> Reading $2 logs..."
+(
+  set -x
+  pm2 logs --raw --lines 10 "$2"
+)''')
+
+def restore_run_script():
+    # Restore the original run.sh from run.bak
+    shutil.copy2(f"{hyperionfolder}/run.bak", f"{hyperionfolder}/run.sh")
+
+
 
 # Set ES index pattern
 es_index = f'{chain}-block-*'
@@ -208,8 +252,8 @@ def spinning_cursor(run):
       sys.stdout.flush()
 
 def startRewrite(gt,lt):
-    run = Popen(indexer_start,stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    time.sleep(10)
+    Popen(indexer_start,stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    time.sleep(2)
     logtail = Popen(tail_indexer_logs, stdin=PIPE, stdout=PIPE, stderr=STDOUT) 
     completedcount = 0
     for linenum, line in enumerate(logtail.stdout):

scripts/hpm.js

@@ -1,7 +1,11 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];

scripts/hyp-config.js

@@ -1,259 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const commander_1 = require("commander");
-const path_1 = __importDefault(require("path"));
-const promises_1 = require("fs/promises");
-const fs_1 = require("fs");
-const eosjs_1 = require("eosjs");
-const cross_fetch_1 = __importDefault(require("cross-fetch"));
-const ws_1 = __importDefault(require("ws"));
-const program = new commander_1.Command();
-const chainsDir = path_1.default.join(path_1.default.resolve(), 'chains');
-async function getConnections() {
-    const connectionsJsonFile = await (0, promises_1.readFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'));
-    return JSON.parse(connectionsJsonFile.toString());
-}
-async function getExampleConfig() {
-    const exampleChainData = await (0, promises_1.readFile)(path_1.default.join(chainsDir, 'example.config.json'));
-    return JSON.parse(exampleChainData.toString());
-}
-async function listChains(flags) {
-    const dirdata = await (0, promises_1.readdir)(chainsDir);
-    const connections = await getConnections();
-    const exampleChain = await getExampleConfig();
-    const configuredTable = [];
-    const pendingTable = [];
-    for (const file of dirdata.filter(f => f.endsWith('.json'))) {
-        if (file === 'example.config.json') {
-            continue;
-        }
-        try {
-            const fileData = await (0, promises_1.readFile)(path_1.default.join(chainsDir, file));
-            const jsonData = JSON.parse(fileData.toString());
-            const chainName = jsonData.settings.chain;
-            const chainConn = connections.chains[chainName];
-            const fullChainName = jsonData.api.chain_name;
-            // validate configs with example
-            let missingSections = 0;
-            let missingFields = 0;
-            for (const key in exampleChain) {
-                if (!jsonData[key]) {
-                    if (flags.fixMissingFields) {
-                        jsonData[key] = exampleChain[key];
-                    }
-                    else {
-                        console.log(`⚠ Section ${key} missing on ${file}`);
-                        missingSections++;
-                    }
-                }
-                else {
-                    for (const subKey in exampleChain[key]) {
-                        if (!jsonData[key].hasOwnProperty(subKey)) {
-                            if (flags.fixMissingFields) {
-                                jsonData[key][subKey] = exampleChain[key][subKey];
-                            }
-                            else {
-                                console.log(`⚠ Field ${subKey} missing from ${key} on ${file} (default = ${exampleChain[key][subKey]})`);
-                                missingFields++;
-                            }
-                        }
-                    }
-                }
-            }
-            if (flags.fixMissingFields) {
-                await (0, promises_1.writeFile)(path_1.default.join(chainsDir, file), JSON.stringify(jsonData, null, 2));
-            }
-            const common = {
-                full_name: fullChainName,
-                name: chainName,
-                parser: jsonData.settings.parser,
-                abi_scan: jsonData.indexer.abi_scan_mode,
-                live_reader: jsonData.indexer.live_reader,
-                start_on: jsonData.indexer.start_on,
-                stop_on: jsonData.indexer.stop_on
-            };
-            if (chainConn) {
-                configuredTable.push({
-                    ...common,
-                    nodeos_http: chainConn?.http,
-                    nodeos_ship: chainConn?.ship,
-                    chain_id: chainConn?.chain_id.substr(0, 16) + '...'
-                });
-            }
-            else {
-                pendingTable.push(common);
-            }
-        }
-        catch (e) {
-            console.log(`Failed to parse ${file} - ${e.message}`);
-        }
-    }
-    console.log(' >>>> Fully Configured Chains');
-    console.table(configuredTable);
-    if (!flags.valid) {
-        console.log(' >>>> Pending Configuration Chains');
-        console.table(pendingTable);
-    }
-}
-async function newChain(shortName, options) {
-    console.log(`Creating new config for ${shortName}...`);
-    const targetPath = path_1.default.join(chainsDir, `${shortName}.config.json`);
-    if ((0, fs_1.existsSync)(targetPath)) {
-        console.error(`Chain config for ${shortName} already defined!`);
-        process.exit(0);
-    }
-    // read example
-    const exampleChain = await getExampleConfig();
-    // read connections.json
-    const connections = await getConnections();
-    if (connections.chains[shortName]) {
-        console.log('Connections already defined!');
-        console.log(connections.chains[shortName]);
-        process.exit(0);
-    }
-    else {
-        connections.chains[shortName] = {
-            name: '',
-            ship: '',
-            http: '',
-            chain_id: '',
-            WS_ROUTER_HOST: '127.0.0.1',
-            WS_ROUTER_PORT: 7001
-        };
-    }
-    const jsonData = exampleChain;
-    jsonData.settings.chain = shortName;
-    if (options.http) {
-        if (options.http.startsWith('http://') || options.http.startsWith('https://')) {
-            console.log(`Testing connection on ${options.http}`);
-        }
-        else {
-            console.error(`Invalid HTTP Endpoint [${options.http}] - Url must start with either http:// or https://`);
-            process.exit(1);
-        }
-        // test nodeos availability
-        const jsonRpc = new eosjs_1.JsonRpc(options.http, { fetch: cross_fetch_1.default });
-        const info = await jsonRpc.get_info();
-        jsonData.api.chain_api = options.http;
-        connections.chains[shortName].chain_id = info.chain_id;
-        connections.chains[shortName].http = options.http;
-        if (info.server_version_string.includes('2.1')) {
-            jsonData.settings.parser = '2.1';
-        }
-        else {
-            jsonData.settings.parser = '1.8';
-        }
-        console.log(`Parser version set to ${jsonData.settings.parser}`);
-        console.log(`Chain ID: ${info.chain_id}`);
-    }
-    if (options.ship) {
-        if (options.ship.startsWith('ws://') || options.ship.startsWith('wss://')) {
-            console.log(`Testing connection on ${options.ship}`);
-        }
-        else {
-            console.error(`Invalid WS Endpoint [${options.ship}] - Url must start with either ws:// or wss://`);
-            process.exit(1);
-        }
-        // test ship availability
-        const status = await new Promise(resolve => {
-            const ws = new ws_1.default(options.ship);
-            ws.on("message", (data) => {
-                try {
-                    const abi = JSON.parse(data.toString());
-                    if (abi.version) {
-                        console.log(`Received SHIP Abi Version: ${abi.version}`);
-                        resolve(true);
-                    }
-                    else {
-                        resolve(false);
-                    }
-                }
-                catch (e) {
-                    console.log(e.message);
-                    resolve(false);
-                }
-                ws.close();
-            });
-            ws.on("error", err => {
-                console.log(err);
-                ws.close();
-                resolve(false);
-            });
-        });
-        if (status) {
-            connections.chains[shortName].ship = options.ship;
-        }
-        else {
-            console.log(`Invalid SHIP Endpoint [${options.ship}]`);
-        }
-    }
-    const fullNameArr = [];
-    shortName.split('-').forEach((word) => {
-        fullNameArr.push(word[0].toUpperCase() + word.substr(1));
-    });
-    const fullChainName = fullNameArr.join(' ');
-    jsonData.api.chain_name = fullChainName;
-    connections.chains[shortName].name = fullChainName;
-    console.log(connections.chains[shortName]);
-    console.log('Saving connections.json...');
-    await (0, promises_1.writeFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
-    console.log(`Saving chains/${shortName}.config.json...`);
-    await (0, promises_1.writeFile)(targetPath, JSON.stringify(jsonData, null, 2));
-}
-async function rmChain(shortName) {
-    console.log(`Removing config for ${shortName}...`);
-    const targetPath = path_1.default.join(chainsDir, `${shortName}.config.json`);
-    if (!(0, fs_1.existsSync)(targetPath)) {
-        console.error(`Chain config for ${shortName} not found!`);
-        process.exit(0);
-    }
-    // create backups
-    const backupDir = path_1.default.join(path_1.default.resolve(), 'configuration_backups');
-    if (!(0, fs_1.existsSync)(backupDir)) {
-        await (0, promises_1.mkdir)(backupDir);
-    }
-    const connections = await getConnections();
-    try {
-        const chainConn = connections.chains[shortName];
-        const now = Date.now();
-        if (chainConn) {
-            await (0, promises_1.writeFile)(path_1.default.join(backupDir, `${shortName}_${now}_connections.json`), JSON.stringify(chainConn, null, 2));
-        }
-        await (0, promises_1.cp)(targetPath, path_1.default.join(backupDir, `${shortName}_${now}_config.json`));
-    }
-    catch (e) {
-        console.log(e.message);
-        console.log('Failed to create backups! Aborting!');
-        process.exit(1);
-    }
-    console.log(`Deleting ${targetPath}`);
-    await (0, promises_1.rm)(targetPath);
-    delete connections.chains[shortName];
-    console.log('Saving connections.json...');
-    await (0, promises_1.writeFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
-    console.log(`✅ ${shortName} removal completed!`);
-}
-// main program
-(() => {
-    const list = program.command('list').alias('ls');
-    list.command('chains')
-        .option('--valid', 'only show valid chains')
-        .option('--fix-missing-fields', 'set defaults on missing fields')
-        .description('list configured chains')
-        .action(listChains);
-    const newCmd = program.command('new').alias('n');
-    newCmd.command('chain <shortName>')
-        .description('initialize new chain config based on example')
-        .option('--http <http_endpoint>', 'define chain api http endpoint')
-        .option('--ship <ship_endpoint>', 'define state history ws endpoint')
-        .action(newChain);
-    const remove = program.command('remove').alias('rm');
-    remove.command('chain <shortName>')
-        .description('backup and delete chain configuration')
-        .action(rmChain);
-    program.parse(process.argv);
-})();
-//# sourceMappingURL=hyp-config.js.map

scripts/hyp-config.ts

@@ -3,17 +3,28 @@ import path from "path";
 import {cp, mkdir, readdir, readFile, rm, writeFile} from "fs/promises";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
 import {HyperionConnections} from "../interfaces/hyperionConnections";
-import {existsSync} from "fs";
+import {copyFileSync, existsSync, rmSync} from "fs";
 import {JsonRpc} from "eosjs";
 import fetch from "cross-fetch";
 import WebSocket from 'ws';
+import * as readline from "readline";
+import * as amqp from "amqplib";
+import IORedis from "ioredis";
+import {Client} from "@elastic/elasticsearch";
 
 const program = new Command();
 const chainsDir = path.join(path.resolve(), 'chains');
+const connectionsPath = path.join(path.resolve(), 'connections.json');
+const exampleConnectionsPath = path.join(path.resolve(), 'example-connections.json');
+const backupDir = path.join(path.resolve(), 'configuration_backups');
 
 async function getConnections(): Promise<HyperionConnections> {
-    const connectionsJsonFile = await readFile(path.join(path.resolve(), 'connections.json'));
-    return JSON.parse(connectionsJsonFile.toString());
+    if (existsSync(connectionsPath)) {
+        const connectionsJsonFile = await readFile(connectionsPath);
+        return JSON.parse(connectionsJsonFile.toString());
+    } else {
+        return null;
+    }
 }
 
 async function getExampleConfig(): Promise<HyperionConfig> {
@@ -85,7 +96,7 @@ async function listChains(flags) {
                     ...common,
                     nodeos_http: chainConn?.http,
                     nodeos_ship: chainConn?.ship,
-                    chain_id: chainConn?.chain_id.substr(0,16) + '...'
+                    chain_id: chainConn?.chain_id.substr(0, 16) + '...'
                 });
             } else {
                 pendingTable.push(common);
@@ -119,6 +130,11 @@ async function newChain(shortName, options) {
     // read connections.json
     const connections = await getConnections();
 
+    if (!connections) {
+        console.log(`The connections.json file is not present, please run "./hyp-config connections init" to configure it.`);
+        process.exit(1);
+    }
+
     if (connections.chains[shortName]) {
         console.log('Connections already defined!');
         console.log(connections.chains[shortName]);
@@ -130,7 +146,8 @@ async function newChain(shortName, options) {
             http: '',
             chain_id: '',
             WS_ROUTER_HOST: '127.0.0.1',
-            WS_ROUTER_PORT: 7001
+            WS_ROUTER_PORT: 7001,
+            control_port: 7002
         };
     }
 
@@ -147,18 +164,25 @@ async function newChain(shortName, options) {
         }
 
         // test nodeos availability
-        const jsonRpc = new JsonRpc(options.http, {fetch});
-        const info = await jsonRpc.get_info();
-        jsonData.api.chain_api = options.http;
-        connections.chains[shortName].chain_id = info.chain_id;
-        connections.chains[shortName].http = options.http;
-        if (info.server_version_string.includes('2.1')) {
-            jsonData.settings.parser = '2.1';
-        } else {
-            jsonData.settings.parser = '1.8';
+        try {
+            const jsonRpc = new JsonRpc(options.http, {fetch});
+            const info = await jsonRpc.get_info();
+            jsonData.api.chain_api = options.http;
+            connections.chains[shortName].chain_id = info.chain_id;
+            connections.chains[shortName].http = options.http;
+            if (info.server_version_string.includes('v3')) {
+                jsonData.settings.parser = '3.2';
+            } else if (info.server_version_string.includes('v2.1')) {
+                jsonData.settings.parser = '2.1';
+            } else {
+                jsonData.settings.parser = '1.8';
+            }
+            console.log(`Parser version set to ${jsonData.settings.parser}`);
+            console.log(`Chain ID: ${info.chain_id}`);
+        } catch (e) {
+            console.error(`Invalid HTTP Endpoint [${options.http}] - ${e.message}`);
+            process.exit(1);
         }
-        console.log(`Parser version set to ${jsonData.settings.parser}`);
-        console.log(`Chain ID: ${info.chain_id}`)
     }
 
     if (options.ship) {
@@ -204,7 +228,7 @@ async function newChain(shortName, options) {
     const fullNameArr = [];
     shortName.split('-').forEach((word: string) => {
         fullNameArr.push(word[0].toUpperCase() + word.substr(1));
-        });
+    });
     const fullChainName = fullNameArr.join(' ');
 
     jsonData.api.chain_name = fullChainName;
@@ -213,7 +237,7 @@ async function newChain(shortName, options) {
     console.log(connections.chains[shortName]);
 
     console.log('Saving connections.json...');
-    await writeFile(path.join(path.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    await writeFile(connectionsPath, JSON.stringify(connections, null, 2));
     console.log(`Saving chains/${shortName}.config.json...`);
     await writeFile(targetPath, JSON.stringify(jsonData, null, 2));
 }
@@ -228,7 +252,6 @@ async function rmChain(shortName) {
     }
 
     // create backups
-    const backupDir = path.join(path.resolve(), 'configuration_backups');
     if (!existsSync(backupDir)) {
         await mkdir(backupDir);
     }
@@ -254,12 +277,310 @@ async function rmChain(shortName) {
     await rm(targetPath);
     delete connections.chains[shortName];
     console.log('Saving connections.json...');
-    await writeFile(path.join(path.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    await writeFile(connectionsPath, JSON.stringify(connections, null, 2));
     console.log(`✅ ${shortName} removal completed!`);
 }
 
+async function getExampleConnections(): Promise<HyperionConnections | null> {
+    try {
+        const connectionsJsonFile = await readFile(exampleConnectionsPath);
+        return JSON.parse(connectionsJsonFile.toString());
+    } catch (e: any) {
+        return null;
+    }
+}
+
+async function checkAMQP(conn: HyperionConnections): Promise<boolean> {
+    console.log(`\n[info] [RABBIT:AMQP] - Testing rabbitmq amqp connection...`);
+    let frameMaxValue = '0x10000';
+    if (conn.amqp.frameMax) {
+        frameMaxValue = conn.amqp.frameMax;
+    }
+    const u = encodeURIComponent(conn.amqp.user);
+    const p = encodeURIComponent(conn.amqp.pass);
+    const v = encodeURIComponent(conn.amqp.vhost)
+    const amqp_url = `amqp://${u}:${p}@${conn.amqp.host}/${v}?frameMax=${frameMaxValue}`;
+    try {
+        const connection = await amqp.connect(amqp_url);
+        if (connection) {
+            console.log('[info] [RABBIT:AMQP] - Connection established!');
+            await connection.close();
+            console.log(`\n[info] [RABBIT:HTTP] - Testing rabbitmq http api connection...`);
+            const apiUrl = `${conn.amqp.protocol}://${conn.amqp.api}/api/nodes`;
+
+            let headers = new Headers();
+            headers.set('Authorization', 'Basic ' + btoa(conn.amqp.user + ":" + conn.amqp.pass));
+            const data = await fetch(apiUrl, {
+                method: 'GET',
+                headers: headers
+            });
+            const resp = await data.json();
+            if (resp.length > 0) {
+                for (const node of resp) {
+                    console.log(`[info] [RABBIT:HTTP] - Node name: ${node.name} (running: ${node.running})`);
+                    if (!node.running) {
+                        return false;
+                    }
+                }
+                return true;
+            } else {
+                return false;
+            }
+        } else {
+            return false;
+        }
+    } catch (reason: any) {
+        console.log('[error] [RABBIT] - ' + reason.message);
+        return false;
+    }
+}
+
+async function checkES(conn: HyperionConnections): Promise<boolean> {
+    console.log(`\n[info] [ES] - Testing elasticsearch connection...`);
+    let es_url;
+    const _es = conn.elasticsearch;
+    if (!_es.protocol) {
+        _es.protocol = 'http';
+    }
+    if (_es.user !== '') {
+        es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${_es.host}`;
+    } else {
+        es_url = `${_es.protocol}://${_es.host}`
+    }
+    // console.log(`Prepared client: ${es_url}`);
+    const client = new Client({
+        node: es_url,
+        ssl: {rejectUnauthorized: false}
+    });
+    try {
+        const result = await client.cat.health();
+        console.log(`[info] [ES] - ${result.body.trim()}`);
+        console.log('[info] [ES] - Connection established!');
+        return true;
+    } catch (reason: any) {
+        console.log('[error] [ES] - ' + reason.message);
+        return false;
+    }
+}
+
+async function checkRedis(conn: HyperionConnections) {
+    console.log(`\n[info] [REDIS] - Testing redis connection...`);
+    try {
+        const ioRedisClient = new IORedis(conn.redis);
+        const pingResult = await ioRedisClient.ping();
+        if (pingResult === 'PONG') {
+            console.log('[info] [REDIS] - Connection established!');
+            ioRedisClient.disconnect();
+            return true;
+        } else {
+            console.log('[error] [REDIS] - PONG expected, got: ' + pingResult);
+            return false;
+        }
+    } catch (reason: any) {
+        console.log('[error] [REDIS] - ' + reason.message);
+        return false;
+    }
+}
+
+async function initConfig() {
+    const connections = await getConnections();
+
+    if (connections) {
+        console.log('connections.json already created!\n use "./hyp-config connections reset" to revert to default');
+        return;
+    }
+
+    const exampleConn = await getExampleConnections();
+
+    const rl = readline.createInterface({input: process.stdin, output: process.stdout});
+    const prompt = (query: string) => new Promise((resolve) => rl.question(query, resolve));
+
+    const conn = exampleConn;
+
+    if (!conn) {
+        console.log('No example-connections.json to use as reference!');
+        process.exit(1);
+    }
+    conn.chains = {};
+
+    // check rabbitmq
+    let amqp_state = false;
+    while (!amqp_state) {
+        amqp_state = await checkAMQP(conn);
+        if (!amqp_state) {
+
+            const amqp_user = await prompt('\n > Enter the RabbitMQ user (or press ENTER to use "hyperion_user"): ');
+            if (amqp_user) {
+                conn.amqp.user = amqp_user as string;
+            } else {
+                conn.amqp.user = 'hyperion_user'
+            }
+
+            const amqp_pass = await prompt(' > Enter the RabbitMQ password (or press ENTER to use "hyperion_password"): ');
+            if (amqp_pass) {
+                conn.amqp.pass = amqp_pass as string;
+            } else {
+                conn.amqp.pass = 'hyperion_password'
+            }
+
+            const amqp_vhost = await prompt(' > Enter the RabbitMQ vhost (or press ENTER to use "hyperion"): ');
+            if (amqp_vhost) {
+                conn.amqp.vhost = amqp_vhost as string;
+            } else {
+                conn.amqp.vhost = 'hyperion'
+            }
+
+            console.log('\n------ current RabbitMQ config -----');
+            console.log(conn.amqp);
+            console.log('--------------------------------------');
+        }
+    }
+
+    console.log(`\n RabbitMQ ✅`);
+
+    // check elasticsearch
+    let elastic_state = false;
+    while (!elastic_state) {
+        elastic_state = await checkES(conn);
+        if (!elastic_state) {
+
+            const es_user = await prompt('\n > Enter the elasticsearch user (or press ENTER to use "elastic"): ');
+            if (es_user) {
+                conn.elasticsearch.user = es_user as string;
+            } else {
+                conn.elasticsearch.user = 'elastic'
+            }
+
+            const es_pass = await prompt(' > Enter the elasticsearch password: ');
+            if (es_pass) {
+                conn.elasticsearch.pass = es_pass as string;
+            }
+
+            const es_proto = await prompt(' > Do you want to use http or https?\n1 = http\n2 = https (default)\n');
+            if (es_proto === "1") {
+                conn.elasticsearch.protocol = 'http';
+            } else {
+                conn.elasticsearch.protocol = 'https';
+            }
+
+            console.log('\n------ current elasticsearch config -----');
+            console.log(conn.elasticsearch);
+            console.log('-------------------------------------------');
+        }
+    }
+
+    console.log(`\n Elasticsearch ✅`);
+
+    // check redis connection
+    let redis_state = false;
+    while (!redis_state) {
+        redis_state = await checkRedis(conn);
+        if (!redis_state) {
+            const redis_host = await prompt('\n > Enter the Redis Server address (or press ENTER to use "127.0.0.1"): ');
+            if (redis_host) {
+                conn.redis.host = redis_host as string;
+            } else {
+                conn.redis.host = '127.0.0.1';
+            }
+
+            const redis_port = await prompt(' > Enter the Redis Server port (or press ENTER to use "6379"): ');
+            if (redis_port) {
+                conn.redis.port = parseInt(redis_port as string);
+            } else {
+                conn.redis.port = 6379;
+            }
+
+            console.log('\n------ current redis config -----');
+            console.log(conn.redis);
+            console.log('-----------------------------------');
+        }
+    }
+
+    console.log(`\n Redis ✅`);
+
+    console.log('Init completed! Saving configuration...');
+
+    await writeFile(connectionsPath, JSON.stringify(conn, null, 2));
+
+    console.log('✅ ✅');
+    rl.close();
+}
+
+async function testConnections() {
+    if (existsSync(connectionsPath)) {
+        try {
+            const conn = await getConnections();
+            if (conn) {
+                const results = {
+                    redis: await checkRedis(conn),
+                    elasticsearch: await checkES(conn),
+                    rabbitmq: await checkAMQP(conn)
+                }
+                console.log('\n------ Testing Completed -----');
+                console.table(results);
+            } else {
+                console.log(`The connections.json was not found!.`);
+            }
+        } catch (e: any) {
+            console.log(e.message);
+        }
+    } else {
+        console.log(`The connections.json file is not present, please run "./hyp-config connections init" to configure it.`);
+    }
+}
+
+async function resetConnections() {
+    try {
+
+        // create backups
+        if (!existsSync(backupDir)) {
+            await mkdir(backupDir);
+        }
+
+        if (existsSync(connectionsPath)) {
+            const rl = readline.createInterface({input: process.stdin, output: process.stdout});
+            const prompt = (query: string) => new Promise((resolve) => rl.question(query, resolve));
+            const confirmation = await prompt('Are you sure you want to reset the connection configuration? Type "YES" to confirm.\n');
+            if (confirmation === 'YES') {
+                copyFileSync(connectionsPath, path.join(backupDir, 'connections.json.bak'));
+                rmSync(connectionsPath);
+                console.log('connections.json removed, please use "./hyp-config connections init" to reconfigure');
+            } else {
+                console.log('Operation canceled. No files were removed.');
+            }
+            rl.close();
+        } else {
+            console.log(`The connections.json file is not present, please run "./hyp-config connections init" to configure it.`);
+        }
+    } catch (e: any) {
+        console.log(e);
+    }
+}
+
+
 // main program
 (() => {
+
+    const connections = program.command('connections');
+
+    // ./hyp-config connections init
+    connections
+        .command('init')
+        .description('create and test the connections.json file')
+        .action(initConfig);
+
+    // ./hyp-config connections test
+    connections
+        .command('test')
+        .description('test connections to the hyperion infrastructure')
+        .action(testConnections);
+
+    // ./hyp-config connections reset
+    connections
+        .command('reset')
+        .description('remove the connections.json file')
+        .action(resetConnections);
+
     const list = program.command('list').alias('ls');
     list.command('chains')
         .option('--valid', 'only show valid chains')

scripts/hyp-repair.ts

@@ -0,0 +1,954 @@
+import { Command } from 'commander';
+import { readFileSync } from 'node:fs';
+import { Client, estypes } from '@elastic/elasticsearch';
+// @ts-ignore
+import cliProgress from 'cli-progress';
+import { JsonRpc } from 'eosjs';
+import fetch from 'cross-fetch';
+import { existsSync, mkdirSync, writeFileSync } from 'fs';
+import { HyperionBlock } from './repair-cli/interfaces.js';
+import {
+    getBlocks,
+    getFirstIndexedBlock,
+    getLastIndexedBlock,
+    initESClient,
+    readChainConfig,
+    readConnectionConfig,
+} from './repair-cli/functions.js';
+import { SearchResponse } from '@elastic/elasticsearch/api/types';
+
+import { WebSocket } from 'ws';
+import { block } from '../definitions/index-templates';
+
+const progressBar = new cliProgress.SingleBar(
+    {},
+    cliProgress.Presets.shades_classic
+);
+const program = new Command();
+const errorRanges: any[] = [];
+let pendingBlock: HyperionBlock | null = null;
+
+const missingBlocks: {
+    start: number;
+    end: number;
+}[] = [];
+
+async function run(
+    client: Client,
+    rpc: JsonRpc,
+    indexName: string,
+    blockInit: number,
+    firstBlock: number,
+    qtdTotal: any,
+    loop: any = 1
+) {
+    let blockInitial = blockInit;
+    let blockFinal = blockInitial - qtdTotal;
+    const tRef = process.hrtime.bigint();
+    progressBar.start(loop, 0);
+    for (let i: any = 1; i <= loop; i++) {
+        if (blockFinal < firstBlock) {
+            blockFinal = firstBlock;
+        }
+        try {
+            let { body } = await getBlocks(
+                client,
+                indexName,
+                blockInitial,
+                blockFinal,
+                qtdTotal
+            );
+            let {
+                hits: { hits },
+            } = body;
+            const blocks = hits.map((obj: any) => obj._source);
+            await findForksOnRange(blocks, rpc);
+            blockInitial = blockFinal;
+            blockFinal = blockInitial - qtdTotal;
+            progressBar.update(i);
+        } catch (e: any) {
+            console.log('Error: ', e);
+            process.exit(1);
+        }
+    }
+    progressBar.stop();
+    console.log('=========== Forked Ranges ===========');
+    console.table(errorRanges);
+    console.log('=========== Missing Ranges ==========');
+    console.table(missingBlocks);
+    const tDiff = Number(process.hrtime.bigint() - tRef) / 1000000;
+    console.log(`Total time: ${Math.round(tDiff / 1000)}s`);
+}
+
+async function findForksOnRange(blocks: HyperionBlock[], rpc: JsonRpc) {
+    const removals: Set<string> = new Set();
+    let start = null;
+    let end = null;
+    if (
+        blocks.length > 0 &&
+        pendingBlock &&
+        blocks[0].block_num !== pendingBlock.block_num
+    ) {
+        blocks.unshift(pendingBlock);
+    }
+
+    let i = 0;
+    for (const currentBlock of blocks) {
+        const currentBlockNumber = currentBlock.block_num;
+        // console.log('current -> ', currentBlockNumber);
+        const previousBlock = blocks[i + 1];
+        if (!previousBlock) {
+            pendingBlock = currentBlock;
+            // console.log('pending -> ', pendingBlock.block_num);
+            return;
+        }
+        i++;
+        // console.log('previous -> ', previousBlock.block_num);
+        if (previousBlock.block_num !== currentBlockNumber - 1) {
+            console.log(
+                `\nBlock number mismatch, expected: ${
+                    currentBlockNumber - 1
+                } got ${previousBlock.block_num}`
+            );
+            missingBlocks.push({
+                start: previousBlock.block_num + 1,
+                end: currentBlockNumber - 1,
+            });
+            continue;
+        }
+
+        if (previousBlock && previousBlock.block_id !== currentBlock.prev_id) {
+            if (start === null) {
+                start = currentBlockNumber - 1;
+                const blockData = await rpc.get_block_info(start);
+                if (blockData && blockData.id !== previousBlock.block_id) {
+                    removals.add(previousBlock.block_id);
+                }
+            }
+        } else {
+            if (start) {
+                const blockData = await rpc.get_block_info(currentBlockNumber);
+                if (blockData) {
+                    if (blockData.id !== currentBlock.block_id) {
+                        removals.add(currentBlock.block_id);
+                    } else {
+                        end = currentBlockNumber + 1;
+                        const range = { start, end, ids: [...removals] };
+                        errorRanges.push(range);
+                        removals.clear();
+                        // console.log(`\n ⚠️⚠️ Forked at ${range.start} to ${range.end}`);
+                        start = null;
+                        end = null;
+                    }
+                }
+            }
+        }
+    }
+}
+
+async function scanChain(chain: string, args: any) {
+    const chainConfig = readChainConfig(chain);
+
+    console.log(chainConfig.settings.index_version);
+
+    const config = readConnectionConfig();
+
+    const client = initESClient(config);
+
+    const jsonRpc = new JsonRpc(config.chains[chain].http, { fetch });
+
+    const ping = await client.ping();
+
+    if (!ping) {
+        console.log('Could not connect to ElasticSearch');
+        process.exit();
+    }
+
+    const blockIndex = `${chain}-block-${chainConfig.settings.index_version}`;
+
+    let firstBlock;
+    let lastBlock;
+
+    if (args.first) {
+        // reduce by 2 to account for the fork end validation
+        firstBlock = args.first - 2;
+    } else {
+        firstBlock = await getFirstIndexedBlock(client, blockIndex);
+    }
+
+    if (args.last) {
+        lastBlock = args.last;
+    } else {
+        lastBlock = await getLastIndexedBlock(client, blockIndex);
+    }
+
+    const totalBlocks = lastBlock - firstBlock;
+
+    let batchSize = 1000;
+    if (args.batch) {
+        batchSize = args.batch;
+    }
+
+    const numberOfBatches = Math.ceil(totalBlocks / batchSize);
+
+    console.log('Range:', firstBlock, lastBlock);
+    console.log('Total Blocks:', totalBlocks);
+    console.log('Batch Size:', batchSize);
+    console.log('Number of Batches:', numberOfBatches);
+
+    await run(
+        client,
+        jsonRpc,
+        blockIndex,
+        lastBlock,
+        firstBlock,
+        batchSize,
+        numberOfBatches
+    );
+    console.log(`Finished checking forked blocks!`);
+
+    if (errorRanges.length > 0 || missingBlocks.length > 0) {
+        if (!existsSync('.repair')) {
+            mkdirSync('.repair');
+        }
+    }
+
+    if (errorRanges.length > 0) {
+        let path = `.repair/${chain}-${
+            firstBlock + 2
+        }-${lastBlock}-forked-blocks.json`;
+        if (args.outFile) {
+            path = args.outFile + '-forked-blocks.json';
+        }
+        writeFileSync(path, JSON.stringify(errorRanges));
+    }
+
+    if (missingBlocks.length > 0) {
+        let path = `.repair/${chain}-${
+            firstBlock + 2
+        }-${lastBlock}-missing-blocks.json`;
+        if (args.outFile) {
+            path = args.outFile + '-missing-blocks.json';
+        }
+        writeFileSync(path, JSON.stringify(missingBlocks));
+    }
+}
+
+async function repairMissing(chain: string, file: string, args: any) {
+    const chainConfig = readChainConfig(chain);
+    const config = readConnectionConfig();
+    const client = initESClient(config);
+    const ping = await client.ping();
+    if (!ping) {
+        console.log('Could not connect to ElasticSearch');
+        process.exit();
+    }
+    await fillMissingBlocksFromFile(args.host, chain, file, args.dry);
+}
+
+async function repairChain(chain: string, file: string, args: any) {
+    const chainConfig = readChainConfig(chain);
+    const config = readConnectionConfig();
+    const client = initESClient(config);
+    const ping = await client.ping();
+    if (!ping) {
+        console.log('Could not connect to ElasticSearch');
+        process.exit();
+    }
+
+    if (args.checkTasks) {
+        const tasks = await client.tasks.list();
+        if (tasks.body && tasks.body.nodes) {
+            for (let node in tasks.body.nodes) {
+                const nodeTasks = tasks.body.nodes[node].tasks;
+                console.log(nodeTasks);
+            }
+        }
+        process.exit(0);
+    }
+
+    const forkedBlocks = JSON.parse(readFileSync(file).toString());
+    const blockIndex = `${chain}-block-${chainConfig.settings.index_version}`;
+    let deleteBlocks = 0;
+    let deleteActions = 0;
+    let deleteDeltas = 0;
+    let deleteAbis = 0;
+    let deleteAccounts = 0;
+    let deleteVoters = 0;
+    let deleteProposals = 0;
+    let deleteLinks = 0;
+    let deletePermissions = 0;
+
+    for (const range of forkedBlocks) {
+        // ACTIONS
+        const searchActions = {
+            index: `${chain}-action-${chainConfig.settings.index_version}-*`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
+
+        if (args.dry) {
+            const resultActions = await client.search<any>(searchActions);
+            if (
+                (resultActions.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                deleteActions += (
+                    resultActions.body.hits.total as estypes.SearchTotalHits
+                )?.value;
+            }
+        } else {
+            const indexExists = await client.indices.exists({
+                index: searchActions.index,
+            });
+
+            if (indexExists.body) {
+                delete searchActions.size;
+                const deletedActionsResult = await client.deleteByQuery(
+                    searchActions
+                );
+                if (
+                    deletedActionsResult &&
+                    deletedActionsResult.body.deleted &&
+                    deletedActionsResult.body.deleted > 0
+                ) {
+                    deleteActions += deletedActionsResult.body.deleted;
+                }
+            } else {
+                console.log(
+                    `Index ${searchActions.index} doesn't exist. Unable to delete.`
+                );
+            }
+        }
+
+        // DELTAS
+        const searchDeltas = {
+            index: `${chain}-delta-${chainConfig.settings.index_version}-*`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
+
+        if (args.dry) {
+            const resultDeltas = await client.search<any>(searchDeltas);
+            if (
+                (resultDeltas.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                deleteDeltas += (
+                    resultDeltas.body.hits.total as estypes.SearchTotalHits
+                )?.value;
+            }
+        } else {
+            const indexExists = await client.indices.exists({
+                index: searchDeltas.index,
+            });
+            if (indexExists.body) {
+                delete searchDeltas.size;
+                const deletedDeltasResult = await client.deleteByQuery(
+                    searchDeltas
+                );
+                if (
+                    deletedDeltasResult &&
+                    deletedDeltasResult.body.deleted &&
+                    deletedDeltasResult.body.deleted > 0
+                ) {
+                    deleteDeltas += deletedDeltasResult.body.deleted;
+                }
+            } else {
+                console.log(
+                    `Index ${searchDeltas.index} doesn't exist. Unable to delete.`
+                );
+            }
+        }
+
+        // ABIS
+        const searchAbis = {
+            index: `${chain}-abi-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {
+                                range: {
+                                    block: { lte: range.start, gte: range.end },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
+
+        if (args.dry) {
+            const resultAbis = await client.search<any>(searchAbis);
+            if (
+                (resultAbis.body.hits.total as estypes.SearchTotalHits)?.value >
+                0
+            ) {
+                deleteAbis += (
+                    resultAbis.body.hits.total as estypes.SearchTotalHits
+                )?.value;
+                console.log('ABIs', { lte: range.start, gte: range.end });
+            }
+        } else {
+            const indexExists = await client.indices.exists({
+                index: searchAbis.index,
+            });
+            if (indexExists.body) {
+                delete searchAbis.size;
+                const deletedAbisResult = await client.deleteByQuery(
+                    searchAbis
+                );
+                if (
+                    deletedAbisResult &&
+                    deletedAbisResult.body.deleted &&
+                    deletedAbisResult.body.deleted > 0
+                ) {
+                    deleteAbis += deletedAbisResult.body.deleted;
+                }
+            } else {
+                console.log(
+                    `Index ${searchAbis.index} doesn't exist. Unable to delete.`
+                );
+            }
+        }
+
+        // ACCOUNTS
+        const searchAccounts = {
+            index: `${chain}-table-accounts-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
+
+        if (args.dry) {
+            const resultAccounts = await client.search<any>(searchAccounts);
+            if (
+                (resultAccounts.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (resultAccounts.body.hits.total as estypes.SearchTotalHits)
+                        ?.value,
+                    'accounts needs to be updated'
+                );
+                deleteAccounts += (
+                    resultAccounts.body.hits.total as estypes.SearchTotalHits
+                )?.value;
+            }
+        } else {
+
+            const indexExists = await client.indices.exists({
+                index: searchAccounts.index,
+            });
+
+            if (indexExists.body) {
+            delete searchAccounts.size;
+            const deletedAccountsResult = await client.deleteByQuery(
+                searchAccounts
+            );
+            if (
+                deletedAccountsResult &&
+                deletedAccountsResult.body.deleted &&
+                deletedAccountsResult.body.deleted > 0
+            ) {
+                deleteAccounts += deletedAccountsResult.body.deleted;
+            }
+        }else {
+            console.log(
+                `Index ${searchAccounts.index} doesn't exist. Unable to delete.`
+            );
+        }
+    }
+
+        // VOTERS
+        const searchVoters = {
+            index: `${chain}-table-voters-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
+
+        if (args.dry) {
+            const resultVoters = await client.search<any>(searchVoters);
+            if (
+                (resultVoters.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (resultVoters.body.hits.total as estypes.SearchTotalHits)
+                        ?.value,
+                    'voters needs to be updated'
+                );
+                deleteVoters += (
+                    resultVoters.body.hits.total as estypes.SearchTotalHits
+                )?.value;
+            }
+        } else {
+            const indexExists = await client.indices.exists({
+                index: searchVoters.index,
+            });
+
+            if (indexExists.body) {
+
+            delete searchVoters.size;
+            const deletedVotersResult = await client.deleteByQuery(
+                searchVoters
+            );
+            if (
+                deletedVotersResult &&
+                deletedVotersResult.body.deleted &&
+                deletedVotersResult.body.deleted > 0
+            ) {
+                deleteVoters += deletedVotersResult.body.deleted;
+            }
+        }else {
+            console.log(
+                `Index ${searchVoters.index} doesn't exist. Unable to delete.`
+            );
+        }
+    }
+
+        // PROPOSALS
+        const searchProposals = {
+            index: `${chain}-table-proposals-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
+
+        if (args.dry) {
+            const resultProposals = await client.search<any>(searchProposals);
+            if (
+                (resultProposals.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (resultProposals.body.hits.total as estypes.SearchTotalHits)
+                        ?.value,
+                    'proposals needs to be updated'
+                );
+                deleteProposals += (
+                    resultProposals.body.hits.total as estypes.SearchTotalHits
+                )?.value;
+            }
+        } else {
+
+
+            const indexExists = await client.indices.exists({index: searchProposals.index});
+
+            if (indexExists.body) {
+            delete searchProposals.size;
+            const deletedProposalsResult = await client.deleteByQuery(
+                searchProposals
+            );
+            if (
+                deletedProposalsResult &&
+                deletedProposalsResult.body.deleted &&
+                deletedProposalsResult.body.deleted > 0
+            ) {
+                deleteProposals += deletedProposalsResult.body.deleted;
+            }
+        }else { console.log(`Index ${searchProposals.index} doesn't exist. Unable to delete.`);}
+    }
+
+        // LINKS
+        const searchLinks = {
+            index: `${chain}-link-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
+
+        if (args.dry) {
+            const resultLinks = await client.search<any>(searchLinks);
+            if (
+                (resultLinks.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (resultLinks.body.hits.total as estypes.SearchTotalHits)
+                        ?.value,
+                    'links needs to be updated'
+                );
+                deleteLinks += (
+                    resultLinks.body.hits.total as estypes.SearchTotalHits
+                )?.value;
+            }
+        } else {
+            const indexExists = await client.indices.exists({index: searchLinks.index});
+
+            if (indexExists.body) {
+            delete searchLinks.size;
+            const deletedLinksResult = await client.deleteByQuery(searchLinks);
+            if (
+                deletedLinksResult &&
+                deletedLinksResult.body.deleted &&
+                deletedLinksResult.body.deleted > 0
+            ) {
+                deleteLinks += deletedLinksResult.body.deleted;
+            }
+        }else { console.log(`Index ${searchLinks.index} doesn't exist. Unable to delete.`);}
+    }
+
+        // PERMISSIONS
+        const searchPermissions = {
+            index: `${chain}-perm-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
+
+        if (args.dry) {
+            const resultPermissions = await client.search<any>(
+                searchPermissions
+            );
+            if (
+                (resultPermissions.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (
+                        resultPermissions.body.hits
+                            .total as estypes.SearchTotalHits
+                    )?.value,
+                    'permissions needs to be updated'
+                );
+                console.log({ lte: range.start, gte: range.end });
+                deletePermissions += (
+                    resultPermissions.body.hits.total as estypes.SearchTotalHits
+                )?.value;
+            }
+        } else {
+
+            const indexExists = await client.indices.exists({index: searchPermissions.index});
+            if (indexExists.body) {
+            delete searchPermissions.size;
+            const deletedPermissionsResult = await client.deleteByQuery(
+                searchPermissions
+            );
+            if (
+                deletedPermissionsResult &&
+                deletedPermissionsResult.body.deleted &&
+                deletedPermissionsResult.body.deleted > 0
+            ) {
+                deletePermissions += deletedPermissionsResult.body.deleted;
+            }
+        }else { console.log(`Index ${searchPermissions.index} doens't exist. Não foi possível realizar a exclusão.`);}
+    }
+
+        for (const id of range.ids) {
+            const searchBlocks = {
+                index: blockIndex,
+                body: {
+                    query: {
+                        bool: { must: [{ term: { block_id: { value: id } } }] },
+                    },
+                },
+            };
+            if (args.dry) {
+                console.log(`Deleting block ${id}...`);
+                const resultBlocks = await client.search<
+                    SearchResponse<HyperionBlock>
+                >(searchBlocks);
+                if (
+                    resultBlocks.body.hits.hits.length > 0 &&
+                    resultBlocks.body.hits.hits[0]._source
+                ) {
+                    deleteBlocks++;
+                }
+            } else {
+                const result = await client.deleteByQuery(searchBlocks);
+                if (result && result.body.deleted && result.body.deleted > 0) {
+                    deleteBlocks += result.body.deleted;
+                }
+            }
+        }
+    }
+
+    if (args.dry) {
+        console.log(
+            `DRY-RUN: Would have deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`
+        );
+        console.dir({
+            blocks: deleteBlocks,
+            actions: deleteActions,
+            deltas: deleteDeltas,
+            abis: deleteAbis,
+            accounts: deleteAccounts,
+            voters: deleteVoters,
+            proposals: deleteProposals,
+            links: deleteLinks,
+            permissions: deletePermissions,
+        });
+    } else {
+        console.log(
+            `Deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`
+        );
+    }
+
+    await fillMissingBlocksFromFile(args.host, chain, file, args.dry);
+}
+
+async function fillMissingBlocksFromFile(host, chain, file, dryRun) {
+    const config = readConnectionConfig();
+    const controlPort = config.chains[chain].control_port;
+    let hyperionIndexer = `ws://localhost:${controlPort}`;
+    if (host) {
+        hyperionIndexer = `ws://${host}:${controlPort}`;
+    }
+    const controller = new WebSocket(hyperionIndexer + '/local');
+
+    // Function to send Chunk
+    async function sendChunk(chunk): Promise<void> {
+        return new Promise((resolve, reject) => {
+            const payload = {
+                event: 'fill_missing_blocks',
+                data: chunk,
+            };
+            controller.send(JSON.stringify(payload));
+
+            // Wait repair_completed confirmation
+            controller.once('message', (data) => {
+                const parsed = JSON.parse(data.toString());
+                if (parsed.event === 'repair_completed') {
+                    // console.log(`Hyperion repair completed for chunk!`);
+                    resolve();
+                }
+            });
+        });
+    }
+
+    controller.on('open', async () => {
+        console.log('Connected to Hyperion Controller');
+        const parsedFile = JSON.parse(readFileSync(file).toString());
+        const chunkSize = 20; // Chunk size
+        const totalLines = parsedFile.length;
+
+        if (!dryRun) {
+            let completedLines = 0;
+
+            for (let i = 0; i < parsedFile.length; i += chunkSize) {
+                const chunk = parsedFile.slice(i, i + chunkSize);
+                await sendChunk(chunk);
+
+                // Atualizar o progresso com base no número total de linhas
+                completedLines += chunk.length;
+                const progress = (completedLines / totalLines) * 100;
+                const progressBar = Array(Math.round(progress / 2))
+                    .fill('#')
+                    .join('');
+                process.stdout.clearLine(0); // Limpar a linha anterior
+                process.stdout.cursorTo(0); // Mover o cursor para o início da linha
+                process.stdout.write(
+                    `Progress: [${progressBar}] ${progress.toFixed(2)}%`
+                );
+            }
+            console.log(); // Pule para a próxima linha após a conclusão
+            controller.close();
+        } else {
+            console.log('Dry run, skipping repair');
+            controller.close();
+        }
+    });
+
+    controller.on('close', () => {
+        console.log('Disconnected from Hyperion Controller');
+    });
+
+    controller.on('error', (err) => {
+        console.log(err);
+    });
+}
+
+function viewFile(file: string) {
+    const data = readFileSync(file, 'utf8');
+    const parsed = JSON.parse(data);
+    console.table(parsed);
+}
+
+// Commander Logic
+
+program
+    .name('Hyperion Repair CLI')
+    .description('CLI to find and repair forked and missing blocks on Hyperion')
+    .version('0.2.2');
+
+program
+    .command('scan <chain>')
+    .description('scan for forked blocks')
+    .option('-d, --dry', 'dry-run, do not delete or repair blocks')
+    .option('-o, --out-file <file>', 'forked-blocks.json output file')
+    .option('-f, --first <number>', 'initial block to start validation')
+    .option('-l, --last <number>', 'last block to validate')
+    .option('-b, --batch <number>', 'batch size to process')
+    .action(scanChain);
+
+program
+    .command('repair <chain> <file>')
+    .description('repair forked blocks')
+    .option('-h, --host <host>', 'Hyperion local control api')
+    .option('-d, --dry', 'dry-run, do not delete or repair blocks')
+    .option('-t, --check-tasks', 'check for running tasks')
+    .action(repairChain);
+
+program
+    .command('fill-missing <chain> <file>')
+    .description('write missing blocks')
+    .option('-h, --host <host>', 'Hyperion local control api')
+    .option('-d, --dry', 'dry-run, do not delete or repair blocks')
+    .action(repairMissing);
+
+program
+    .command('view <file>')
+    .description('view forked blocks')
+    .action(viewFile);
+
+program
+    .command('connect')
+    .option('-h, --host <host>', 'Hyperion local control api')
+    .action(async (args) => {
+        let hyperionIndexer = 'ws://localhost:4321';
+        let valid = false;
+        if (args.host) {
+            hyperionIndexer = args.host;
+        }
+        try {
+            const controller = new WebSocket(hyperionIndexer + '/local');
+            controller.on('open', () => {
+                valid = true;
+                controller.close();
+            });
+            controller.on('close', () => {
+                if (valid) {
+                    console.log(
+                        `✅  Hyperion Indexer Online - ${hyperionIndexer}`
+                    );
+                }
+            });
+            controller.on('error', (err) => {
+                console.log('Error:', err.message);
+                console.log(
+                    `Failed to connect on Hyperion Indexer at ${hyperionIndexer}, please use "--host ws://ADDRESS:PORT" to specify a remote indexer connection`
+                );
+            });
+        } catch (e) {
+            console.log(e);
+        }
+    });
+
+program.parse();

scripts/repair-cli/functions.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getBlocks = exports.getLastIndexedBlock = exports.getFirstIndexedBlock = exports.readChainConfig = exports.initESClient = exports.readConnectionConfig = void 0;
+const node_fs_1 = require("node:fs");
+const elasticsearch_1 = require("@elastic/elasticsearch");
+function readConnectionConfig() {
+    const file = (0, node_fs_1.readFileSync)('connections.json', 'utf8');
+    return JSON.parse(file);
+}
+exports.readConnectionConfig = readConnectionConfig;
+function initESClient(config) {
+    const node = `${config.elasticsearch.protocol}://${config.elasticsearch.host}`;
+    return new elasticsearch_1.Client({
+        nodes: [node],
+        auth: {
+            username: config.elasticsearch.user,
+            password: config.elasticsearch.pass
+        },
+        ssl: {
+            rejectUnauthorized: false
+        }
+    });
+}
+exports.initESClient = initESClient;
+function readChainConfig(chain) {
+    const file = (0, node_fs_1.readFileSync)(`chains/${chain}.config.json`, 'utf8');
+    return JSON.parse(file);
+}
+exports.readChainConfig = readChainConfig;
+async function getFirstIndexedBlock(client, blockIndex) {
+    const { body } = await client.search({
+        index: blockIndex,
+        size: 1,
+        body: {
+            sort: [{ block_num: { order: 'asc' } }]
+        }
+    });
+    if (body.hits.hits[0]._source) {
+        return body.hits.hits[0]._source.block_num;
+    }
+    else {
+        console.log('No blocks indexed yet');
+        process.exit();
+    }
+}
+exports.getFirstIndexedBlock = getFirstIndexedBlock;
+async function getLastIndexedBlock(client, blockIndex) {
+    const { body } = await client.search({
+        index: blockIndex,
+        size: 1,
+        body: {
+            sort: [{ block_num: { order: 'desc' } }]
+        }
+    });
+    if (body.hits.hits[0]._source) {
+        return body.hits.hits[0]._source.block_num;
+    }
+    else {
+        console.log('No blocks indexed yet');
+        process.exit();
+    }
+}
+exports.getLastIndexedBlock = getLastIndexedBlock;
+async function getBlocks(client, indexName, blocoInicial, blocoFinal, size) {
+    return await client.search({
+        index: indexName,
+        size: size,
+        body: {
+            sort: [{ block_num: { order: "desc" } }],
+            query: {
+                range: {
+                    block_num: {
+                        gte: blocoFinal,
+                        lte: blocoInicial,
+                    },
+                },
+            },
+        }
+    });
+}
+exports.getBlocks = getBlocks;
+//# sourceMappingURL=functions.js.map

scripts/repair-cli/functions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"functions.js","sourceRoot":"","sources":["functions.ts"],"names":[],"mappings":";;;AAAA,qCAAqC;AACrC,0DAAuD;AAGvD,SAAgB,oBAAoB;IAChC,MAAM,IAAI,GAAG,IAAA,sBAAY,EAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAHD,oDAGC;AAED,SAAgB,YAAY,CAAC,MAAW;IACpC,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,aAAa,CAAC,QAAQ,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IAC/E,OAAO,IAAI,sBAAM,CAAC;QACd,KAAK,EAAE,CAAC,IAAI,CAAC;QACb,IAAI,EAAE;YACF,QAAQ,EAAE,MAAM,CAAC,aAAa,CAAC,IAAI;YACnC,QAAQ,EAAE,MAAM,CAAC,aAAa,CAAC,IAAI;SACtC;QACD,GAAG,EAAE;YACD,kBAAkB,EAAE,KAAK;SAC5B;KACJ,CAAC,CAAC;AACP,CAAC;AAZD,oCAYC;AAED,SAAgB,eAAe,CAAC,KAAa;IACzC,MAAM,IAAI,GAAG,IAAA,sBAAY,EAAC,UAAU,KAAK,cAAc,EAAE,MAAM,CAAC,CAAC;IACjE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAHD,0CAGC;AAEM,KAAK,UAAU,oBAAoB,CAAC,MAAc,EAAE,UAAkB;IACzE,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,MAAM,CAAC,MAAM,CAAwC;QACtE,KAAK,EAAE,UAAU;QACjB,IAAI,EAAE,CAAC;QACP,IAAI,EAAE;YACF,IAAI,EAAE,CAAC,EAAC,SAAS,EAAE,EAAC,KAAK,EAAE,KAAK,EAAC,EAAC,CAAC;SACtC;KACJ,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;KAC9C;SAAM;QACH,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,EAAE,CAAC;KAClB;AACL,CAAC;AAdD,oDAcC;AAEM,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,UAAkB;IACxE,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,MAAM,CAAC,MAAM,CAAwC;QACtE,KAAK,EAAE,UAAU;QACjB,IAAI,EAAE,CAAC;QACP,IAAI,EAAE;YACF,IAAI,EAAE,CAAC,EAAC,SAAS,EAAE,EAAC,KAAK,EAAE,MAAM,EAAC,EAAC,CAAC;SACvC;KACJ,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;KAC9C;SAAM;QACH,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,EAAE,CAAC;KAClB;AACL,CAAC;AAdD,kDAcC;AAEM,KAAK,UAAU,SAAS,CAAC,MAAW,EAAE,SAAiB,EAAE,YAAiB,EAAE,UAAe,EAAE,IAAS;IACzG,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC;QACvB,KAAK,EAAE,SAAS;QAChB,IAAI,EAAE,IAAI;QACV,IAAI,EAAE;YACF,IAAI,EAAE,CAAC,EAAC,SAAS,EAAE,EAAC,KAAK,EAAE,MAAM,EAAC,EAAC,CAAC;YACpC,KAAK,EAAE;gBACH,KAAK,EAAE;oBACH,SAAS,EAAE;wBACP,GAAG,EAAE,UAAU;wBACf,GAAG,EAAE,YAAY;qBACpB;iBACJ;aACJ;SACJ;KACJ,CAAC,CAAC;AACP,CAAC;AAhBD,8BAgBC"}

scripts/repair-cli/functions.ts

@@ -0,0 +1,78 @@
+import {readFileSync} from "node:fs";
+import {Client, estypes} from "@elastic/elasticsearch";
+import {HyperionBlock} from "./interfaces.js";
+
+export function readConnectionConfig() {
+    const file = readFileSync('connections.json', 'utf8');
+    return JSON.parse(file);
+}
+
+export function initESClient(config: any) {
+    const node = `${config.elasticsearch.protocol}://${config.elasticsearch.host}`;
+    return new Client({
+        nodes: [node],
+        auth: {
+            username: config.elasticsearch.user,
+            password: config.elasticsearch.pass
+        },
+        ssl: {
+            rejectUnauthorized: false
+        }
+    });
+}
+
+export function readChainConfig(chain: string) {
+    const file = readFileSync(`chains/${chain}.config.json`, 'utf8');
+    return JSON.parse(file);
+}
+
+export async function getFirstIndexedBlock(client: Client, blockIndex: string) {
+    const {body} = await client.search<estypes.SearchResponse<HyperionBlock>>({
+        index: blockIndex,
+        size: 1,
+        body: {
+            sort: [{block_num: {order: 'asc'}}]
+        }
+    });
+    if (body.hits.hits[0]._source) {
+        return body.hits.hits[0]._source.block_num;
+    } else {
+        console.log('No blocks indexed yet');
+        process.exit();
+    }
+}
+
+export async function getLastIndexedBlock(client: Client, blockIndex: string) {
+    const {body} = await client.search<estypes.SearchResponse<HyperionBlock>>({
+        index: blockIndex,
+        size: 1,
+        body: {
+            sort: [{block_num: {order: 'desc'}}]
+        }
+    });
+    if (body.hits.hits[0]._source) {
+        return body.hits.hits[0]._source.block_num;
+    } else {
+        console.log('No blocks indexed yet');
+        process.exit();
+    }
+}
+
+export async function getBlocks(client: any, indexName: string, blocoInicial: any, blocoFinal: any, size: any) {
+    return await client.search({
+        index: indexName,
+        size: size,
+        body: {
+            sort: [{block_num: {order: "desc"}}],
+            query: {
+                range: {
+                    block_num: {
+                        gte: blocoFinal,
+                        lte: blocoInicial,
+                    },
+                },
+            },
+        }
+    });
+}
+

scripts/repair-cli/interfaces.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=interfaces.js.map

scripts/repair-cli/interfaces.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["interfaces.ts"],"names":[],"mappings":""}

scripts/repair-cli/interfaces.ts

@@ -0,0 +1,11 @@
+export interface HyperionBlock {
+    "@timestamp": string,
+    "block_num": number,
+    "block_id": string,
+    "producer": string,
+    "new_producers"?: any,
+    "schedule_version": number,
+    "cpu_usage": number,
+    "net_usage": number,
+    "prev_id": string
+}

temp/act.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'act',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'act',
+
+    mappings: {
+        action: {
+            "@act": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "act": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@act'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            act: data.act
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/batch.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'batch',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'batch',
+        
+    mappings: {
+        action: {
+            "@batch": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "batch_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@batch'] = {
+            coopname: data.coopname,
+            action: data.action,
+            batch_id: data.batch_id,
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/decision.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'decision',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'decision',
+    
+    mappings: {
+        action: {
+            "@decision": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "decision": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+           
+        action['@decision'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            decision: data.decision
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/exec.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'exec',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'exec',
+
+    mappings: {
+        action: {
+            "@exec": {
+                "properties": {
+                    "executer": {"type": "keyword"},
+                    "coopname": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@exec'] = {
+            executer: data.executer,
+            coopname: data.coopname,
+            decision_id: data.decision_id,
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/statement.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'statement',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'statement',
+
+    mappings: {
+        action: {
+            "@statement": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "statement": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+           
+        action['@statement'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            statement: data.statement
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/voteagainst.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'voteagainst',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'voteagainst',
+
+    mappings: {
+        action: {
+            "@voteagainst": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "member": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@voteagainst'] = {
+            coopname: data.coopname,
+            member: data.member,
+            decision_id: decision_id
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/votefor.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'votefor',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'votefor',
+
+    mappings: {
+        action: {
+            "@votefor": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "member": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@votefor'] = {
+            coopname: data.coopname,
+            member: data.member,
+            decision_id: decision_id
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

tsconfig.json

@@ -6,6 +6,7 @@
     "allowSyntheticDefaultImports": true,
     "noImplicitAny": false,
     "moduleResolution": "node",
+    "noEmitOnError": true,
     "sourceMap": true,
     "allowJs": false,
     "typeRoots": [

workers/deserializer.ts

@@ -8,39 +8,40 @@ import {Type} from "../addons/eosjs-native/eosjs-serialize";
 import {debugLog, hLog} from "../helpers/common_functions";
 import {createHash} from "crypto";
 import flatstr from 'flatstr';
+import {Options} from "amqplib";
 
 const FJS = require('fast-json-stringify');
 
-const lightBlockSerializer = FJS({
-    title: 'Light Block',
-    type: 'object',
-    properties: {
-        '@timestamp': {type: 'string'},
-        block_num: {type: 'integer'},
-        block_id: {type: 'string'},
-        prev_id: {type: 'string'},
-        producer: {type: 'string'},
-        new_producers: {
-            type: 'object',
-            nullable: true,
-            properties: {
-                version: {type: 'integer'},
-                producers: {
-                    type: 'array',
-                    items: {
-                        properties: {
-                            block_signing_key: {type: 'string'},
-                            producer_name: {type: 'string'}
-                        }
-                    }
-                }
-            }
-        },
-        schedule_version: {type: 'integer'},
-        cpu_usage: {type: 'integer'},
-        net_usage: {type: 'integer'}
-    }
-});
+// const lightBlockSerializer = FJS({
+//     title: 'Light Block',
+//     type: 'object',
+//     properties: {
+//         '@timestamp': {type: 'string'},
+//         block_num: {type: 'integer'},
+//         block_id: {type: 'string'},
+//         prev_id: {type: 'string'},
+//         producer: {type: 'string'},
+//         new_producers: {
+//             type: 'object',
+//             nullable: true,
+//             properties: {
+//                 version: {type: 'integer'},
+//                 producers: {
+//                     type: 'array',
+//                     items: {
+//                         properties: {
+//                             block_signing_key: {type: 'string'},
+//                             producer_name: {type: 'string'}
+//                         }
+//                     }
+//                 }
+//             }
+//         },
+//         schedule_version: {type: 'integer'},
+//         cpu_usage: {type: 'integer'},
+//         net_usage: {type: 'integer'}
+//     }
+// });
 
 const index_queues = require('../definitions/index-queues').index_queues;
 const {AbiDefinitions} = require("../definitions/abi_def");
@@ -124,6 +125,9 @@ export default class MainDSWorker extends HyperionWorker {
 
     allowedDynamicContracts: Set<string> = new Set<string>();
 
+    backpressureQueue: any[] = [];
+    waitToSend = false;
+
     constructor() {
 
         super();
@@ -290,6 +294,16 @@ export default class MainDSWorker extends HyperionWorker {
             this.ch.consume(process.env['worker_queue'], (data) => {
                 this.consumerQueue.push(data).catch(console.log);
             });
+            this.ch.on('drain', args => {
+                this.waitToSend = false;
+                while (this.backpressureQueue.length > 0) {
+                    const msg = this.backpressureQueue.shift();
+                    const status = this.controlledSendToQueue(msg.queue, msg.payload, msg.options);
+                    if (!status) {
+                        break;
+                    }
+                }
+            });
         }
     }
 
@@ -410,6 +424,16 @@ export default class MainDSWorker extends HyperionWorker {
                         live: process.env.live_mode
                     });
                 }
+
+                // stream light block
+                if (this.allowStreaming) {
+                    this.ch.publish('', this.chain + ':stream', Buffer.from(JSON.stringify(light_block)), {
+                        headers: {
+                            event: 'block',
+                            blockNum: light_block.block_num
+                        }
+                    });
+                }
             }
 
             // Process Delta Traces (must be done first to catch ABI updates)
@@ -468,6 +492,7 @@ export default class MainDSWorker extends HyperionWorker {
                                 hLog(`${block_num} was filtered with ${inline_count} actions!`);
                             }
                             try {
+                                trace[1].signatures = signatures;
                                 this.routeToPool(trace[1], {
                                     block_num,
                                     block_id,
@@ -475,8 +500,7 @@ export default class MainDSWorker extends HyperionWorker {
                                     ts,
                                     inline_count,
                                     filtered,
-                                    live: process.env['live_mode'],
-                                    signatures
+                                    live: process.env['live_mode']
                                 });
                             } catch (e) {
                                 hLog(e);
@@ -495,6 +519,7 @@ export default class MainDSWorker extends HyperionWorker {
             return {
                 block_num: res['this_block']['block_num'],
                 block_id: res['this_block']['block_id'],
+                block_ts,
                 trx_ids: onBlockTransactions,
                 size: _traces.length
             };
@@ -611,15 +636,33 @@ export default class MainDSWorker extends HyperionWorker {
         }
 
         const pool_queue = `${this.chain}:ds_pool:${selected_q}`;
-        if (this.ch_ready) {
-            // console.log('selected_q', pool_queue);
-            this.ch.sendToQueue(pool_queue, bufferFromJson(trace, true), {headers});
-            return true;
+        const payload = bufferFromJson(trace, true);
+
+        if (!this.waitToSend) {
+            if (this.ch_ready) {
+                this.controlledSendToQueue(pool_queue, payload, {headers});
+                return true;
+            } else {
+                return false;
+            }
         } else {
+            this.backpressureQueue.push({
+                queue: pool_queue,
+                payload: payload,
+                options: {headers}
+            });
             return false;
         }
     }
 
+    controlledSendToQueue(pool_queue: string, payload: Buffer, options: Options.Publish): boolean {
+        const enqueueResult = this.ch.sendToQueue(pool_queue, payload, options);
+        if (!enqueueResult) {
+            this.waitToSend = true;
+        }
+        return enqueueResult;
+    }
+
     createSerialBuffer(inputArray) {
         return new Serialize.SerialBuffer({textEncoder: this.txEnc, textDecoder: this.txDec, array: inputArray});
     }
@@ -1149,6 +1192,7 @@ export default class MainDSWorker extends HyperionWorker {
                 let jsonRow = await this.processContractRowNative(payload, block_num);
 
                 if (jsonRow?.value && !jsonRow['_blacklisted']) {
+                    debugLog(jsonRow);
                     debugLog('Delta DS failed ->>', jsonRow);
                     jsonRow = await this.processContractRowNative(payload, block_num - 1);
                     debugLog('Retry with previous ABI ->>', jsonRow);
@@ -1389,9 +1433,33 @@ export default class MainDSWorker extends HyperionWorker {
         },
 
         // Global Chain configuration update
-        // "global_property": async (global_property, block_num, block_ts, row, block_id) => {
-        //     hLog(block_num, global_property);
-        // },
+        "global_property": async (global_property, block_num, block_ts) => {
+            if (global_property.proposed_schedule.version !== 0) {
+                hLog("Proposed Schedule version: " + global_property.proposed_schedule.version + " at block: " + global_property.proposed_schedule_block_num);
+                try {
+                    const payload = {
+                        block_num: global_property.proposed_schedule_block_num,
+                        '@timestamp': block_ts,
+                        version: global_property.proposed_schedule.version,
+                        producers: global_property.proposed_schedule.producers
+                    };
+                    payload.producers.forEach((producer: any) => {
+                        producer.name = producer.producer_name;
+                        delete producer.producer_name;
+                        if (producer.authority[0] === 'block_signing_authority_v0') {
+                            producer.authority = producer.authority[1];
+                            producer.keys = producer.authority.keys.map((key: any) => {
+                                return key.key;
+                            });
+                            delete producer.authority;
+                        }
+                    });
+                    await this.pushToIndexQueue(payload, 'schedule');
+                } catch (e) {
+                    hLog("Failed to parse proposed schedule: " + e.message);
+                }
+            }
+        },
 
         // Activated Protocol features
         // "protocol_state": async (protocol_state, block_num, block_ts, row, block_id) => {
@@ -1458,6 +1526,8 @@ export default class MainDSWorker extends HyperionWorker {
                                 await this.deltaStructHandlers[key](data[1], block_num, block_ts, row, block_id);
                             } catch (e) {
                                 hLog(`Delta struct [${key}] processing error: ${e.message}`);
+                                hLog(e);
+                                hLog(data[1]);
                             }
                         }
                     }
@@ -1602,41 +1672,46 @@ export default class MainDSWorker extends HyperionWorker {
 
         this.tableHandlers[EOSIO_ALIAS + ':producers'] = (delta) => {
             const data = delta['data'];
-            delta['@producers'] = {
-                total_votes: parseFloat(data['total_votes']),
-                is_active: data['is_active'],
-                unpaid_blocks: data['unpaid_blocks']
-            };
-            delete delta['data'];
+            if (data) {
+                delta['@producers'] = {
+                    total_votes: parseFloat(data['total_votes']),
+                    is_active: data['is_active'],
+                    unpaid_blocks: data['unpaid_blocks']
+                };
+                delete delta['data'];
+            }
         };
 
         this.tableHandlers[EOSIO_ALIAS + ':userres'] = (delta) => {
             const data = delta['data'];
-            const net = parseFloat(data['net_weight'].split(" ")[0]);
-            const cpu = parseFloat(data['cpu_weight'].split(" ")[0]);
-            delta['@userres'] = {
-                owner: data['owner'],
-                net_weight: net,
-                cpu_weight: cpu,
-                total_weight: parseFloat((net + cpu).toFixed(4)),
-                ram_bytes: parseInt(data['ram_bytes'])
-            };
-            delete delta['data'];
+            if (data['net_weight'] && data['cpu_weight']) {
+                const net = parseFloat(data['net_weight'].split(" ")[0]);
+                const cpu = parseFloat(data['cpu_weight'].split(" ")[0]);
+                delta['@userres'] = {
+                    owner: data['owner'],
+                    net_weight: net,
+                    cpu_weight: cpu,
+                    total_weight: parseFloat((net + cpu).toFixed(4)),
+                    ram_bytes: parseInt(data['ram_bytes'])
+                };
+                delete delta['data'];
+            }
         };
 
         this.tableHandlers[EOSIO_ALIAS + ':delband'] = (delta) => {
             const data = delta['data'];
-            const net = parseFloat(data['net_weight'].split(" ")[0]);
-            const cpu = parseFloat(data['cpu_weight'].split(" ")[0]);
-            delta['@delband'] = {
-                from: data['from'],
-                to: data['to'],
-                net_weight: net,
-                cpu_weight: cpu,
-                total_weight: parseFloat((net + cpu).toFixed(4))
-            };
-            delete delta['data'];
-            // hLog(delta);
+            if (data['net_weight'] && data['cpu_weight']) {
+                const net = parseFloat(data['net_weight'].split(" ")[0]);
+                const cpu = parseFloat(data['cpu_weight'].split(" ")[0]);
+                delta['@delband'] = {
+                    from: data['from'],
+                    to: data['to'],
+                    net_weight: net,
+                    cpu_weight: cpu,
+                    total_weight: parseFloat((net + cpu).toFixed(4))
+                };
+                delete delta['data'];
+            }
         };
 
         this.tableHandlers[EOSIO_ALIAS + '.msig:proposal'] = async (delta) => {

workers/ds-pool.ts

@@ -21,6 +21,48 @@ interface CustomAbiDef {
     endingBlock: number;
 }
 
+function cleanActionTrace(t: any) {
+    try {
+        if (t.return_value === '') {
+            delete t.return_value;
+        }
+        if (t.context_free === false) {
+            delete t.context_free;
+        }
+        if (t.elapsed === '0') {
+            delete t.elapsed;
+        }
+
+        // remove act_digest from grouped receipts since it was written to the action
+        if (t.receipts && t.receipts.length > 0) {
+            t.act_digest = t.receipts[0].act_digest;
+            for (let receipt of t.receipts) {
+                delete receipt.act_digest;
+            }
+        } else {
+            delete t.receipts;
+        }
+
+        delete t.console;
+        delete t.receiver;
+
+        // onblock action case
+        if (t.signatures && t.signatures.length === 0) {
+            delete t.signatures;
+        }
+
+        if (t.inline_count === 0) {
+            delete t.inline_count;
+        }
+
+        if (t.net_usage_words === 0) {
+            delete t.net_usage_words;
+        }
+    } catch (e) {
+        console.log(e);
+    }
+}
+
 export default class DSPoolWorker extends HyperionWorker {
 
     abi;
@@ -412,8 +454,8 @@ export default class DSPoolWorker extends HyperionWorker {
     }
 
     async processTraces(transaction_trace, extra) {
-        const {cpu_usage_us, net_usage_words} = transaction_trace;
-        const {block_num, block_id, producer, ts, inline_count, filtered, live, signatures} = extra;
+        const {cpu_usage_us, net_usage_words, signatures} = transaction_trace;
+        const {block_num, block_id, producer, ts, inline_count, filtered, live} = extra;
 
         if (transaction_trace.status === 0) {
             let action_count = 0;
@@ -464,20 +506,12 @@ export default class DSPoolWorker extends HyperionWorker {
             }
 
             const _finalTraces = [];
+
             if (_processedTraces.length > 1) {
                 const act_digests = {};
 
                 // collect digests & receipts
                 for (const _trace of _processedTraces) {
-
-                    if (this.conf.settings.dsp_parser) {
-                        if (_trace.console !== '') {
-                            await parseDSPEvent(this, _trace);
-                        }
-                    } else {
-                        delete _trace.console;
-                    }
-
                     if (act_digests[_trace.receipt.act_digest]) {
                         act_digests[_trace.receipt.act_digest].push(_trace.receipt);
                     } else {
@@ -488,18 +522,17 @@ export default class DSPoolWorker extends HyperionWorker {
                 // Apply notified accounts to first trace instance
                 for (const _trace of _processedTraces) {
                     if (act_digests[_trace.receipt.act_digest]) {
-                        const notifiedSet = new Set();
+                        // const notifiedSet = new Set();
                         _trace['receipts'] = [];
                         for (const _receipt of act_digests[_trace.receipt.act_digest]) {
-                            notifiedSet.add(_receipt.receiver);
+                            // notifiedSet.add(_receipt.receiver);
                             _trace['code_sequence'] = _receipt['code_sequence'];
                             delete _receipt['code_sequence'];
                             _trace['abi_sequence'] = _receipt['abi_sequence'];
                             delete _receipt['abi_sequence'];
-                            delete _receipt['act_digest'];
                             _trace['receipts'].push(_receipt);
                         }
-                        _trace['notified'] = [...notifiedSet];
+                        // _trace['notified'] = [...notifiedSet];
                         delete act_digests[_trace.receipt.act_digest];
                         delete _trace['receipt'];
                         delete _trace['receiver'];
@@ -514,10 +547,12 @@ export default class DSPoolWorker extends HyperionWorker {
                 _trace['code_sequence'] = _trace['receipt'].code_sequence;
                 _trace['abi_sequence'] = _trace['receipt'].abi_sequence;
                 _trace['act_digest'] = _trace['receipt'].act_digest;
-                _trace['notified'] = [_trace['receipt'].receiver];
+
+                // notified array is not required since receipts.receiver can be indexed directly
+                // _trace['notified'] = [_trace['receipt'].receiver];
+
                 delete _trace['receipt']['code_sequence'];
                 delete _trace['receipt']['abi_sequence'];
-                delete _trace['receipt']['act_digest'];
                 _trace['receipts'] = [_trace['receipt']];
                 delete _trace['receipt'];
                 _finalTraces.push(_trace);
@@ -528,6 +563,7 @@ export default class DSPoolWorker extends HyperionWorker {
             const redisPayload = new Map<string, IORedis.ValueType>();
 
             for (const uniqueAction of _finalTraces) {
+                cleanActionTrace(uniqueAction);
                 const payload = Buffer.from(flatstr(JSON.stringify(uniqueAction)));
                 redisPayload.set(uniqueAction.global_sequence.toString(), payload);
                 this.actionDsCounter++;
@@ -566,20 +602,24 @@ export default class DSPoolWorker extends HyperionWorker {
 
     pushToActionStreamingQueue(payload, uniqueAction) {
         if (this.allowStreaming && this.conf.features['streaming'].traces) {
-            const notifArray = new Set();
-            uniqueAction.act.authorization.forEach(auth => {
-                notifArray.add(auth.actor);
-            });
-            uniqueAction.notified.forEach(acc => {
-                notifArray.add(acc);
-            });
-            const headers = {
-                event: 'trace',
-                account: uniqueAction['act']['account'],
-                name: uniqueAction['act']['name'],
-                notified: [...notifArray].join(",")
-            };
-            this.ch.publish('', this.chain + ':stream', payload, {headers});
+            try {
+                const notifArray = new Set();
+                uniqueAction.act.authorization.forEach(auth => {
+                    notifArray.add(auth.actor);
+                });
+                uniqueAction.receipts.forEach(rec => {
+                    notifArray.add(rec.receiver);
+                });
+                const headers = {
+                    event: 'trace',
+                    account: uniqueAction['act']['account'],
+                    name: uniqueAction['act']['name'],
+                    notified: [...notifArray].join(",")
+                };
+                this.ch.publish('', this.chain + ':stream', payload, {headers});
+            } catch (e) {
+                hLog(e);
+            }
         }
     }
 

workers/indexer.ts

@@ -11,20 +11,11 @@ export default class IndexerWorker extends HyperionWorker {
     private temp_indexed_count = 0;
 
     esRoutes: ElasticRoutes;
-    distributionMap;
 
     constructor() {
         super();
 
-        if (process.env.distribution) {
-            try {
-                this.distributionMap = JSON.parse(process.env.distribution);
-            } catch {
-                hLog('Failed to parse distribution map');
-            }
-        }
-
-        this.esRoutes = new ElasticRoutes(this.manager, this.distributionMap);
+        this.esRoutes = new ElasticRoutes(this.manager);
         this.indexQueue = cargo((payload: Message[], callback) => {
             if (this.ch_ready && payload) {
                 if (this.esRoutes.routes[process.env.type]) {

workers/state-reader.ts

@@ -22,6 +22,7 @@ export default class StateReader extends HyperionWorker {
     private completionMonitoring: NodeJS.Timeout;
     private types: Map<string, Type>;
     private local_block_num = parseInt(process.env.first_block, 10) - 1;
+    private local_block_id = '';
     private queueSizeCheckInterval = 5000;
     private local_distributed_count = 0;
     private lastPendingCount = 0;
@@ -40,6 +41,10 @@ export default class StateReader extends HyperionWorker {
     private shipInitStatus: any;
 
     private forkedBlocks = new Map<string, number>();
+    private idle = true;
+    private repairMode = false;
+    private internalPendingRanges = [];
+
 
     constructor() {
         super();
@@ -122,7 +127,7 @@ export default class StateReader extends HyperionWorker {
                             console.log('Message nacked!');
                             console.log(err.message);
                         } else {
-                            process.send({event: 'read_block', live: this.isLiveReader});
+                            process.send({event: 'read_block', live: this.isLiveReader, block_num: d.num});
                         }
                     });
 
@@ -189,17 +194,56 @@ export default class StateReader extends HyperionWorker {
 
     newRange(data: any) {
         debugLog(`new_range [${data.first_block},${data.last_block}]`);
+
+        if (this.repairMode) {
+            hLog(`Received Repair request [${data.first_block},${data.last_block}]`);
+        }
+
         this.local_distributed_count = 0;
         clearInterval(this.completionMonitoring);
         this.completionMonitoring = null;
         this.completionSignaled = false;
         this.local_last_block = data.last_block;
+
+        // make sure no data is requested from before the first indexed block on ship
+        if (parseInt(data.first_block) < this.shipInitStatus['trace_begin_block']) {
+            data.first_block = this.shipInitStatus['trace_begin_block'];
+            hLog("Impossible to repair requested range, first block is before the first indexed block on ship, trimming to " + data.first_block);
+        }
+
         this.range_size = parseInt(data.last_block) - parseInt(data.first_block);
-        if (this.allowRequests) {
-            this.requestBlockRange(data.first_block, data.last_block);
-            this.pendingRequest = null;
+
+        // if range size is not positive we need to immediately signal completion to master
+        if (this.range_size <= 0) {
+            this.emitCompletedSignal();
+        }
+
+        // make sure the range doesn't exceed the batch size
+        if (this.range_size > this.conf.scaling.batch_size) {
+            // split the range into multiple ranges
+            const ranges = [];
+            let start = parseInt(data.first_block);
+            let end = parseInt(data.last_block);
+            while (start < end) {
+                const s = start;
+                const e = start + this.conf.scaling.batch_size;
+                ranges.push({
+                    first_block: s,
+                    last_block: e > end ? end : e
+                });
+                start = e + 1;
+            }
+            console.log(`Splitting range [${data.first_block},${data.last_block}] into ${ranges.length} ranges`);
+            this.internalPendingRanges = ranges;
+            const next = this.internalPendingRanges.shift();
+            this.newRange(next);
         } else {
-            this.pendingRequest = [data.first_block, data.last_block];
+            if (this.allowRequests) {
+                this.requestBlockRange(data.first_block, data.last_block);
+                this.pendingRequest = null;
+            } else {
+                this.pendingRequest = [data.first_block, data.last_block];
+            }
         }
     }
 
@@ -231,7 +275,7 @@ export default class StateReader extends HyperionWorker {
             case 'stop': {
                 if (this.isLiveReader) {
                     console.log('[LIVE READER] Closing Websocket');
-                    this.ship.close();
+                    this.ship.close(true);
                     setTimeout(() => {
                         console.log('[LIVE READER] Process killed');
                         process.exit(1);
@@ -262,19 +306,25 @@ export default class StateReader extends HyperionWorker {
                             pending++;
                         }
                     });
-                    if (pending === this.lastPendingCount && pending > 0) {
-                        // console.log(`[${process.env['worker_id']}] Pending blocks: ${pending}`);
-                    } else {
+                    if (!(pending === this.lastPendingCount && pending > 0)) {
                         this.lastPendingCount = pending;
                     }
                 }
                 if (pending === 0) {
                     debugLog(`Reader completed - ${this.range_size} - ${this.local_distributed_count}`);
                     clearInterval(this.completionMonitoring);
-                    process.send({
-                        event: 'completed',
-                        id: process.env['worker_id']
-                    });
+
+                    // check if there are any pending ranges, then signal completion to master
+                    if (this.internalPendingRanges.length === 0) {
+                        process.send({
+                            event: 'completed',
+                            id: process.env['worker_id']
+                        });
+                    } else {
+                        // process next range in queue
+                        const next = this.internalPendingRanges.shift();
+                        this.newRange(next);
+                    }
                 }
             }, 200);
         }
@@ -286,7 +336,7 @@ export default class StateReader extends HyperionWorker {
                 try {
                     await this.onMessage(block);
                 } catch (e) {
-                    console.log(e);
+                    hLog(e);
                 }
             }
             this.ackBlockRange(block_array.length);
@@ -309,62 +359,88 @@ export default class StateReader extends HyperionWorker {
         }
 
         if (!process.env.worker_role) {
-            console.log("[FATAL ERROR] undefined role! Exiting now!");
-            this.ship.close();
+            hLog("[FATAL ERROR] undefined role! Exiting now!");
+            this.ship.close(false);
             process.exit(1);
             return;
         }
 
         // NORMAL OPERATION MODE
         if (!this.recovery) {
+
             const result = deserialize('result', data, this.txEnc, this.txDec, this.types);
+
+            // ship status message
             if (result[0] === 'get_status_result_v0') {
                 this.shipInitStatus = result[1];
                 hLog(`\n| SHIP Status Report\n| Init block: ${this.shipInitStatus['chain_state_begin_block']}\n| Head block: ${this.shipInitStatus['chain_state_end_block']}`);
                 const chain_state_begin_block = this.shipInitStatus['chain_state_begin_block'];
-                if (!this.conf.indexer.disable_reading) {
+                if (!this.conf.indexer.disable_reading || process.env['worker_role'] === 'repair_reader') {
+
                     switch (process.env['worker_role']) {
+
+                        // range reader setup
                         case 'reader': {
                             if (chain_state_begin_block > process.env.first_block) {
+
+                                // skip snapshot block until a solution to parse it is found
+                                const nextBlock = chain_state_begin_block + 2;
                                 hLog(`First saved block is ahead of requested range! - Req: ${process.env.first_block} | First: ${chain_state_begin_block}`);
-                                hLog('Requesting a single block');
-                                if (this.conf.settings.ignore_snapshot) {
-                                    this.local_block_num = chain_state_begin_block;
-                                    process.send({event: 'update_init_block', block_num: chain_state_begin_block + 2,});
-                                    this.newRange({
-                                        first_block: chain_state_begin_block + 1,
-                                        last_block: chain_state_begin_block + this.conf.scaling.batch_size
-                                    });
-                                } else {
-                                    process.send({event: 'update_init_block', block_num: chain_state_begin_block + 1});
-                                    this.newRange({
-                                        first_block: chain_state_begin_block,
-                                        last_block: chain_state_begin_block + 1
-                                    });
-                                }
+                                this.local_block_num = nextBlock - 1;
+                                process.send({
+                                    event: 'update_init_block',
+                                    block_num: nextBlock
+                                });
+                                this.newRange({
+                                    first_block: nextBlock,
+                                    last_block: nextBlock + this.conf.scaling.batch_size
+                                });
                             } else {
                                 this.requestBlocks(0);
                             }
                             break;
                         }
+
+                        // live reader setup
                         case 'continuous_reader': {
                             this.requestBlocks(parseInt(process.env['worker_last_processed_block'], 10));
                             break;
                         }
+
+                        case 'repair_reader': {
+                            this.repairMode = true;
+                            hLog('Waiting for operations...');
+                            process.send({
+                                event: 'repair_reader_ready'
+                            });
+                            break;
+                        }
+
                     }
                 } else {
-                    this.ship.close();
+                    this.ship.close(true);
                     process.exit(1);
                 }
+
             } else {
+
+                // ship block message
                 const res = result[1];
 
                 if (res['this_block']) {
+                    this.idle = false;
                     const blk_num = res['this_block']['block_num'];
+                    const blk_id = res['this_block']['block_id'];
+
                     const lib = res['last_irreversible'];
                     const task_payload = {num: blk_num, content: data};
 
                     if (res.block && res.traces && res.deltas) {
+
+                        if (this.repairMode) {
+                            hLog("Repaired block: " + blk_num);
+                        }
+
                         debugLog(`block_num: ${blk_num}, block_size: ${res.block.length}, traces_size: ${res.traces.length}, deltas_size: ${res.deltas.length}`);
                     } else {
                         if (!res.traces) {
@@ -381,18 +457,35 @@ export default class StateReader extends HyperionWorker {
                     if (this.isLiveReader) {
 
                         // LIVE READER MODE
-                        if (blk_num !== this.local_block_num + 1) {
-                            hLog(`Expected: ${this.local_block_num + 1}, received: ${blk_num}`);
+                        let prev_id;
+                        if (res['prev_block'] && res['prev_block']['block_id']) {
+                            prev_id = res['prev_block']['block_id'];
+                        }
+
+                        // // 2% chance to modify the previous block id to simulate a fork
+                        // if (Math.random() < 0.02) {
+                        //     // prev_id = prev_id + 'a';
+                        //     this.local_block_num = this.local_block_num - 1;
+                        // }
+
+                        if ((this.local_block_id && prev_id && this.local_block_id !== prev_id) || (blk_num !== this.local_block_num + 1)) {
+
+                            console.log((this.local_block_id && prev_id && this.local_block_id !== prev_id), (blk_num !== this.local_block_num + 1));
+                            console.log(blk_num, this.local_block_num + 1);
+
+                            hLog(`Unlinked block at ${prev_id} with previous block ${this.local_block_id}`);
+                            hLog(`Forked block: ${blk_num}`);
                             try {
                                 // delete all previously stored data for the forked blocks
                                 await this.handleFork(res);
                             } catch (e) {
                                 hLog(`Failed to handle fork during live reading! - Error: ${e.message}`);
                             }
-                        } else {
-                            this.local_block_num = blk_num;
                         }
 
+                        this.local_block_num = blk_num;
+                        this.local_block_id = blk_id;
+
                         if (lib.block_num > this.local_lib) {
                             this.local_lib = lib.block_num;
                             // emit lib update event
@@ -400,22 +493,32 @@ export default class StateReader extends HyperionWorker {
                         }
 
                         await this.stageOneDistQueue.push(task_payload);
+
                         return 1;
+
                     } else {
 
                         // Detect skipped first block
                         if (!this.receivedFirstBlock) {
                             if (blk_num !== this.local_block_num + 1) {
-                                hLog(`WARNING: First block received was #${blk_num}, but #${this.local_block_num + 1} was expected!`);
-                                hLog(`Make sure the block.log file contains the requested range, check with "eosio-blocklog --smoke-test"`);
+                                // Special case for block 2 that is actually the first block on ship
+                                if (blk_num !== 2) {
+                                    hLog(`WARNING: First block received was #${blk_num}, but #${this.local_block_num + 1} was expected!`);
+                                    hLog(`Make sure the block.log file contains the requested range, check with "eosio-blocklog --smoke-test"`);
+                                }
                                 this.local_block_num = blk_num - 1;
+                                this.local_distributed_count++;
+                                process.send({
+                                    event: 'skipped_block',
+                                    block_num: this.local_block_num + 1
+                                })
                             }
                             this.receivedFirstBlock = true;
                         }
 
                         // BACKLOG MODE
                         if (this.future_block !== 0 && this.future_block === blk_num) {
-                            console.log('Missing block ' + blk_num + ' received!');
+                            hLog('Missing block ' + blk_num + ' received!');
                         } else {
                             this.future_block = 0;
                         }
@@ -440,7 +543,18 @@ export default class StateReader extends HyperionWorker {
                         }
                     }
                 } else {
-                    return 0;
+                    if (this.repairMode) {
+                        hLog("No data on requested block!");
+                    }
+                    this.idle = true;
+                    // hLog(`Reader is idle! - Head at: ${result[1].head.block_num}`);
+                    // this.ship.close(true);
+                    // const queueSize = [this.stageOneDistQueue.length(), this.blockReadingQueue.length()];
+                    // hLog(queueSize);
+                    // process.send({
+                    //     event: 'kill_worker',
+                    //     id: process.env['worker_id']
+                    // });
                 }
             }
 
@@ -500,8 +614,12 @@ export default class StateReader extends HyperionWorker {
         request.start_block_num = parseInt(first_block > 0 ? first_block.toString() : '1', 10);
         request.end_block_num = parseInt(last_block.toString(), 10);
         const reqType = 'get_blocks_request_' + this.shipRev;
-        debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`);
-        this.send([reqType, request]);
+        if (this.ship.connected) {
+            debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`);
+            this.send([reqType, request]);
+        } else {
+            hLog('[Warning] Request failed - SHIP is not online!');
+        }
     }
 
     private send(req_data: (string | any)[]) {
@@ -514,8 +632,12 @@ export default class StateReader extends HyperionWorker {
         this.local_block_num = request.start_block_num - 1;
         request.end_block_num = parseInt(last, 10);
         const reqType = 'get_blocks_request_' + this.shipRev;
-        debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`);
-        this.send([reqType, request]);
+        if (this.ship.connected) {
+            debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`);
+            this.send([reqType, request]);
+        } else {
+            hLog('[Warning] Request failed - SHIP is not online!');
+        }
     }
 
     private async deleteForkedBlock(block_id: string) {
@@ -529,6 +651,7 @@ export default class StateReader extends HyperionWorker {
             refresh: true,
             body: searchBody
         });
+
         if (dbqResultDelta.body && dbqResultDelta.statusCode === 200) {
             hLog(`${dbqResultDelta.body.deleted} deltas removed from ${block_id}`);
         } else {
@@ -541,6 +664,7 @@ export default class StateReader extends HyperionWorker {
             refresh: true,
             body: searchBody
         });
+
         if (dbqResultAction.body && dbqResultAction.statusCode === 200) {
             hLog(`${dbqResultAction.body.deleted} traces removed from ${block_id}`);
         } else {
@@ -564,7 +688,11 @@ export default class StateReader extends HyperionWorker {
             if (blockData.body) {
                 const targetBlockId = blockData.body._source.block_id;
                 this.forkedBlocks.set(targetBlockId, Date.now());
-                this.deleteForkedBlock(targetBlockId).catch(console.log);
+                try {
+                    await this.deleteForkedBlock(targetBlockId);
+                } catch (e: any) {
+                    hLog(`Error deleting forked block ${targetBlockId}: ${e.message}`);
+                }
             }
         }
     }
@@ -588,41 +716,41 @@ export default class StateReader extends HyperionWorker {
         this.send(['get_blocks_ack_request_v0', {num_messages: size}]);
     }
 
-    private startQueueWatcher() {
-        setInterval(() => {
-            if (this.isLiveReader) {
-                this.manager.checkQueueSize(this.chain + ":live_blocks").then(value => {
-                    if (value > this.conf.scaling.block_queue_limit) {
-                        this.allowRequests = false;
-                    } else {
-                        this.allowRequests = true;
-                        if (this.pendingRequest) {
-                            this.processPending();
-                        }
-                    }
-                });
-            } else {
-                let checkArr = [];
-                for (let i = 0; i < this.conf.scaling.ds_queues; i++) {
-                    const q = this.chain + ":blocks:" + (i + 1);
-                    checkArr.push(this.manager.checkQueueSize(q));
-                }
-                Promise.all(checkArr).then(data => {
-                    if (data.some(el => el > this.conf.scaling.block_queue_limit)) {
-                        if (this.allowRequests) {
-                            hLog('ship reader paused!', data);
-                        }
-                        this.allowRequests = false;
-                    } else {
-                        this.allowRequests = true;
-                        if (this.pendingRequest) {
-                            this.processPending();
-                        }
-                    }
-                });
-            }
-        }, this.queueSizeCheckInterval);
-    }
+    // private startQueueWatcher() {
+    //     setInterval(() => {
+    //         if (this.isLiveReader) {
+    //             this.manager.checkQueueSize(this.chain + ":live_blocks").then(value => {
+    //                 if (value > this.conf.scaling.block_queue_limit) {
+    //                     this.allowRequests = false;
+    //                 } else {
+    //                     this.allowRequests = true;
+    //                     if (this.pendingRequest) {
+    //                         this.processPending();
+    //                     }
+    //                 }
+    //             });
+    //         } else {
+    //             let checkArr = [];
+    //             for (let i = 0; i < this.conf.scaling.ds_queues; i++) {
+    //                 const q = this.chain + ":blocks:" + (i + 1);
+    //                 checkArr.push(this.manager.checkQueueSize(q));
+    //             }
+    //             Promise.all(checkArr).then(data => {
+    //                 if (data.some(el => el > this.conf.scaling.block_queue_limit)) {
+    //                     if (this.allowRequests) {
+    //                         hLog('ship reader paused!', data);
+    //                     }
+    //                     this.allowRequests = false;
+    //                 } else {
+    //                     this.allowRequests = true;
+    //                     if (this.pendingRequest) {
+    //                         this.processPending();
+    //                     }
+    //                 }
+    //             });
+    //         }
+    //     }, this.queueSizeCheckInterval);
+    // }
 
     private processPending() {
         this.requestBlockRange(this.pendingRequest[0], this.pendingRequest[1]);
@@ -641,9 +769,10 @@ export default class StateReader extends HyperionWorker {
         );
     }
 
+
     private handleLostConnection() {
         this.recovery = true;
-        this.ship.close();
+        this.ship.close(false);
         hLog(`Retrying connection in 5 seconds... [attempt: ${this.reconnectCount + 1}]`);
         debugLog(`PENDING REQUESTS:', ${this.pendingRequest}`);
         debugLog(`LOCAL BLOCK:', ${this.local_block_num}`);
@@ -652,4 +781,11 @@ export default class StateReader extends HyperionWorker {
             this.startWS();
         }, 5000);
     }
+
+    private emitCompletedSignal() {
+        process.send({
+            event: 'completed',
+            id: process.env['worker_id']
+        });
+    }
 }

workers/ws-router.ts

@@ -1,7 +1,7 @@
 import {HyperionWorker} from "./hyperionWorker";
 
 import {Server, Socket} from "socket.io";
-import {checkFilter, hLog} from "../helpers/common_functions";
+import {checkDeltaFilter, checkFilter, hLog} from "../helpers/common_functions";
 import {createServer} from "http";
 
 const greylist = ['eosio.token'];
@@ -34,21 +34,21 @@ export default class WSRouter extends HyperionWorker {
         this.ch.consume(this.q, this.onConsume.bind(this));
     }
 
+    appendIdAndEmit(event, data) {
+        this.io.emit(event, {
+            chain_id: this.manager.conn.chains[this.chain]?.chain_id,
+            ...data
+        });
+    };
 
     onIpcMessage(msg: any): void {
         switch (msg.event) {
             case 'lib_update': {
-                this.io.emit('lib_update', {
-                    chain_id: this.manager.conn.chains[this.chain]?.chain_id,
-                    ...msg.data
-                });
+                this.appendIdAndEmit('lib_update', msg.data);
                 break;
             }
             case 'fork_event': {
-                this.io.emit('fork_event', {
-                    chain_id: this.manager.conn.chains[this.chain]?.chain_id,
-                    ...msg.data
-                });
+                this.appendIdAndEmit('fork_event', msg.data);
                 break;
             }
         }
@@ -68,7 +68,17 @@ export default class WSRouter extends HyperionWorker {
         // push to plugin handlers
         this.mLoader.processStreamEvent(msg);
 
+        // process incoming live messages
         switch (msg.properties.headers.event) {
+            case 'block': {
+                // forward block events to all APIs
+                this.io.of('/').emit('block', {
+                    serverTime: Date.now(),
+                    blockNum: msg.properties.headers.blockNum,
+                    content: msg.content
+                });
+                break;
+            }
             case 'trace': {
                 const actHeader = msg.properties.headers;
                 const code = actHeader.account;
@@ -146,7 +156,7 @@ export default class WSRouter extends HyperionWorker {
             }
 
             default: {
-                console.log('Unindentified message!');
+                console.log('Unidentified message!');
                 console.log(msg);
             }
         }
@@ -206,7 +216,10 @@ export default class WSRouter extends HyperionWorker {
     addActionRequest(data, id) {
         const req = data.request;
         if (typeof req.account !== 'string') {
-            return {status: 'FAIL', reason: 'invalid request'};
+            return {
+                status: 'FAIL',
+                reason: 'invalid request'
+            };
         }
         if (greylist.indexOf(req.contract) !== -1) {
             if (req.account === '' || req.account === req.contract) {
@@ -219,6 +232,7 @@ export default class WSRouter extends HyperionWorker {
         const link = {
             type: 'action',
             relay: id,
+            reqUUID: data.reqUUID,
             client: data.client_socket,
             filters: req.filters,
             account: req.account,
@@ -230,7 +244,10 @@ export default class WSRouter extends HyperionWorker {
             if (req.account !== '') {
                 this.appendToL1Map(this.notifiedMap, req.account, link);
             } else {
-                return {status: 'FAIL', reason: 'invalid request'};
+                return {
+                    status: 'FAIL',
+                    reason: 'invalid request'
+                };
             }
         }
         this.addToClientIndex(data, id, [req.contract, req.action, req.account]);
@@ -257,6 +274,7 @@ export default class WSRouter extends HyperionWorker {
         const link = {
             type: 'delta',
             relay: id,
+            reqUUID: data.reqUUID,
             client: data.client_socket,
             filters: data.request.filters,
             payer: data.request.payer,
@@ -268,7 +286,10 @@ export default class WSRouter extends HyperionWorker {
             if (req.payer !== '' && req.payer !== '*') {
                 this.appendToL1Map(this.payerMap, req.payer, link);
             } else {
-                return {status: 'FAIL', reason: 'invalid request'};
+                return {
+                    status: 'FAIL',
+                    reason: 'invalid request'
+                };
             }
         }
         this.addToClientIndex(data, id, [req.code, req.table, req.payer]);
@@ -340,7 +361,7 @@ export default class WSRouter extends HyperionWorker {
                         if (result.status === 'OK') {
                             callback(result);
                         } else {
-                            callback(result.reason);
+                            callback(result);
                         }
                         break;
                     }
@@ -349,7 +370,7 @@ export default class WSRouter extends HyperionWorker {
                         if (result.status === 'OK') {
                             callback(result);
                         } else {
-                            callback(result.reason);
+                            callback(result);
                         }
                         break;
                     }
@@ -412,7 +433,7 @@ export default class WSRouter extends HyperionWorker {
                 });
             }
             if (allow) {
-                relay.emit('trace', {client: link.client, message: msg});
+                relay.emit('trace', {client: link.client, req: link.reqUUID, message: msg});
                 this.totalRoutedMessages++;
             }
         }
@@ -427,15 +448,15 @@ export default class WSRouter extends HyperionWorker {
             } else {
                 allow = true;
             }
-            // if (link.filters?.length > 0) {
-            //     // check filters
-            //     const _parsedMsg = JSON.parse(msg);
-            //     allow = link.filters.every(filter => {
-            //         return checkDeltaFilter(filter, _parsedMsg);
-            //     });
-            // }
+            if (link.filters?.length > 0) {
+                // check filters
+                const _parsedMsg = JSON.parse(msg);
+                allow = link.filters.every(filter => {
+                    return checkDeltaFilter(filter, _parsedMsg);
+                });
+            }
             if (allow) {
-                relay.emit('delta', {client: link.client, message: msg});
+                relay.emit('delta', {client: link.client, req: link.reqUUID, message: msg});
                 this.totalRoutedMessages++;
             }
         }