update all packages - pending migrations

- stream-client v2.0.0 compatibility
- improved transitions on high volume streams
- added api.stream_scroll_limit (default -1 -> unlimited)
specify the maximum number of documents that can be requested during a history request in the streaming api
- added api.stream_scroll_batch (default 500)
specify the number of documents per streamed history request

.gitignore

@@ -77,9 +77,6 @@ addons/**/*.js.map
 scripts/hpm.js
 scripts/hpm.js.map
 
-scripts/hyp-config.js
-scripts/hyp-config.js.map
-
 plugins/.state.json
 plugins/repos
 plugins/*.js
@@ -91,3 +88,12 @@ docker/rabbitmq/data
 docker/eosio/data
 
 configuration_backups
+
+scripts/hyp-config.js
+scripts/hyp-config.js.map
+
+scripts/hyp-repair.js
+scripts/hyp-repair.js.map
+
+scripts/repair-cli/*.js
+scripts/repair-cli/*.js.map

api/helpers/functions.ts

@@ -2,7 +2,8 @@ import {createHash} from "crypto";
 import * as _ from "lodash";
 import {FastifyInstance, FastifyReply, FastifyRequest, FastifySchema, HTTPMethods} from "fastify";
 import got from "got";
-import {checkFilter, hLog} from "../../helpers/common_functions";
+import {checkDeltaFilter, checkFilter, hLog} from "../../helpers/common_functions";
+import {Socket} from "socket.io";
 
 const deltaQueryFields = ['code', 'table', 'scope', 'payer'];
 
@@ -15,51 +16,141 @@ export async function streamPastDeltas(fastify: FastifyInstance, socket, data) {
     deltaQueryFields.forEach(f => {
         addTermMatch(data, search_body, f);
     });
+
+    const onDemandDeltaFilters = [];
+    if (data.filters.length > 0) {
+        data.filters.forEach(f => {
+            if (f.field && f.value) {
+                if (f.field.startsWith('@') && !f.field.startsWith('data')) {
+                    const _q = {};
+                    _q[f.field] = f.value;
+                    search_body.query.bool.must.push({'term': _q});
+                } else {
+                    onDemandDeltaFilters.push(f);
+                }
+            }
+        });
+    }
+
     const responseQueue = [];
+
     let counter = 0;
+    let total = 0;
+    let totalFiltered = 0;
+    let longScroll = false;
+
+    // console.log(JSON.stringify(search_body, null, 2));
+
     const init_response = await fastify.elastic.search({
         index: fastify.manager.chain + '-delta-*',
         scroll: '30s',
-        size: 20,
+        size: fastify.manager.config.api.stream_scroll_batch || 500,
         body: search_body,
     });
+
+    const scrollLimit = fastify.manager.config.api.stream_scroll_limit;
+    if (scrollLimit && scrollLimit !== -1 && init_response.body.hits.total.value > scrollLimit) {
+        const errorMsg = `Requested ${init_response.body.hits.total.value} deltas, limit is ${scrollLimit}.`;
+        socket.emit('message', {
+            reqUUID: socket.data.reqUUID,
+            type: 'delta_trace', mode: 'history', messages: [],
+            error: errorMsg
+        });
+        return {status: false, error: errorMsg};
+    }
+
+    if (init_response.body.hits.total.value > 10000) {
+        total = init_response.body.hits.total.value;
+        longScroll = true;
+        hLog(`Attention! Long scroll (deltas) is running!`);
+    }
+
+    // emit first block
+    if (init_response.body.hits.hits.length > 0) {
+        emitTraceInit(socket, init_response.body.hits.hits[0]._source.block_num, init_response.body.hits.total.value);
+    }
+
     responseQueue.push(init_response);
+
+    let lastTransmittedBlock = 0;
+    let pendingScrollId = '';
     while (responseQueue.length) {
+        let filterCount = 0;
         const {body} = responseQueue.shift();
+        pendingScrollId = body['_scroll_id'];
+        const enqueuedMessages = [];
         counter += body['hits']['hits'].length;
+
+        for (const doc of body['hits']['hits']) {
+            let allow = false;
+            if (onDemandDeltaFilters.length > 0) {
+                allow = onDemandDeltaFilters.every(filter => {
+                    return checkDeltaFilter(filter, doc._source);
+                });
+            } else {
+                allow = true;
+            }
+            if (allow) {
+                enqueuedMessages.push(doc._source);
+            } else {
+                filterCount++;
+            }
+            // set last block
+            if (doc._source.block_num > lastTransmittedBlock) {
+                lastTransmittedBlock = doc._source.block_num;
+            }
+        }
+
+        totalFiltered += filterCount;
+
         if (socket.connected) {
-            socket.emit('message', {
-                type: 'delta_trace',
-                mode: 'history',
-                messages: body['hits']['hits'].map(doc => doc._source),
-            });
+            if (enqueuedMessages.length > 0) {
+                socket.emit('message', {
+                    reqUUID: socket.data.reqUUID,
+                    type: 'delta_trace',
+                    mode: 'history',
+                    messages: enqueuedMessages,
+                    filtered: filterCount
+                });
+            }
         } else {
             hLog('LOST CLIENT');
             break;
         }
+
+        if (longScroll) {
+            hLog(`Progress: ${counter + totalFiltered}/${total}`);
+        }
+
         if (body['hits'].total.value === counter) {
-            hLog(`${counter} past deltas streamed to ${socket.id}`);
+            hLog(`${counter} past deltas streamed to ${socket.id} (${totalFiltered} filtered)`);
             break;
         }
 
         const next_response = await fastify.elastic.scroll({
-            body: {
-                scroll_id: body['_scroll_id'],
-                scroll: '30s'
-            }
+            body: {scroll_id: body['_scroll_id'], scroll: '30s'}
         });
         responseQueue.push(next_response);
     }
+
+    // destroy scroll context
+    await fastify.elastic.clearScroll({scroll_id: pendingScrollId});
+    return {status: true, lastTransmittedBlock};
+}
+
+export function emitTraceInit(socket: Socket, firstBlock: number, totalResults: number) {
+    socket.emit('message', {
+        reqUUID: socket.data.reqUUID,
+        type: 'trace_init',
+        mode: 'history',
+        first_block: firstBlock,
+        results: totalResults
+    });
 }
 
 export async function streamPastActions(fastify: FastifyInstance, socket, data) {
-    const search_body = {
-        query: {bool: {must: []}},
-        sort: {global_sequence: 'asc'},
-    };
-
+    const search_body = {query: {bool: {must: []}}, sort: {global_sequence: 'asc'}};
     await addBlockRangeOpts(data, search_body, fastify);
-
     if (data.account !== '') {
         search_body.query.bool.must.push({
             bool: {
@@ -96,18 +187,50 @@ export async function streamPastActions(fastify: FastifyInstance, socket, data)
 
     const responseQueue = [];
     let counter = 0;
+    let total = 0;
+    let totalFiltered = 0;
+    let longScroll = false;
 
     const init_response = await fastify.elastic.search({
         index: fastify.manager.chain + '-action-*',
         scroll: '30s',
-        size: 20,
+        size: fastify.manager.config.api.stream_scroll_batch || 500,
         body: search_body,
     });
+
+    const scrollLimit = fastify.manager.config.api.stream_scroll_limit;
+    if (scrollLimit && scrollLimit !== -1 && init_response.body.hits.total.value > scrollLimit) {
+        const errorMsg = `Requested ${init_response.body.hits.total.value} actions, limit is ${scrollLimit}.`;
+        socket.emit('message', {
+            reqUUID: socket.data.reqUUID,
+            type: 'action_trace', mode: 'history', messages: [],
+            error: `Requested ${init_response.body.hits.total.value} actions, limit is ${scrollLimit}.`
+        });
+        return {status: false, error: errorMsg};
+    }
+
+    if (init_response.body.hits.total.value > 10000) {
+        total = init_response.body.hits.total.value;
+        longScroll = true;
+        hLog(`Attention! Long scroll (actions) is running!`);
+    }
+
+    // emit first block
+    if (init_response.body.hits.hits.length > 0) {
+        emitTraceInit(socket, init_response.body.hits.hits[0]._source.block_num, init_response.body.hits.total.value);
+    }
+
     responseQueue.push(init_response);
+
+    let lastTransmittedBlock = 0;
+    let pendingScrollId = '';
     while (responseQueue.length) {
+        let filterCount = 0;
         const {body} = responseQueue.shift();
+        pendingScrollId = body['_scroll_id'];
         const enqueuedMessages = [];
         counter += body['hits']['hits'].length;
+
         for (const doc of body['hits']['hits']) {
             let allow = false;
             if (onDemandFilters.length > 0) {
@@ -119,32 +242,55 @@ export async function streamPastActions(fastify: FastifyInstance, socket, data)
             }
             if (allow) {
                 enqueuedMessages.push(doc._source);
+            } else {
+                filterCount++;
+            }
+            // set last block
+            if (doc._source.block_num > lastTransmittedBlock) {
+                lastTransmittedBlock = doc._source.block_num;
             }
         }
+
+        totalFiltered += filterCount;
+
         if (socket.connected) {
-            socket.emit('message', {type: 'action_trace', mode: 'history', messages: enqueuedMessages});
+            if (enqueuedMessages.length > 0) {
+                socket.emit('message', {
+                    reqUUID: socket.data.reqUUID,
+                    type: 'action_trace',
+                    mode: 'history',
+                    messages: enqueuedMessages,
+                    filtered: filterCount
+                });
+            }
         } else {
             hLog('LOST CLIENT');
             break;
         }
+
+        if (longScroll) {
+            hLog(`Progress: ${counter + totalFiltered}/${total}`);
+        }
+
         if (body['hits'].total.value === counter) {
-            hLog(`${counter} past actions streamed to ${socket.id}`);
+            hLog(`${counter} past actions streamed to ${socket.id} (${totalFiltered} filtered)`);
             break;
         }
-        if (init_response.body.hits.total.value < 1000) {
-            const next_response = await fastify.elastic.scroll({
-                body: {scroll_id: body['_scroll_id'], scroll: '30s'}
-            });
-            responseQueue.push(next_response);
-        } else {
-            hLog('Request too large!');
-            socket.emit('message', {type: 'action_trace', mode: 'history', messages: []});
-        }
+
+        const next_response = await fastify.elastic.scroll({
+            body: {scroll_id: body['_scroll_id'], scroll: '30s'}
+        });
+        responseQueue.push(next_response);
     }
+
+    // destroy scroll context
+    await fastify.elastic.clearScroll({scroll_id: pendingScrollId});
+    return {status: true, lastTransmittedBlock};
+
 }
 
 export function addTermMatch(data, search_body, field) {
-    if (data[field] !== '*' && data[field] !== '') {
+    if (data[field] && data[field] !== '*' && data[field] !== '') {
         const termQuery = {};
         termQuery[field] = data[field];
         search_body.query.bool.must.push({'term': termQuery});
@@ -217,6 +363,8 @@ export function extendResponseSchema(responseProps: any) {
         cached: {type: "boolean"},
         hot_only: {type: "boolean"},
         lib: {type: "number"},
+        last_indexed_block: {type: "number"},
+        last_indexed_block_time: {type: "string"},
         total: {
             type: "object",
             properties: {
@@ -367,7 +515,8 @@ function bigint2Milliseconds(input: bigint) {
 
 const defaultRouteCacheMap = {
     get_resource_usage: 3600,
-    get_creator: 3600 * 24
+    get_creator: 3600 * 24,
+    health: 10
 }
 
 export async function timedQuery(
@@ -384,16 +533,31 @@ export async function timedQuery(
         request.method === 'POST' ? request.body : request.query
     );
 
+    let lastIndexedBlockNumber = null;
+    let lastIndexedBlockTime = null;
+    const lastIndexedBlock = await fastify.redis.get(`${fastify.manager.chain}:last_idx_block`);
+    if (lastIndexedBlock) {
+        const arr = lastIndexedBlock.split("@");
+        if (arr.length === 2) {
+            lastIndexedBlockNumber = parseInt(arr[0], 10);
+            lastIndexedBlockTime = arr[1];
+        }
+    }
+
     if (cachedResponse && !request.query["ignoreCache"]) {
         // add cached query time
         cachedResponse['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
+        if (lastIndexedBlock) {
+            cachedResponse['last_indexed_block'] = lastIndexedBlockNumber;
+            cachedResponse['last_indexed_block_time'] = lastIndexedBlockTime;
+        }
         return cachedResponse;
     }
 
     // call query function
     const response = await queryFunction(fastify, request);
 
-    // save response to cash
+    // save response to cache
     if (hash) {
         let EX = null;
         if (defaultRouteCacheMap[route]) {
@@ -405,6 +569,10 @@ export async function timedQuery(
     // add normal query time
     if (response) {
         response['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
+        if (lastIndexedBlock) {
+            response['last_indexed_block'] = lastIndexedBlockNumber;
+            response['last_indexed_block_time'] = lastIndexedBlockTime;
+        }
         return response;
     } else {
         return {};

api/plugins/fastify-eosjs.ts

@@ -3,17 +3,16 @@ import {Api} from "eosjs/dist";
 import fp from "fastify-plugin";
 
 export default fp(async (fastify: FastifyInstance, options: FastifyPluginOptions): Promise<void> => {
-	const rpc = fastify.manager.nodeosJsonRPC;
-	const chain_data = await rpc.get_info();
-	const api = new Api({
-		rpc,
-		signatureProvider: null,
-		chainId: chain_data.chain_id,
-		textDecoder: new TextDecoder(),
-		textEncoder: new TextEncoder(),
-	});
-	fastify.decorate('eosjs', {api, rpc});
+    const rpc = fastify.manager.nodeosJsonRPC;
+    const api = new Api({
+        rpc,
+        signatureProvider: null,
+        chainId: options.chain_id,
+        textDecoder: new TextDecoder(),
+        textEncoder: new TextEncoder(),
+    });
+    fastify.decorate('eosjs', {api, rpc});
 }, {
-	fastify: '>=2.0.0',
-	name: 'fastify-eosjs'
+    fastify: '>=2.0.0',
+    name: 'fastify-eosjs'
 });

api/routes/v1-history/get_actions/get_actions.ts

@@ -6,6 +6,7 @@ import * as AbiEOS from "@eosrio/node-abieos";
 import {ApiResponse} from "@elastic/elasticsearch";
 import {TextDecoder, TextEncoder} from "util";
 import {JsonRpc} from "eosjs/dist";
+import {terms} from "../../v2-history/get_actions/definitions";
 
 const abi_remapping = {
     "_Bool": "bool"
@@ -144,7 +145,6 @@ async function getContractAtBlock(esClient, rpc, chain, accountName: string, blo
 }
 
 
-const terms = ["notified", "act.authorization.actor"];
 const extendedActions = new Set(["transfer", "newaccount", "updateauth"]);
 
 async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
@@ -182,8 +182,8 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
     }
     pos = parseInt(reqBody.pos || 0, 10);
     offset = parseInt(reqBody.offset || 20, 10);
-    let from, size;
-    from = size = 0;
+    let from = 0;
+    let size = 0;
     if (pos === -1) {
         if (offset < 0) {
             from = 0;
@@ -272,6 +272,7 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
         }
     };
+    // console.log(JSON.stringify(esOpts, null, 2));
     const pResults = await Promise.all([fastify.eosjs.rpc.get_info(), fastify.elastic['search'](esOpts)]);
     const results = pResults[1];
     const response = {

api/routes/v1-history/get_transaction/get_transaction.ts

@@ -1,265 +1,190 @@
 import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
-import {createHash} from "crypto";
-import flatstr from 'flatstr';
 import {GetInfoResult} from "eosjs/dist/eosjs-rpc-interfaces";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
-	if (typeof request.body === 'string') {
-		request.body = JSON.parse(request.body)
-	}
-	const body: any = request.body;
-	const redis = fastify.redis;
-	const trxId = body.id.toLowerCase();
-	const conf = fastify.manager.config;
-	const cachedData = await redis.hgetall('trx_' + trxId);
 
-	const response: any = {
-		"id": body.id,
-		"trx": {
-			"receipt": {
-				"status": "executed",
-				"cpu_usage_us": 0,
-				"net_usage_words": 0,
-				"trx": [1, {
-					"signatures": "",
-					"compression": "none",
-					"packed_context_free_data": "",
-					"packed_trx": ""
-				}]
-			},
-			"trx": {}
-		},
-		"block_num": 0,
-		"block_time": "",
-		"last_irreversible_block": undefined,
-		"traces": []
-	};
+    if (request.method === 'POST' && !request.body) {
+        throw new Error("missing POST body");
+    }
 
-	let hits;
+    if (typeof request.body === 'string') {
+        request.body = JSON.parse(request.body)
+    }
+    const body: any = request.body;
+    const redis = fastify.redis;
+    const trxId = body.id.toLowerCase();
 
-	// build get_info request with caching
-	const $getInfo = new Promise<GetInfoResult>(resolve => {
-		const key = `${fastify.manager.chain}_get_info`;
-		fastify.redis.get(key).then(value => {
-			if (value) {
-				resolve(JSON.parse(value));
-			} else {
-				fastify.eosjs.rpc.get_info().then(value1 => {
-					fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
-					resolve(value1);
-				}).catch((reason) => {
-					console.log(reason);
-					response.error = 'failed to get last_irreversible_block_num'
-					resolve(null);
-				});
-			}
-		});
-	});
+    const conf = fastify.manager.config;
+    const cachedData = await redis.hgetall('trx_' + trxId);
 
-	// reconstruct hits from cached data
-	if (cachedData && Object.keys(cachedData).length > 0) {
-		const gsArr = [];
-		for (let cachedDataKey in cachedData) {
-			gsArr.push(cachedData[cachedDataKey]);
-		}
-		gsArr.sort((a, b) => {
-			return a.global_sequence - b.global_sequence;
-		});
-		hits = gsArr.map(value => {
-			return {
-				_source: JSON.parse(value)
-			};
-		});
-		const promiseResults = await Promise.all([
-			redis.ttl('trx_' + trxId),
-			$getInfo
-		]);
-		response.cache_expires_in = promiseResults[0];
-		response.last_irreversible_block = promiseResults[1].last_irreversible_block_num;
-	}
+    const response: any = {
+        "id": body.id,
+        "trx": {
+            "receipt": {
+                "status": "executed",
+                "cpu_usage_us": 0,
+                "net_usage_words": 0,
+                "trx": [1, {
+                    "signatures": "",
+                    "compression": "none",
+                    "packed_context_free_data": "",
+                    "packed_trx": ""
+                }]
+            },
+            "trx": {}
+        },
+        "block_num": 0,
+        "block_time": "",
+        "last_irreversible_block": undefined,
+        "traces": []
+    };
 
-	// search on ES if cache is not present
-	if (!hits) {
-		const _size = conf.api.limits.get_trx_actions || 100;
-		const blockHint = parseInt(body.block_num_hint, 10);
-		let indexPattern = '';
-		if (blockHint) {
-			const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
-			indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
-		} else {
-			indexPattern = fastify.manager.chain + '-action-*';
-		}
-		let pResults;
-		try {
+    let hits;
 
-			// build search request
-			const $search = fastify.elastic.search({
-				index: indexPattern,
-				size: _size,
-				body: {
-					query: {bool: {must: [{term: {trx_id: trxId}}]}},
-					sort: {global_sequence: "asc"}
-				}
-			});
+    // build get_info request with caching
+    const $getInfo = new Promise<GetInfoResult>(resolve => {
+        const key = `${fastify.manager.chain}_get_info`;
+        fastify.redis.get(key).then(value => {
+            if (value) {
+                resolve(JSON.parse(value));
+            } else {
+                fastify.eosjs.rpc.get_info().then(value1 => {
+                    fastify.redis.set(key, JSON.stringify(value1), 'EX', 6);
+                    resolve(value1);
+                }).catch((reason) => {
+                    console.log(reason);
+                    response.error = 'failed to get last_irreversible_block_num'
+                    resolve(null);
+                });
+            }
+        });
+    });
 
-			// execute in parallel
-			pResults = await Promise.all([$getInfo, $search]);
-		} catch (e) {
-			console.log(e.message);
-			if (e.meta.statusCode === 404) {
-				return response;
-			}
-		}
-		hits = pResults[1]['body']['hits']['hits'];
-		response.last_irreversible_block = pResults[0].last_irreversible_block_num;
-	}
+    // reconstruct hits from cached data
+    if (cachedData && Object.keys(cachedData).length > 0) {
+        const gsArr = [];
+        for (let cachedDataKey in cachedData) {
+            gsArr.push(cachedData[cachedDataKey]);
+        }
+        gsArr.sort((a, b) => {
+            return a.global_sequence - b.global_sequence;
+        });
+        hits = gsArr.map(value => {
+            return {
+                _source: JSON.parse(value)
+            };
+        });
+        const promiseResults = await Promise.all([
+            redis.ttl('trx_' + trxId),
+            $getInfo
+        ]);
+        response.cache_expires_in = promiseResults[0];
+        response.last_irreversible_block = promiseResults[1].last_irreversible_block_num;
+    }
+
+    // search on ES if cache is not present
+    if (!hits) {
+        const _size = conf.api.limits.get_trx_actions || 100;
+        const blockHint = parseInt(body.block_num_hint, 10);
+        let indexPattern = '';
+        if (blockHint) {
+            const idxPart = Math.ceil(blockHint / conf.settings.index_partition_size).toString().padStart(6, '0');
+            indexPattern = fastify.manager.chain + `-action-${conf.settings.index_version}-${idxPart}`;
+        } else {
+            indexPattern = fastify.manager.chain + '-action-*';
+        }
+        let pResults;
+        try {
+
+            // build search request
+            const $search = fastify.elastic.search({
+                index: indexPattern,
+                size: _size,
+                body: {
+                    query: {bool: {must: [{term: {trx_id: trxId}}]}},
+                    sort: {global_sequence: "asc"}
+                }
+            });
+
+            // execute in parallel
+            pResults = await Promise.all([$getInfo, $search]);
+        } catch (e) {
+            console.log(e.message);
+            if (e.meta.statusCode === 404) {
+                return response;
+            }
+        }
+        hits = pResults[1]['body']['hits']['hits'];
+        response.last_irreversible_block = pResults[0].last_irreversible_block_num;
+    }
 
 
-	if (hits.length > 0) {
-		const actions = hits;
-		response.trx.trx = {
-			"expiration": "",
-			"ref_block_num": 0,
-			"ref_block_prefix": 0,
-			"max_net_usage_words": 0,
-			"max_cpu_usage_ms": 0,
-			"delay_sec": 0,
-			"context_free_actions": [],
-			"actions": [],
-			"transaction_extensions": [],
-			"signatures": [],
-			"context_free_data": []
-		};
-		let traces = {};
-		let seqNum = 0;
-		for (let action of actions) {
-			action = action._source;
-			mergeActionMeta(action);
-
-			// action.act['hex_data'] = Buffer.from(flatstr(JSON.stringify(action.act.data))).toString('hex');
-
-			if (action.parent === 0) {
-				response.trx.trx.actions.push(action.act);
-			}
-			response.block_num = action.block_num;
-			response.block_time = action['@timestamp'];
-			seqNum += 1;
-			let trace = {
-				receipt: {
-					receiver: action.act.account,
-					global_sequence: String(action.global_sequence),
-					auth_sequence: [
-						action.act.authorization[0].actor,
-						seqNum
-					],
-					act_digest: '',
-					recv_sequence: seqNum,
-					code_sequence: 1,
-					abi_sequence: 1
-				},
-				act: action.act,
-				account_ram_deltas: action.account_ram_deltas || [],
-				context_free: false,
-				block_num: action.block_num,
-				block_time: action['@timestamp'],
-				console: "",
-				elapsed: 0,
-				except: null,
-				inline_traces: [],
-				producer_block_id: "",
-				trx_id: body.id,
-				notified: action.notified
-			};
-			let hash = createHash('sha256');
-			hash.update(JSON.stringify(action.act));
-			trace.receipt.act_digest = hash.digest('hex');
-			traces[action.global_sequence] = trace;
-		}
-
-		actions.forEach(action => {
-			action = action._source;
-			for (let i = 0; i < traces[action.global_sequence].notified.length; i++) {
-				if (traces[action.global_sequence].notified[i] === action.act.account) {
-					traces[action.global_sequence].notified.splice(i, 1);
-					break;
-				}
-			}
-			if (action.parent !== 0 && action.parent) {
-				if (traces[action.parent]) {
-					for (let i = 0; i < traces[action.parent].notified.length; i++) {
-						if (traces[action.parent].notified[i] === action.act.account) {
-							traces[action.parent].notified.splice(i, 1);
-							break;
-						}
-					}
-					traces[action.parent].inline_traces.push(traces[action.global_sequence]);
-				}
-			}
-		});
-
-		actions.forEach(action => {
-			action = action._source;
-			response.traces.push(traces[action.global_sequence]);
-			if (traces[action.global_sequence] && traces[action.global_sequence].notified) {
-				traces[action.global_sequence].notified.forEach((note, index) => {
-					seqNum += 1;
-					let trace = {
-						receipt: {
-							receiver: note,
-							global_sequence: String(action.global_sequence + index + 1),
-							auth_sequence: [
-								action.act.authorization[0].actor,
-								seqNum
-							],
-							act_digest: traces[action.global_sequence].receipt.act_digest,
-							recv_sequence: seqNum,
-							code_sequence: 1,
-							abi_sequence: 1
-						},
-						account_ram_deltas: action.account_ram_deltas || [],
-						act: action.act,
-						block_num: action.block_num,
-						block_time: action['@timestamp'],
-						console: "",
-						context_free: false,
-						elapsed: 0,
-						except: null,
-						inline_traces: [],
-						producer_block_id: "",
-						trx_id: body.id,
-					};
-					traces[action.global_sequence].inline_traces.unshift(trace);
-					response.traces.push(trace);
-				});
-				delete traces[action.global_sequence].notified;
-			}
-		});
-	} else {
-		const errmsg = "Transaction " + body.id.toLowerCase() + " not found in history and no block hint was given";
-		return {
-			code: 500,
-			message: "Internal Service Error",
-			error: {
-				code: 3040011,
-				name: "tx_not_found",
-				what: "The transaction can not be found",
-				details: [
-					{
-						message: errmsg,
-						file: "",
-						line_number: 1,
-						method: "get_transaction"
-					}
-				]
-			}
-		}
-	}
-	return response;
+    if (hits.length > 0) {
+        const actions = hits;
+        response.trx.trx = {
+            "expiration": "",
+            "ref_block_num": 0,
+            "ref_block_prefix": 0,
+            "max_net_usage_words": 0,
+            "max_cpu_usage_ms": 0,
+            "delay_sec": 0,
+            "context_free_actions": [],
+            "actions": [],
+            "transaction_extensions": [],
+            "signatures": [],
+            "context_free_data": []
+        };
+        for (let action of actions) {
+            action = action._source;
+            mergeActionMeta(action);
+            response.block_num = action.block_num;
+            response.block_time = action['@timestamp'];
+            for (const receipt of action.receipts) {
+                if (action.act_digest) {
+                    receipt.act_digest = action.act_digest.toLowerCase();
+                }
+                response.traces.push({
+                    receipt: receipt,
+                    act: action.act,
+                    account_ram_deltas: action.account_ram_deltas || [],
+                    context_free: false,
+                    block_num: action.block_num,
+                    block_time: action['@timestamp'],
+                    console: "",
+                    elapsed: 0,
+                    except: null,
+                    inline_traces: [],
+                    producer_block_id: "",
+                    trx_id: body.id
+                });
+            }
+        }
+    } else {
+        const errmsg = "Transaction " + body.id.toLowerCase() + " not found in history and no block hint was given";
+        return {
+            code: 500,
+            message: "Internal Service Error",
+            error: {
+                code: 3040011,
+                name: "tx_not_found",
+                what: "The transaction can not be found",
+                details: [
+                    {
+                        message: errmsg,
+                        file: "",
+                        line_number: 1,
+                        method: "get_transaction"
+                    }
+                ]
+            }
+        }
+    }
+    return response;
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply) => {
-		reply.send(await timedQuery(getTransaction, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getTransaction, fastify, request, route));
+    }
 }

api/routes/v2-history/get_actions/functions.ts

@@ -70,6 +70,15 @@ function addRangeQuery(queryStruct, prop, pkey, query) {
 
 export function applyTimeFilter(query, queryStruct) {
     if (query['after'] || query['before']) {
+
+        if (query['after']?.includes(' ')) {
+            query['after'] = query['after'].replace(' ', 'T');
+        }
+
+        if (query['before']?.includes(' ')) {
+            query['before'] = query['before'].replace(' ', 'T');
+        }
+
         if (query['after']?.includes('T') || query['before']?.includes('T')) {
             let _lte = "now";
             let _gte = "0";

api/routes/v2-history/get_actions/get_actions.ts

@@ -52,12 +52,15 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
         indexPattern = fastify.manager.chain + '-action';
     }
 
-    const esResults = await fastify.elastic.search({
+    const esOpts = {
         "index": indexPattern,
         "from": skip || 0,
         "size": (limit > maxActions ? maxActions : limit) || 10,
         "body": query_body
-    });
+    };
+
+    // console.log(JSON.stringify(esOpts, null, 2));
+    const esResults = await fastify.elastic.search(esOpts);
 
     const results = esResults['body']['hits'];
     const response: any = {
@@ -107,13 +110,14 @@ async function getActions(fastify: FastifyInstance, request: FastifyRequest) {
             }
 
             if (query.simple) {
+                let notified = new Set(action.receipts.map(r => r.receiver));
                 response.simple_actions.push({
                     block: action['block_num'],
                     irreversible: response.lib !== 0 ? action['block_num'] < response.lib : undefined,
                     timestamp: action['@timestamp'],
                     transaction_id: action['trx_id'],
                     actors: action['act']['authorization'].map(a => `${a.actor}@${a.permission}`).join(","),
-                    notified: action['notified'].join(','),
+                    notified: [...notified].join(','),
                     contract: action['act']['account'],
                     action: action['act']['name'],
                     data: action['act']['data']

api/routes/v2-history/get_actions/index.ts

@@ -3,6 +3,7 @@ import {getActionsHandler} from "./get_actions";
 import {addApiRoute, extendQueryStringSchema, extendResponseSchema, getRouteName} from "../../../helpers/functions";
 
 export const getActionResponseSchema = {
+    "@timestamp": {type: "string"},
     "timestamp": {type: "string"},
     "block_num": {type: "number"},
     "block_id": {type: "string"},

api/routes/v2-history/get_schedule/get_schedule.ts

@@ -5,69 +5,101 @@ import {base58ToBinary, binaryToBase58} from "eosjs/dist/eosjs-numeric";
 import {Search} from "@elastic/elasticsearch/api/requestParams";
 
 function convertToLegacyKey(block_signing_key: string) {
-	if (block_signing_key.startsWith("PUB_K1_")) {
-		const buf = base58ToBinary(37, block_signing_key.substr(7));
-		const data = buf.slice(0, buf.length - 4);
-		const merged = Buffer.concat([
-			data,
-			createHash('ripemd160')
-				.update(data)
-				.digest()
-				.slice(0, 4)
-		]);
-		return "EOS" + binaryToBase58(merged);
-	} else {
-		return block_signing_key;
-	}
+    if (block_signing_key.startsWith("PUB_K1_")) {
+        const buf = base58ToBinary(37, block_signing_key.substr(7));
+        const data = buf.slice(0, buf.length - 4);
+        const merged = Buffer.concat([
+            data,
+            createHash('ripemd160')
+                .update(data)
+                .digest()
+                .slice(0, 4)
+        ]);
+        return "EOS" + binaryToBase58(merged);
+    } else {
+        return block_signing_key;
+    }
 }
 
 async function getSchedule(fastify: FastifyInstance, request: FastifyRequest) {
-
-	const query: any = request.query;
-
-	const response: any = {
-		producers: []
-	};
-	const searchParams: Search<any> = {
-		track_total_hits: true,
-		index: fastify.manager.chain + "-block-*",
-		size: 1,
-		body: {
-			query: {
-				bool: {
-					must: []
-				}
-			},
-			sort: {block_num: "desc"}
-		}
-	};
-	if (query.version) {
-		searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": query.version}}});
-	} else {
-		searchParams.body.query.bool.must.push({
-			"exists": {
-				"field": "new_producers.version"
-			}
-		});
-	}
-	const apiResponse = await fastify.elastic.search(searchParams);
-	const results = apiResponse.body.hits.hits;
-	if (results) {
-		response.timestamp = results[0]._source["@timestamp"];
-		response.block_num = results[0]._source.block_num;
-		response.version = results[0]._source.new_producers.version;
-		response.producers = results[0]._source.new_producers.producers.map(prod => {
-			return {
-				...prod,
-				legacy_key: convertToLegacyKey(prod.block_signing_key)
-			}
-		});
-	}
-	return response;
+    const query: any = request.query;
+    const response: any = {
+        producers: []
+    };
+    if (query.mode === 'activated') {
+        const searchParams: Search<any> = {
+            track_total_hits: true,
+            index: fastify.manager.chain + "-block-*",
+            size: 1,
+            body: {
+                query: {bool: {must: []}},
+                sort: {block_num: query.sort || "desc"}
+            }
+        };
+        if (query.version) {
+            searchParams.body.query.bool.must.push({"term": {"new_producers.version": {"value": query.version}}});
+        } else {
+            searchParams.body.query.bool.must.push({"exists": {"field": "new_producers.version"}});
+        }
+        const apiResponse = await fastify.elastic.search(searchParams);
+        const results = apiResponse.body.hits.hits;
+        if (results && results.length > 0) {
+            response.timestamp = results[0]._source["@timestamp"];
+            response.block_num = results[0]._source.block_num;
+            response.version = results[0]._source.new_producers.version;
+            response.producers = results[0]._source.new_producers.producers.map(prod => {
+                return {
+                    ...prod,
+                    legacy_key: convertToLegacyKey(prod.block_signing_key)
+                }
+            });
+        }
+    } else {
+        // default "proposed" mode
+        const searchParams: Search<any> = {
+            track_total_hits: true,
+            index: fastify.manager.chain + "-schedule-*",
+            size: 1,
+            body: {
+                query: {bool: {must: []}},
+                sort: {version: query.sort || "desc"}
+            }
+        };
+        if (query.version) {
+            searchParams.body.query.bool.must.push({"term": {"version": {"value": query.version}}});
+        } else {
+            searchParams.body.query.bool.must.push({"exists": {"field": "version"}});
+        }
+        if (query.producer) {
+            searchParams.body.query.bool.must.push({"term": {"producers.name": {"value": query.producer}}});
+        }
+        if (query.key) {
+            searchParams.body.query.bool.must.push({"term": {"producers.keys": {"value": query.key}}});
+        }
+        const apiResponse = await fastify.elastic.search(searchParams);
+        const results = apiResponse.body.hits.hits;
+        if (results && results.length > 0) {
+            response.timestamp = results[0]._source["@timestamp"];
+            response.proposal_block_num = results[0]._source.block_num;
+            response.version = results[0]._source.version;
+            response.producers = results[0]._source.producers;
+            // sort producers by name
+            // response.producers.sort((a, b) => {
+            //     if (a.name < b.name) {
+            //         return -1;
+            //     }
+            //     if (a.name > b.name) {
+            //         return 1;
+            //     }
+            //     return 0;
+            // });
+        }
+    }
+    return response;
 }
 
 export function getScheduleHandler(fastify: FastifyInstance, route: string) {
-	return async (request: FastifyRequest, reply: FastifyReply) => {
-		reply.send(await timedQuery(getSchedule, fastify, request, route));
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getSchedule, fastify, request, route));
+    }
 }

api/routes/v2-history/get_schedule/index.ts

@@ -15,7 +15,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: 'string'
                 },
                 "key": {
-                    description: 'search by key',
+                    description: 'search by signing key',
                     type: 'string'
                 },
                 "after": {
@@ -26,6 +26,10 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     description: 'filter before specified date (ISO8601)',
                     type: 'string'
                 },
+                "mode": {
+                    description: 'search mode (activated or proposed)',
+                    type: 'string',
+                },
                 "version": {
                     description: 'schedule version',
                     type: 'integer',
@@ -36,6 +40,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         response: extendResponseSchema({
             "timestamp": {type: "string"},
             "block_num": {type: "number"},
+            "proposed_block_num": {type: "number"},
             "version": {type: "number"},
             "producers": {
                 type: "array",
@@ -43,8 +48,10 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                     type: "object",
                     properties: {
                         "producer_name": {type: "string"},
+                        "name": {type: "string"},
                         "block_signing_key": {type: "string"},
-                        "legacy_key": {type: "string"}
+                        "legacy_key": {type: "string"},
+                        "keys": {type: "array", items: {type: "string"}}
                     }
                 }
             },

api/routes/v2/health/health.ts

@@ -63,7 +63,14 @@ interface ServiceResponse<T> {
 
 async function checkElastic(fastify: FastifyInstance): Promise<ServiceResponse<ESService>> {
     try {
-        let esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
+        const esStatusCache = await fastify.redis.get(`${fastify.manager.chain}::es_status`);
+        let esStatus;
+        if (esStatusCache) {
+            esStatus = JSON.parse(esStatusCache);
+        } else {
+            esStatus = await fastify.elastic.cat.health({format: 'json', v: true});
+            await fastify.redis.set(`${fastify.manager.chain}::es_status`, JSON.stringify(esStatus), 'EX', 30 * 60);
+        }
         let firstIndexedBlock: number;
         const fib = await fastify.redis.get(`${fastify.manager.chain}::fib`);
         if (fib) {

api/server.ts

@@ -1,4 +1,4 @@
-import {hLog} from "../helpers/common_functions";
+import {hLog, waitUntilReady} from "../helpers/common_functions";
 import {ConfigurationModule} from "../modules/config";
 import {ConnectionManager} from "../connections/manager.class";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
@@ -16,7 +16,7 @@ import {io, Socket} from "socket.io-client";
 import {CacheManager} from "./helpers/cacheManager";
 
 import {bootstrap} from 'global-agent';
-bootstrap();
+import {Api} from "eosjs";
 
 class HyperionApiServer {
 
@@ -29,6 +29,7 @@ class HyperionApiServer {
 
     socketManager: SocketManager;
     mLoader: HyperionModuleLoader;
+    private pluginParams: any;
 
     constructor() {
 
@@ -37,6 +38,11 @@ class HyperionApiServer {
 
         const cm = new ConfigurationModule();
         this.conf = cm.config;
+
+        if (this.conf.settings.use_global_agent) {
+            bootstrap();
+        }
+
         this.chain = this.conf.settings.chain;
         process.title = `hyp-${this.chain}-api`;
         this.manager = new ConnectionManager(cm);
@@ -111,6 +117,7 @@ class HyperionApiServer {
             },
             fastify_redis: this.manager.conn.redis,
             fastify_eosjs: this.manager,
+            chain_id: '',
         } as any;
 
         if (!this.conf.api.disable_rate_limit) {
@@ -140,26 +147,25 @@ class HyperionApiServer {
             pluginParams.fastify_swagger = docsConfig;
         }
 
-        registerPlugins(this.fastify, pluginParams);
-
-        this.addGenericTypeParsing();
+        this.pluginParams = pluginParams;
     }
 
     activateStreaming() {
-        console.log('Importing stream module');
+        console.log('Importing stream module...');
         import('./socketManager').then((mod) => {
             const connOpts = this.manager.conn.chains[this.chain];
-
             let _port = 57200;
             if (connOpts.WS_ROUTER_PORT) {
                 _port = connOpts.WS_ROUTER_PORT;
             }
-
             let _host = "127.0.0.1";
             if (connOpts.WS_ROUTER_HOST) {
                 _host = connOpts.WS_ROUTER_HOST;
             }
-
+            if (_host === "0.0.0.0") {
+                hLog(`[ERROR] WS Router Host is set to 0.0.0.0, please use a fixed IP address instead. Can't start streaming.`);
+                return;
+            }
             this.socketManager = new mod.SocketManager(
                 this.fastify,
                 `http://${_host}:${_port}`,
@@ -187,6 +193,47 @@ class HyperionApiServer {
 
     async init() {
 
+        const rpc = this.manager.nodeosJsonRPC;
+        await waitUntilReady(async () => {
+            try {
+                const chain_data = await rpc.get_info();
+                if (chain_data && chain_data.chain_id) {
+                    this.pluginParams.chain_id = chain_data.chain_id;
+                    return true;
+                } else {
+                    return false;
+                }
+            } catch (e) {
+                hLog(e.message);
+                return false;
+            }
+        }, 20, 5000, () => {
+            hLog('Failed to validate chain api!');
+            process.exit(1);
+        });
+        hLog('Chain API validated!');
+
+        // Wait for Elasticsearch availability
+        await waitUntilReady(async () => {
+            try {
+                const esInfo = await this.manager.elasticsearchClient.info();
+                hLog(`Elasticsearch: ${esInfo.body.version.number} | Lucene: ${esInfo.body.version.lucene_version}`);
+                return true;
+            } catch (e) {
+                console.log(e.message);
+                return false;
+            }
+        }, 10, 5000, () => {
+            hLog('Failed to check elasticsearch version!');
+            process.exit();
+        });
+        hLog('Elasticsearch validated!');
+
+        hLog('Registering plugins...');
+
+        registerPlugins(this.fastify, this.pluginParams);
+        this.addGenericTypeParsing();
+
         await this.mLoader.init();
 
         // add custom plugin routes

api/socketManager.ts

@@ -1,4 +1,4 @@
-import {hLog} from '../helpers/common_functions';
+import {hLog, sleep} from '../helpers/common_functions';
 import {Server, Socket} from 'socket.io';
 import {createAdapter} from 'socket.io-redis';
 import {io} from 'socket.io-client';
@@ -6,6 +6,7 @@ import {FastifyInstance} from "fastify";
 import IORedis from "ioredis";
 import {App, TemplatedApp} from 'uWebSockets.js';
 import {streamPastActions, streamPastDeltas} from "./helpers/functions";
+import {randomUUID} from "crypto";
 
 export interface StreamDeltasRequest {
     code: string;
@@ -40,17 +41,14 @@ export class SocketManager {
     private readonly url;
     private readonly server: FastifyInstance;
     private readonly uwsApp: TemplatedApp;
+    private readonly chainId: string;
+    private currentBlockNum: number;
 
     constructor(fastify: FastifyInstance, url, redisOptions) {
         this.server = fastify;
         this.url = url;
         this.uwsApp = App({});
 
-        // this.io = new Server(fastify.server, {
-        // 	allowEIO3: true,
-        // 	transports: ['websocket', 'polling'],
-        // });
-
         // WS Server for public access
         this.io = new Server({
             transports: ['websocket'],
@@ -59,9 +57,11 @@ export class SocketManager {
 
         this.io.attachApp(this.uwsApp);
 
+        this.chainId = this.server.manager.conn.chains[this.server.manager.chain].chain_id
+        hLog(`[SocketManager] chain_id: ${this.chainId}`);
         const pubClient = new IORedis(redisOptions);
         const subClient = pubClient.duplicate();
-        this.io.adapter(createAdapter({pubClient, subClient}));
+        this.io.adapter(createAdapter({pubClient, subClient, key: this.chainId}));
 
         this.io.on('connection', (socket: Socket) => {
 
@@ -84,10 +84,42 @@ export class SocketManager {
             socket.on('delta_stream_request', async (data: StreamDeltasRequest, callback) => {
                 if (typeof callback === 'function' && data) {
                     try {
+                        // generate random uuid
+                        socket.data.reqUUID = randomUUID();
+                        const lastHistoryBlock = await new Promise<number>((resolve) => {
+                            // start sending realtime data
+                            this.emitToRelay(data, 'delta_request', socket, (emissionResult) => {
+                                callback(emissionResult);
+                                resolve(emissionResult.currentBlockNum);
+                            });
+                        });
+                        // push history data
                         if (data.start_from) {
-                            await streamPastDeltas(this.server, socket, data);
+                            data.read_until = lastHistoryBlock;
+                            console.log('Performing primary scroll request...');
+                            let ltb = 0;
+                            const hStreamResult = await streamPastDeltas(this.server, socket, data);
+                            if (hStreamResult.status === false) {
+                                return;
+                            } else {
+                                ltb = hStreamResult.lastTransmittedBlock;
+                                let attempts = 0;
+                                await sleep(500);
+                                while (ltb > 0 && lastHistoryBlock > ltb && attempts < 3) {
+                                    attempts++;
+                                    console.log(`Performing fill request from ${ltb}...`);
+                                    data.start_from = hStreamResult.lastTransmittedBlock + 1;
+                                    data.read_until = lastHistoryBlock;
+                                    const r = await streamPastDeltas(this.server, socket, data);
+                                    if (r.status === false) {
+                                        console.log(r);
+                                        return;
+                                    } else {
+                                        ltb = r.lastTransmittedBlock;
+                                    }
+                                }
+                            }
                         }
-                        this.emitToRelay(data, 'delta_request', socket, callback);
                     } catch (e) {
                         console.log(e);
                     }
@@ -97,10 +129,42 @@ export class SocketManager {
             socket.on('action_stream_request', async (data: StreamActionsRequest, callback) => {
                 if (typeof callback === 'function' && data) {
                     try {
+                        // generate random uuid
+                        socket.data.reqUUID = randomUUID();
+                        const lastHistoryBlock = await new Promise<number>((resolve) => {
+                            // start sending realtime data
+                            this.emitToRelay(data, 'action_request', socket, (emissionResult) => {
+                                callback(emissionResult);
+                                resolve(emissionResult.currentBlockNum);
+                            });
+                        });
+                        // push history data
                         if (data.start_from) {
-                            await streamPastActions(this.server, socket, data);
+                            data.read_until = lastHistoryBlock;
+                            console.log('Performing primary scroll request...');
+                            let ltb = 0;
+                            const hStreamResult = await streamPastActions(this.server, socket, data);
+                            if (hStreamResult.status === false) {
+                                return;
+                            } else {
+                                ltb = hStreamResult.lastTransmittedBlock;
+                                let attempts = 0;
+                                await sleep(500);
+                                while (ltb > 0 && lastHistoryBlock > ltb && attempts < 3) {
+                                    attempts++;
+                                    console.log(`Performing fill request from ${hStreamResult.lastTransmittedBlock}...`);
+                                    data.start_from = hStreamResult.lastTransmittedBlock + 1;
+                                    data.read_until = lastHistoryBlock;
+                                    const r = await streamPastActions(this.server, socket, data);
+                                    if (r.status === false) {
+                                        console.log(r);
+                                        return;
+                                    } else {
+                                        ltb = r.lastTransmittedBlock;
+                                    }
+                                }
+                            }
                         }
-                        this.emitToRelay(data, 'action_request', socket, callback);
                     } catch (e) {
                         console.log(e);
                     }
@@ -160,18 +224,40 @@ export class SocketManager {
             this.emitToClient(traceData, 'action_trace');
         });
 
-        // Relay LIB info to clients;
-        this.relay.on('lib_update', (data) => {
-            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-                this.io.emit('lib_update', data);
+        this.relay.on('block', (blockData) => {
+            try {
+                // const decodedBlock = JSON.parse(blockData.content.toString());
+                // console.log(blockData.serverTime, blockData.blockNum, decodedBlock);
+                this.currentBlockNum = blockData.blockNum;
+            } catch (e) {
+                hLog(`Failed to decode incoming live block ${blockData.blockNum}: ${e.message}`);
             }
         });
 
-        // Relay LIB info to clients;
-        this.relay.on('fork_event', (data) => {
-            hLog(data);
-            if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
-                this.io.emit('fork_event', data);
+        this.addRelayForwarding('lib_update');
+
+        this.addRelayForwarding('fork_event');
+
+        // // Relay LIB info to clients;
+        // this.relay.on('lib_update', (data) => {
+        //     if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+        //         this.io.emit('lib_update', data);
+        //     }
+        // });
+        //
+        // // Relay fork info to clients;
+        // this.relay.on('fork_event', (data) => {
+        //     if (this.server.manager.conn.chains[this.server.manager.chain].chain_id === data.chain_id) {
+        //         this.io.emit('fork_event', data);
+        //     }
+        // });
+    }
+
+    // Relay events to clients
+    addRelayForwarding(event: string) {
+        this.relay.on(event, (data: any) => {
+            if (data.chain_id && this.chainId === data.chain_id) {
+                this.io.emit(event, data);
             }
         });
     }
@@ -180,6 +266,7 @@ export class SocketManager {
         if (this.io.sockets.sockets.has(traceData.client)) {
             this.io.sockets.sockets.get(traceData.client).emit('message', {
                 type: type,
+                reqUUID: traceData.req,
                 mode: 'live',
                 message: traceData.message,
             });
@@ -189,10 +276,13 @@ export class SocketManager {
     emitToRelay(data, type, socket, callback) {
         if (this.relay.connected) {
             this.relay.emit('event', {
+                reqUUID: socket.data.reqUUID,
                 type: type,
                 client_socket: socket.id,
                 request: data,
             }, (response) => {
+                response['reqUUID'] = socket.data.reqUUID;
+                response['currentBlockNum'] = this.currentBlockNum;
                 callback(response);
             });
         } else {

chains/example.config.json

@@ -7,6 +7,8 @@
     "server_addr": "127.0.0.1",
     "server_port": 7000,
     "stream_port": 1234,
+    "stream_scroll_limit": -1,
+    "stream_scroll_batch": 500,
     "server_name": "127.0.0.1:7000",
     "provider_name": "Example Provider",
     "provider_url": "https://example.com",
@@ -77,7 +79,7 @@
     "auto_mode_switch": false,
     "hot_warm_policy": false,
     "custom_policy": "",
-    "bypass_index_map": false,
+    "use_global_agent": false,
     "index_partition_size": 10000000,
     "es_replicas": 0
   },

definitions/index-templates.ts

@@ -16,29 +16,29 @@ const chain = cm.config.settings.chain;
 // update number of replicas if set to larger than 0
 let replicas = 0;
 if (cm.config.settings.es_replicas) {
-	replicas = cm.config.settings.es_replicas;
+    replicas = cm.config.settings.es_replicas;
 }
 
 const defaultIndexSettings = {
-	"index": {
-		"number_of_shards": shards,
-		"refresh_interval": refresh,
-		"number_of_replicas": replicas,
-		"codec": compression
-	}
+    "index": {
+        "number_of_shards": shards,
+        "refresh_interval": refresh,
+        "number_of_replicas": replicas,
+        "codec": compression
+    }
 };
 
 const actionSettings = {
-	index: {
-		codec: compression,
-		refresh_interval: refresh,
-		number_of_shards: shards * 2,
-		number_of_replicas: replicas,
-		sort: {
-			field: "global_sequence",
-			order: "desc"
-		}
-	}
+    index: {
+        codec: compression,
+        refresh_interval: refresh,
+        number_of_shards: shards * 2,
+        number_of_replicas: replicas,
+        sort: {
+            field: "global_sequence",
+            order: "desc"
+        }
+    }
 };
 
 // actionSettings.index["lifecycle"] = {
@@ -47,158 +47,158 @@ const actionSettings = {
 // };
 
 if (cm.config.settings.hot_warm_policy) {
-	actionSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
+    actionSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
 }
 
 const transferProps = {
-	"properties": {
-		"from": {"type": "keyword"},
-		"to": {"type": "keyword"},
-		"amount": {"type": "float"},
-		"symbol": {"type": "keyword"},
-		"memo": {"type": "text"}
-	}
+    "properties": {
+        "from": {"type": "keyword"},
+        "to": {"type": "keyword"},
+        "amount": {"type": "float"},
+        "symbol": {"type": "keyword"},
+        "memo": {"type": "text"}
+    }
 };
 
 export const action = {
-	order: 0,
-	index_patterns: [
-		chain + "-action-*"
-	],
-	settings: actionSettings,
-	mappings: {
-		properties: {
-			"@timestamp": {"type": "date"},
-			"ds_error": {"type": "boolean"},
-			"global_sequence": {"type": "long"},
-			"account_ram_deltas.delta": {"type": "integer"},
-			"account_ram_deltas.account": {"type": "keyword"},
-			"act.authorization.permission": {"enabled": false},
-			"act.authorization.actor": {"type": "keyword"},
-			"act.account": {"type": "keyword"},
-			"act.name": {"type": "keyword"},
-			"act.data": {"enabled": false},
-			"block_num": {"type": "long"},
-			"block_id": {"type": "keyword"},
-			"action_ordinal": {"type": "long"},
-			"creator_action_ordinal": {"type": "long"},
-			"cpu_usage_us": {"type": "integer"},
-			"net_usage_words": {"type": "integer"},
-			"code_sequence": {"type": "integer"},
-			"abi_sequence": {"type": "integer"},
-			"trx_id": {"type": "keyword"},
-			"producer": {"type": "keyword"},
-			"signatures": {"enabled": false},
-			"inline_count": {"type": "short"},
-			"max_inline": {"type": "short"},
-			"inline_filtered": {"type": "boolean"},
-			"receipts": {
-				"properties": {
-					"global_sequence": {"type": "long"},
-					"recv_sequence": {"type": "long"},
-					"receiver": {"type": "keyword"},
-					"auth_sequence": {
-						"properties": {
-							"account": {"type": "keyword"},
-							"sequence": {"type": "long"}
-						}
-					}
-				}
-			},
+    order: 0,
+    index_patterns: [
+        chain + "-action-*"
+    ],
+    settings: actionSettings,
+    mappings: {
+        properties: {
+            "@timestamp": {"type": "date"},
+            "ds_error": {"type": "boolean"},
+            "global_sequence": {"type": "long"},
+            "account_ram_deltas.delta": {"type": "integer"},
+            "account_ram_deltas.account": {"type": "keyword"},
+            "act.authorization.permission": {"enabled": false},
+            "act.authorization.actor": {"type": "keyword"},
+            "act.account": {"type": "keyword"},
+            "act.name": {"type": "keyword"},
+            "act.data": {"enabled": false},
+            "block_num": {"type": "long"},
+            "block_id": {"type": "keyword"},
+            "action_ordinal": {"type": "long"},
+            "creator_action_ordinal": {"type": "long"},
+            "cpu_usage_us": {"type": "integer"},
+            "net_usage_words": {"type": "integer"},
+            "code_sequence": {"type": "integer"},
+            "abi_sequence": {"type": "integer"},
+            "trx_id": {"type": "keyword"},
+            "producer": {"type": "keyword"},
+            "signatures": {"enabled": false},
+            "inline_count": {"type": "short"},
+            "max_inline": {"type": "short"},
+            "inline_filtered": {"type": "boolean"},
+            "receipts": {
+                "properties": {
+                    "global_sequence": {"type": "long"},
+                    "recv_sequence": {"type": "long"},
+                    "receiver": {"type": "keyword"},
+                    "auth_sequence": {
+                        "properties": {
+                            "account": {"type": "keyword"},
+                            "sequence": {"type": "long"}
+                        }
+                    }
+                }
+            },
 
-			// eosio::newaccount
-			"@newaccount": {
-				"properties": {
-					"active": {"type": "object"},
-					"owner": {"type": "object"},
-					"newact": {"type": "keyword"}
-				}
-			},
+            // eosio::newaccount
+            "@newaccount": {
+                "properties": {
+                    "active": {"type": "object"},
+                    "owner": {"type": "object"},
+                    "newact": {"type": "keyword"}
+                }
+            },
 
-			// eosio::updateauth
-			"@updateauth": {
-				"properties": {
-					"permission": {"type": "keyword"},
-					"parent": {"type": "keyword"},
-					"auth": {"type": "object"}
-				}
-			},
+            // eosio::updateauth
+            "@updateauth": {
+                "properties": {
+                    "permission": {"type": "keyword"},
+                    "parent": {"type": "keyword"},
+                    "auth": {"type": "object"}
+                }
+            },
 
-			// *::transfer
-			"@transfer": transferProps,
+            // *::transfer
+            "@transfer": transferProps,
 
-			// eosio::unstaketorex
-			"@unstaketorex": {
-				"properties": {
-					"owner": {"type": "keyword"},
-					"receiver": {"type": "keyword"},
-					"amount": {"type": "float"}
-				}
-			},
+            // eosio::unstaketorex
+            "@unstaketorex": {
+                "properties": {
+                    "owner": {"type": "keyword"},
+                    "receiver": {"type": "keyword"},
+                    "amount": {"type": "float"}
+                }
+            },
 
-			// eosio::buyrex
-			"@buyrex": {
-				"properties": {
-					"from": {"type": "keyword"},
-					"amount": {"type": "float"}
-				}
-			},
+            // eosio::buyrex
+            "@buyrex": {
+                "properties": {
+                    "from": {"type": "keyword"},
+                    "amount": {"type": "float"}
+                }
+            },
 
-			// eosio::buyram
-			"@buyram": {
-				"properties": {
-					"payer": {"type": "keyword"},
-					"receiver": {"type": "keyword"},
-					"quant": {"type": "float"}
-				}
-			},
+            // eosio::buyram
+            "@buyram": {
+                "properties": {
+                    "payer": {"type": "keyword"},
+                    "receiver": {"type": "keyword"},
+                    "quant": {"type": "float"}
+                }
+            },
 
-			// eosio::buyrambytes
-			"@buyrambytes": {
-				"properties": {
-					"payer": {"type": "keyword"},
-					"receiver": {"type": "keyword"},
-					"bytes": {"type": "long"}
-				}
-			},
+            // eosio::buyrambytes
+            "@buyrambytes": {
+                "properties": {
+                    "payer": {"type": "keyword"},
+                    "receiver": {"type": "keyword"},
+                    "bytes": {"type": "long"}
+                }
+            },
 
-			// eosio::delegatebw
-			"@delegatebw": {
-				"properties": {
-					"from": {"type": "keyword"},
-					"receiver": {"type": "keyword"},
-					"stake_cpu_quantity": {"type": "float"},
-					"stake_net_quantity": {"type": "float"},
-					"transfer": {"type": "boolean"},
-					"amount": {"type": "float"}
-				}
-			},
+            // eosio::delegatebw
+            "@delegatebw": {
+                "properties": {
+                    "from": {"type": "keyword"},
+                    "receiver": {"type": "keyword"},
+                    "stake_cpu_quantity": {"type": "float"},
+                    "stake_net_quantity": {"type": "float"},
+                    "transfer": {"type": "boolean"},
+                    "amount": {"type": "float"}
+                }
+            },
 
-			// eosio::undelegatebw
-			"@undelegatebw": {
-				"properties": {
-					"from": {"type": "keyword"},
-					"receiver": {"type": "keyword"},
-					"unstake_cpu_quantity": {"type": "float"},
-					"unstake_net_quantity": {"type": "float"},
-					"amount": {"type": "float"}
-				}
-			}
-		}
-	}
+            // eosio::undelegatebw
+            "@undelegatebw": {
+                "properties": {
+                    "from": {"type": "keyword"},
+                    "receiver": {"type": "keyword"},
+                    "unstake_cpu_quantity": {"type": "float"},
+                    "unstake_net_quantity": {"type": "float"},
+                    "amount": {"type": "float"}
+                }
+            }
+        }
+    }
 };
 
 const deltaSettings = {
-	index: {
-		codec: compression,
-		number_of_shards: shards * 2,
-		refresh_interval: refresh,
-		number_of_replicas: replicas,
-		sort: {
-			field: ["block_num", "scope", "primary_key"],
-			order: ["desc", "asc", "asc"]
-		}
-	}
+    index: {
+        codec: compression,
+        number_of_shards: shards * 2,
+        refresh_interval: refresh,
+        number_of_replicas: replicas,
+        sort: {
+            field: ["block_num", "scope", "primary_key"],
+            order: ["desc", "asc", "asc"]
+        }
+    }
 };
 
 // deltaSettings.index["lifecycle"] = {
@@ -207,319 +207,339 @@ const deltaSettings = {
 // };
 
 if (cm.config.settings.hot_warm_policy) {
-	deltaSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
+    deltaSettings["routing"] = {"allocation": {"exclude": {"data": "warm"}}};
 }
 
 export const delta = {
-	"index_patterns": [chain + "-delta-*"],
-	"settings": deltaSettings,
-	"mappings": {
-		"properties": {
+    "index_patterns": [chain + "-delta-*"],
+    "settings": deltaSettings,
+    "mappings": {
+        "properties": {
 
-			// base fields
-			"@timestamp": {"type": "date"},
-			"present": {"type": "byte"},
-			"ds_error": {"type": "boolean"},
-			"block_id": {"type": "keyword"},
-			"block_num": {"type": "long"},
-			"deleted_at": {"type": "long"},
-			"code": {"type": "keyword"},
-			"scope": {"type": "keyword"},
-			"table": {"type": "keyword"},
-			"payer": {"type": "keyword"},
-			"primary_key": {"type": "keyword"},
-			"data": {"enabled": false},
-			"value": {"enabled": false},
+            // base fields
+            "@timestamp": {"type": "date"},
+            "present": {"type": "byte"},
+            "ds_error": {"type": "boolean"},
+            "block_id": {"type": "keyword"},
+            "block_num": {"type": "long"},
+            "deleted_at": {"type": "long"},
+            "code": {"type": "keyword"},
+            "scope": {"type": "keyword"},
+            "table": {"type": "keyword"},
+            "payer": {"type": "keyword"},
+            "primary_key": {"type": "keyword"},
+            "data": {"enabled": false},
+            "value": {"enabled": false},
 
-			// eosio.msig::approvals
-			"@approvals.proposal_name": {"type": "keyword"},
-			"@approvals.provided_approvals": {"type": "object"},
-			"@approvals.requested_approvals": {"type": "object"},
+            // eosio.msig::approvals
+            "@approvals.proposal_name": {"type": "keyword"},
+            "@approvals.provided_approvals": {"type": "object"},
+            "@approvals.requested_approvals": {"type": "object"},
 
-			// eosio.msig::proposal
-			"@proposal.proposal_name": {"type": "keyword"},
-			"@proposal.transaction": {"enabled": false},
+            // eosio.msig::proposal
+            "@proposal.proposal_name": {"type": "keyword"},
+            "@proposal.transaction": {"enabled": false},
 
-			// *::accounts
-			"@accounts.amount": {"type": "float"},
-			"@accounts.symbol": {"type": "keyword"},
+            // *::accounts
+            "@accounts.amount": {"type": "float"},
+            "@accounts.symbol": {"type": "keyword"},
 
-			// eosio::voters
-			"@voters.is_proxy": {"type": "boolean"},
-			"@voters.producers": {"type": "keyword"},
-			"@voters.last_vote_weight": {"type": "double"},
-			"@voters.proxied_vote_weight": {"type": "double"},
-			"@voters.staked": {"type": "float"},
-			"@voters.proxy": {"type": "keyword"},
+            // eosio::voters
+            "@voters.is_proxy": {"type": "boolean"},
+            "@voters.producers": {"type": "keyword"},
+            "@voters.last_vote_weight": {"type": "double"},
+            "@voters.proxied_vote_weight": {"type": "double"},
+            "@voters.staked": {"type": "float"},
+            "@voters.proxy": {"type": "keyword"},
 
-			// eosio::producers
-			"@producers.total_votes": {"type": "double"},
-			"@producers.is_active": {"type": "boolean"},
-			"@producers.unpaid_blocks": {"type": "long"},
+            // eosio::producers
+            "@producers.total_votes": {"type": "double"},
+            "@producers.is_active": {"type": "boolean"},
+            "@producers.unpaid_blocks": {"type": "long"},
 
-			// eosio::global
-			"@global": {
-				"properties": {
-					"last_name_close": {"type": "date"},
-					"last_pervote_bucket_fill": {"type": "date"},
-					"last_producer_schedule_update": {"type": "date"},
-					"perblock_bucket": {"type": "double"},
-					"pervote_bucket": {"type": "double"},
-					"total_activated_stake": {"type": "double"},
-					"total_voteshare_change_rate": {"type": "double"},
-					"total_unpaid_voteshare": {"type": "double"},
-					"total_producer_vote_weight": {"type": "double"},
-					"total_ram_bytes_reserved": {"type": "long"},
-					"total_ram_stake": {"type": "long"},
-					"total_unpaid_blocks": {"type": "long"},
-				}
-			}
-		}
-	}
+            // eosio::global
+            "@global": {
+                "properties": {
+                    "last_name_close": {"type": "date"},
+                    "last_pervote_bucket_fill": {"type": "date"},
+                    "last_producer_schedule_update": {"type": "date"},
+                    "perblock_bucket": {"type": "double"},
+                    "pervote_bucket": {"type": "double"},
+                    "total_activated_stake": {"type": "double"},
+                    "total_voteshare_change_rate": {"type": "double"},
+                    "total_unpaid_voteshare": {"type": "double"},
+                    "total_producer_vote_weight": {"type": "double"},
+                    "total_ram_bytes_reserved": {"type": "long"},
+                    "total_ram_stake": {"type": "long"},
+                    "total_unpaid_blocks": {"type": "long"},
+                }
+            }
+        }
+    }
 };
 
 export const abi = {
-	"index_patterns": [chain + "-abi-*"],
-	"settings": defaultIndexSettings,
-	"mappings": {
-		"properties": {
-			"@timestamp": {"type": "date"},
-			"block": {"type": "long"},
-			"account": {"type": "keyword"},
-			"abi": {"enabled": false},
-			"abi_hex": {"enabled": false},
-			"actions": {"type": "keyword"},
-			"tables": {"type": "keyword"}
-		}
-	}
+    "index_patterns": [chain + "-abi-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "@timestamp": {"type": "date"},
+            "block": {"type": "long"},
+            "account": {"type": "keyword"},
+            "abi": {"enabled": false},
+            "abi_hex": {"enabled": false},
+            "actions": {"type": "keyword"},
+            "tables": {"type": "keyword"}
+        }
+    }
 };
 
 export const permissionLink = {
-	"index_patterns": [chain + "-link-*"],
-	"settings": defaultIndexSettings,
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"@timestamp": {"type": "date"},
-			"present": {"type": "byte"},
-			"account": {"type": "keyword"},
-			"code": {"type": "keyword"},
-			"action": {"type": "keyword"},
-			"permission": {"type": "keyword"}
-		}
-	}
+    "index_patterns": [chain + "-link-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "present": {"type": "byte"},
+            "account": {"type": "keyword"},
+            "code": {"type": "keyword"},
+            "action": {"type": "keyword"},
+            "permission": {"type": "keyword"}
+        }
+    }
 };
 
 export const permission = {
-	"index_patterns": [chain + "-perm-*"],
-	"settings": defaultIndexSettings,
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"present": {"type": "byte"},
-			"owner": {"type": "keyword"},
-			"name": {"type": "keyword"},
-			"parent": {"type": "keyword"},
-			"last_updated": {"type": "date"},
-			"auth": {"type": "object"}
-		}
-	}
+    "index_patterns": [chain + "-perm-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "present": {"type": "byte"},
+            "owner": {"type": "keyword"},
+            "name": {"type": "keyword"},
+            "parent": {"type": "keyword"},
+            "last_updated": {"type": "date"},
+            "auth": {"type": "object"}
+        }
+    }
 };
 
 export const resourceLimits = {
-	"index_patterns": [chain + "-reslimits-*"],
-	"settings": defaultIndexSettings,
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"@timestamp": {"type": "date"},
-			"owner": {"type": "keyword"},
-			"total_weight": {"type": "long"},
-			"net_weight": {"type": "long"},
-			"cpu_weight": {"type": "long"},
-			"ram_bytes": {"type": "long"}
-		}
-	}
+    "index_patterns": [chain + "-reslimits-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "owner": {"type": "keyword"},
+            "total_weight": {"type": "long"},
+            "net_weight": {"type": "long"},
+            "cpu_weight": {"type": "long"},
+            "ram_bytes": {"type": "long"}
+        }
+    }
 };
 
 export const generatedTransaction = {
-	"index_patterns": [chain + "-gentrx-*"],
-	"settings": defaultIndexSettings,
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"@timestamp": {"type": "date"},
-			"sender": {"type": "keyword"},
-			"sender_id": {"type": "keyword"},
-			"payer": {"type": "keyword"},
-			"trx_id": {"type": "keyword"},
-			"actions": {"enabled": false},
-			"packed_trx": {"enabled": false}
-		}
-	}
+    "index_patterns": [chain + "-gentrx-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "sender": {"type": "keyword"},
+            "sender_id": {"type": "keyword"},
+            "payer": {"type": "keyword"},
+            "trx_id": {"type": "keyword"},
+            "actions": {"enabled": false},
+            "packed_trx": {"enabled": false}
+        }
+    }
 };
 
 export const failedTransaction = {
-	"index_patterns": [chain + "-trxerr-*"],
-	"settings": defaultIndexSettings,
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"@timestamp": {"type": "date"},
-			"status": {"type": "short"}
-		}
-	}
+    "index_patterns": [chain + "-trxerr-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "status": {"type": "short"}
+        }
+    }
+};
+
+export const schedule = {
+    "index_patterns": [chain + "-schedule-*"],
+    "settings": {
+        ...defaultIndexSettings,
+        sort: {
+            field: "version",
+            order: "desc"
+        }
+    },
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "version": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "producers.name": {"type": "keyword"},
+            "producers.keys": {"type": "keyword"},
+        }
+    }
 };
 
 export const resourceUsage = {
-	"index_patterns": [chain + "-userres-*"],
-	"settings": defaultIndexSettings,
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"@timestamp": {"type": "date"},
-			"owner": {"type": "keyword"},
-			"net_used": {"type": "long"},
-			"net_total": {"type": "long"},
-			"net_pct": {"type": "float"},
-			"cpu_used": {"type": "long"},
-			"cpu_total": {"type": "long"},
-			"cpu_pct": {"type": "float"},
-			"ram": {"type": "long"}
-		}
-	}
+    "index_patterns": [chain + "-userres-*"],
+    "settings": defaultIndexSettings,
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "@timestamp": {"type": "date"},
+            "owner": {"type": "keyword"},
+            "net_used": {"type": "long"},
+            "net_total": {"type": "long"},
+            "net_pct": {"type": "float"},
+            "cpu_used": {"type": "long"},
+            "cpu_total": {"type": "long"},
+            "cpu_pct": {"type": "float"},
+            "ram": {"type": "long"}
+        }
+    }
 };
 
 
 export const logs = {
-	"index_patterns": [chain + "-logs-*"],
-	"settings": defaultIndexSettings
+    "index_patterns": [chain + "-logs-*"],
+    "settings": defaultIndexSettings
 };
 
 export const block = {
-	"index_patterns": [chain + "-block-*"],
-	"settings": {
-		"index": {
-			"codec": compression,
-			"number_of_shards": shards,
-			"refresh_interval": refresh,
-			"number_of_replicas": replicas,
-			"sort.field": "block_num",
-			"sort.order": "desc"
-		}
-	},
-	"mappings": {
-		"properties": {
-			"@timestamp": {"type": "date"},
-			"block_num": {"type": "long"},
-			"block_id": {"type": "keyword"},
-			"prev_id": {"type": "keyword"},
-			"producer": {"type": "keyword"},
-			"new_producers.producers.block_signing_key": {"enabled": false},
-			"new_producers.producers.producer_name": {"type": "keyword"},
-			"new_producers.version": {"type": "long"},
-			"schedule_version": {"type": "double"},
-			"cpu_usage": {"type": "integer"},
-			"net_usage": {"type": "integer"}
-		}
-	}
+    "index_patterns": [chain + "-block-*"],
+    "settings": {
+        "index": {
+            "codec": compression,
+            "number_of_shards": shards,
+            "refresh_interval": refresh,
+            "number_of_replicas": replicas,
+            "sort.field": "block_num",
+            "sort.order": "desc"
+        }
+    },
+    "mappings": {
+        "properties": {
+            "@timestamp": {"type": "date"},
+            "block_num": {"type": "long"},
+            "block_id": {"type": "keyword"},
+            "prev_id": {"type": "keyword"},
+            "producer": {"type": "keyword"},
+            "new_producers.producers.block_signing_key": {"enabled": false},
+            "new_producers.producers.producer_name": {"type": "keyword"},
+            "new_producers.version": {"type": "long"},
+            "schedule_version": {"type": "double"},
+            "cpu_usage": {"type": "integer"},
+            "net_usage": {"type": "integer"}
+        }
+    }
 };
 
 export const tableProposals = {
-	"index_patterns": [chain + "-table-proposals-*"],
-	"settings": {
-		"index": {
-			"codec": compression,
-			"number_of_shards": shards,
-			"refresh_interval": refresh,
-			"number_of_replicas": replicas,
-			"sort.field": "block_num",
-			"sort.order": "desc"
-		}
-	},
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"present": {"type": "byte"},
-			"proposal_name": {"type": "keyword"},
-			"requested_approvals": {"type": "object"},
-			"provided_approvals": {"type": "object"},
-			"executed": {"type": "boolean"}
-		}
-	}
+    "index_patterns": [chain + "-table-proposals-*"],
+    "settings": {
+        "index": {
+            "codec": compression,
+            "number_of_shards": shards,
+            "refresh_interval": refresh,
+            "number_of_replicas": replicas,
+            "sort.field": "block_num",
+            "sort.order": "desc"
+        }
+    },
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "present": {"type": "byte"},
+            "proposal_name": {"type": "keyword"},
+            "requested_approvals": {"type": "object"},
+            "provided_approvals": {"type": "object"},
+            "executed": {"type": "boolean"}
+        }
+    }
 };
 
 export const tableAccounts = {
-	"index_patterns": [chain + "-table-accounts-*"],
-	"settings": {
-		"index": {
-			"codec": compression,
-			"number_of_shards": shards,
-			"refresh_interval": refresh,
-			"number_of_replicas": replicas,
-			"sort.field": "amount",
-			"sort.order": "desc"
-		}
-	},
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"present": {"type": "byte"},
-			"code": {"type": "keyword"},
-			"scope": {"type": "keyword"},
-			"amount": {"type": "float"},
-			"symbol": {"type": "keyword"}
-		}
-	}
+    "index_patterns": [chain + "-table-accounts-*"],
+    "settings": {
+        "index": {
+            "codec": compression,
+            "number_of_shards": shards,
+            "refresh_interval": refresh,
+            "number_of_replicas": replicas,
+            "sort.field": "amount",
+            "sort.order": "desc"
+        }
+    },
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "present": {"type": "byte"},
+            "code": {"type": "keyword"},
+            "scope": {"type": "keyword"},
+            "amount": {"type": "float"},
+            "symbol": {"type": "keyword"}
+        }
+    }
 };
 
 // noinspection JSUnusedGlobalSymbols
 export const tableDelBand = {
-	"index_patterns": [chain + "-table-delband-*"],
-	"settings": {
-		"index": {
-			"codec": compression,
-			"number_of_shards": shards,
-			"refresh_interval": refresh,
-			"number_of_replicas": replicas,
-			"sort.field": "total_weight",
-			"sort.order": "desc"
-		}
-	},
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"from": {"type": "keyword"},
-			"to": {"type": "keyword"},
-			"total_weight": {"type": "float"},
-			"net_weight": {"type": "float"},
-			"cpu_weight": {"type": "float"}
-		}
-	}
+    "index_patterns": [chain + "-table-delband-*"],
+    "settings": {
+        "index": {
+            "codec": compression,
+            "number_of_shards": shards,
+            "refresh_interval": refresh,
+            "number_of_replicas": replicas,
+            "sort.field": "total_weight",
+            "sort.order": "desc"
+        }
+    },
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "from": {"type": "keyword"},
+            "to": {"type": "keyword"},
+            "total_weight": {"type": "float"},
+            "net_weight": {"type": "float"},
+            "cpu_weight": {"type": "float"}
+        }
+    }
 };
 
 export const tableVoters = {
-	"index_patterns": [chain + "-table-voters-*"],
-	"settings": {
-		"index": {
-			"codec": compression,
-			"number_of_shards": shards,
-			"refresh_interval": refresh,
-			"number_of_replicas": replicas,
-			"sort.field": "last_vote_weight",
-			"sort.order": "desc"
-		}
-	},
-	"mappings": {
-		"properties": {
-			"block_num": {"type": "long"},
-			"voter": {"type": "keyword"},
-			"producers": {"type": "keyword"},
-			"last_vote_weight": {"type": "double"},
-			"is_proxy": {"type": "boolean"},
-			"proxied_vote_weight": {"type": "double"},
-			"staked": {"type": "double"},
-			"proxy": {"type": "keyword"}
-		}
-	}
+    "index_patterns": [chain + "-table-voters-*"],
+    "settings": {
+        "index": {
+            "codec": compression,
+            "number_of_shards": shards,
+            "refresh_interval": refresh,
+            "number_of_replicas": replicas,
+            "sort.field": "last_vote_weight",
+            "sort.order": "desc"
+        }
+    },
+    "mappings": {
+        "properties": {
+            "block_num": {"type": "long"},
+            "voter": {"type": "keyword"},
+            "producers": {"type": "keyword"},
+            "last_vote_weight": {"type": "double"},
+            "is_proxy": {"type": "boolean"},
+            "proxied_vote_weight": {"type": "double"},
+            "staked": {"type": "double"},
+            "proxy": {"type": "keyword"}
+        }
+    }
 };

ecosystem.config.js

@@ -5,7 +5,7 @@ const path = require('path');
 const apps = [];
 const chainsRoot = path.join(path.resolve(), 'chains');
 readdirSync(chainsRoot)
-    .filter(f => f.endsWith('.config.json'))
+    .filter(f => f.endsWith('.config.json') && !f.startsWith('example'))
     .forEach(value => {
         const configFile = readFileSync(path.join(chainsRoot, value))
         const config = JSON.parse(configFile.toString());

example-connections.json

@@ -27,8 +27,9 @@
       "chain_id": "aca376f206b8fc25a6ed44dbdc66547c36c6c33e3a119ffbeaef943642f0e906",
       "http": "http://127.0.0.1:8888",
       "ship": "ws://127.0.0.1:8080",
+      "WS_ROUTER_HOST": "127.0.0.1",
       "WS_ROUTER_PORT": 7001,
-      "WS_ROUTER_HOST": "127.0.0.1"
+      "control_port": 7002
     }
   }
 }

helpers/common_functions.ts

@@ -6,245 +6,288 @@ import {join} from "path";
 let config;
 const conf_path = join(__dirname, `../${process.env.CONFIG_JSON}`);
 if (existsSync(conf_path)) {
-	try {
-		config = require(conf_path);
-	} catch (e) {
-		console.log(e.message);
-		process.exit(1);
-	}
+    try {
+        config = require(conf_path);
+    } catch (e) {
+        console.log(e.message);
+        process.exit(1);
+    }
 }
 
 if (!config) {
-	console.log(`Configuration not found: ${conf_path}`);
-	process.exit(1);
+    console.log(`Configuration not found: ${conf_path}`);
+    process.exit(1);
 }
 
 function getLastResult(results: ApiResponse) {
-	if (results.body.hits?.hits?.length > 0) {
-		return parseInt(results.body.hits.hits[0].sort[0], 10);
-	} else {
-		return 1;
-	}
+    if (results.body.hits?.hits?.length > 0) {
+        return parseInt(results.body.hits.hits[0].sort[0], 10);
+    } else {
+        return 1;
+    }
 }
 
 export async function getLastIndexedBlockByDelta(es_client: Client, chain: string) {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-delta-*',
-		size: 1,
-		body: {
-			query: {bool: {filter: {match_all: {}}}},
-			sort: [{block_num: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-delta-*',
+        size: 1,
+        body: {
+            query: {bool: {filter: {match_all: {}}}},
+            sort: [{block_num: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export async function getLastIndexedBlock(es_client: Client, chain: string) {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-block-*',
-		size: 1,
-		body: {
-			query: {bool: {filter: {match_all: {}}}},
-			sort: [{block_num: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-block-*',
+        size: 1,
+        body: {
+            query: {bool: {filter: {match_all: {}}}},
+            sort: [{block_num: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export async function getLastIndexedBlockWithTotalBlocks(es_client: Client, chain: string): Promise<[number, number]> {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-block-*',
-		size: 1,
-		body: {
-			query: {bool: {filter: {match_all: {}}}},
-			sort: [{block_num: {order: "desc"}}],
-			track_total_hits: true
-		}
-	});
-	let lastBlock = getLastResult(results);
-	let totalBlocks = results.body.hits.total.value || 1;
-	return [lastBlock, totalBlocks];
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-block-*',
+        size: 1,
+        body: {
+            query: {bool: {filter: {match_all: {}}}},
+            sort: [{block_num: {order: "desc"}}],
+            track_total_hits: true
+        }
+    });
+    let lastBlock = getLastResult(results);
+    let totalBlocks = results.body.hits.total.value || 1;
+    return [lastBlock, totalBlocks];
 }
 
 export async function getFirstIndexedBlock(es_client: Client, chain: string): Promise<number> {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-block-*',
-		size: 1,
-		body: {
-			query: {bool: {filter: {match_all: {}}}},
-			sort: [{block_num: {order: "asc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-block-*',
+        size: 1,
+        body: {
+            query: {bool: {filter: {match_all: {}}}},
+            sort: [{block_num: {order: "asc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 
 export async function getLastIndexedABI(es_client: Client, chain: string) {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-abi-*',
-		size: 1,
-		body: {
-			query: {
-				match_all: {}
-			},
-			sort: [{block: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-abi-*',
+        size: 1,
+        body: {
+            query: {
+                match_all: {}
+            },
+            sort: [{block: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export async function getLastIndexedBlockByDeltaFromRange(es_client: Client, chain: string, first: number, last: number) {
-	const results: ApiResponse = await es_client.search({
-		index: chain + '-delta-*',
-		size: 1,
-		body: {
-			query: {
-				range: {
-					block_num: {
-						"gte": first,
-						"lt": last,
-						"boost": 2
-					}
-				}
-			},
-			sort: [{block_num: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results: ApiResponse = await es_client.search({
+        index: chain + '-delta-*',
+        size: 1,
+        body: {
+            query: {
+                range: {
+                    block_num: {
+                        "gte": first,
+                        "lt": last,
+                        "boost": 2
+                    }
+                }
+            },
+            sort: [{block_num: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export async function getLastIndexedBlockFromRange(es_client: Client, chain: string, first: number, last: number) {
-	const results = await es_client.search({
-		index: chain + '-block-*',
-		size: 1,
-		body: {
-			query: {
-				range: {
-					block_num: {
-						"gte": first,
-						"lt": last,
-						"boost": 2
-					}
-				}
-			},
-			sort: [{block_num: {order: "desc"}}]
-		}
-	});
-	return getLastResult(results);
+    const results = await es_client.search({
+        index: chain + '-block-*',
+        size: 1,
+        body: {
+            query: {
+                range: {
+                    block_num: {
+                        "gte": first,
+                        "lt": last,
+                        "boost": 2
+                    }
+                }
+            },
+            sort: [{block_num: {order: "desc"}}]
+        }
+    });
+    return getLastResult(results);
 }
 
 export function messageAllWorkers(cl, payload) {
-	for (const c in cl.workers) {
-		if (cl.workers.hasOwnProperty(c)) {
-			const _w = cl.workers[c];
-			if (_w) {
-				try {
-					if (_w.isConnected()) {
-						_w.send(payload);
-					} else {
-						hLog('Worker is not connected!');
-					}
-				} catch (e) {
-					hLog('Failed to message worker!');
-					hLog(e);
-				}
-			} else {
-				hLog('Worker not found!');
-			}
-		}
-	}
+    for (const c in cl.workers) {
+        if (cl.workers.hasOwnProperty(c)) {
+            const _w = cl.workers[c];
+            if (_w) {
+                try {
+                    if (_w.isConnected()) {
+                        _w.send(payload);
+                    } else {
+                        hLog('Worker is not connected!');
+                    }
+                } catch (e) {
+                    hLog('Failed to message worker!');
+                    hLog(e);
+                }
+            } else {
+                hLog('Worker not found!');
+            }
+        }
+    }
 }
 
 export function serialize(type, value, txtEnc, txtDec, types) {
-	const buffer = new Serialize.SerialBuffer({
-		textEncoder: txtEnc,
-		textDecoder: txtDec
-	});
-	Serialize.getType(types, type).serialize(buffer, value);
-	return buffer.asUint8Array();
+    const buffer = new Serialize.SerialBuffer({
+        textEncoder: txtEnc,
+        textDecoder: txtDec
+    });
+    Serialize.getType(types, type).serialize(buffer, value);
+    return buffer.asUint8Array();
 }
 
 export function deserialize(type, array, txtEnc, txtDec, types) {
-	const buffer = new Serialize.SerialBuffer({
-		textEncoder: txtEnc,
-		textDecoder: txtDec,
-		array
-	});
-	return Serialize.getType(types, type).deserialize(buffer, new Serialize.SerializerState({bytesAsUint8Array: true}));
+    const buffer = new Serialize.SerialBuffer({
+        textEncoder: txtEnc,
+        textDecoder: txtDec,
+        array
+    });
+    return Serialize.getType(types, type).deserialize(buffer, new Serialize.SerializerState({bytesAsUint8Array: true}));
 }
 
 function getNested(path_array, jsonObj) {
-	const nextPath = path_array.shift();
-	const nextValue = jsonObj[nextPath];
-	if (!nextValue) {
-		return null;
-	} else {
-		if (typeof nextValue !== 'object') {
-			return nextValue;
-		} else {
-			if (Array.isArray(nextValue)) {
-				return nextValue;
-			} else {
-				return getNested(path_array, nextValue);
-			}
-		}
-	}
+    const nextPath = path_array.shift();
+    const nextValue = jsonObj[nextPath];
+    if (!nextValue) {
+        return null;
+    } else {
+        if (typeof nextValue !== 'object') {
+            return nextValue;
+        } else {
+            if (Array.isArray(nextValue)) {
+                return nextValue;
+            } else {
+                return getNested(path_array, nextValue);
+            }
+        }
+    }
+}
+
+export function checkDeltaFilter(filter, _source) {
+    if (filter.field && filter.value) {
+        let fieldValue = getNested(filter.field.split("."), _source);
+        if (!fieldValue) {
+            const fArray = filter.field.split(".");
+            if (fArray[0].startsWith('@')) {
+                const tableName = fArray[0].replace('@', '');
+                if (_source.table === tableName) {
+                    fArray[0] = 'data';
+                    fieldValue = getNested(fArray, _source);
+                }
+            }
+        }
+        if (fieldValue) {
+            if (Array.isArray(fieldValue)) {
+                return fieldValue.indexOf(filter.value) !== -1;
+            } else {
+                return fieldValue === filter.value;
+            }
+        } else {
+            return !filter.value;
+        }
+    } else {
+        return false;
+    }
 }
 
 export function checkFilter(filter, _source) {
-	if (filter.field && filter.value) {
-		let fieldValue = getNested(filter.field.split("."), _source);
-		if (!fieldValue) {
-			const fArray = filter.field.split(".");
-			if (fArray[0].startsWith('@')) {
-				const actName = fArray[0].replace('@', '');
-				if (_source.act.name === actName) {
-					fArray[0] = 'data';
-					fArray.unshift('act');
-					fieldValue = getNested(fArray, _source);
-				}
-			}
-		}
-		if (fieldValue) {
-			if (Array.isArray(fieldValue)) {
-				return fieldValue.indexOf(filter.value) !== -1;
-			} else {
-				return fieldValue === filter.value;
-			}
-		} else {
-			return !filter.value;
-		}
-	} else {
-		return false;
-	}
+    if (filter.field && filter.value) {
+        let fieldValue = getNested(filter.field.split("."), _source);
+        if (!fieldValue) {
+            const fArray = filter.field.split(".");
+            if (fArray[0].startsWith('@')) {
+                const actName = fArray[0].replace('@', '');
+                if (_source.act.name === actName) {
+                    fArray[0] = 'data';
+                    fArray.unshift('act');
+                    fieldValue = getNested(fArray, _source);
+                }
+            }
+        }
+        if (fieldValue) {
+            if (Array.isArray(fieldValue)) {
+                return fieldValue.indexOf(filter.value) !== -1;
+            } else {
+                return fieldValue === filter.value;
+            }
+        } else {
+            return !filter.value;
+        }
+    } else {
+        return false;
+    }
 }
 
 export function hLog(input: any, ...extra: any[]) {
-	let role;
-	if (process.env.worker_role) {
-		const id = parseInt(process.env.worker_id);
-		role = `[${process.pid} - ${(id < 10 ? '0' : '') + id.toString()}_${process.env.worker_role}]`;
-	} else {
-		if (process.env.script && process.env.script === './api/server.js') {
-			role = `[${process.pid} - api]`;
-		} else {
-			role = `[${process.pid} - 00_master]`;
-		}
-	}
-	if (process.env.TRACE_LOGS === 'true') {
-		const e = new Error();
-		const frame = e.stack.split("\n")[2];
-		const where = frame.split(" ")[6].split(/[:()]/);
-		const arr = where[1].split("/");
-		const fileName = arr[arr.length - 1];
-		const lineNumber = where[2];
-		role += ` ${fileName}:${lineNumber}`;
-	}
-	console.log(role, input, ...extra);
+    let role;
+    if (process.env.worker_role) {
+        const id = parseInt(process.env.worker_id);
+        role = `[${process.pid} - ${(id < 10 ? '0' : '') + id.toString()}_${process.env.worker_role}]`;
+    } else {
+        if (process.env.script && process.env.script === './api/server.js') {
+            role = `[${process.pid} - api]`;
+        } else {
+            role = `[${process.pid} - 00_master]`;
+        }
+    }
+    if (process.env.TRACE_LOGS === 'true') {
+        const e = new Error();
+        const frame = e.stack.split("\n")[2];
+        const where = frame.split(" ")[6].split(/[:()]/);
+        const arr = where[1].split("/");
+        const fileName = arr[arr.length - 1];
+        const lineNumber = where[2];
+        role += ` ${fileName}:${lineNumber}`;
+    }
+    console.log(role, input, ...extra);
 }
 
 export function debugLog(text: any, ...extra: any[]) {
-	if (config.settings.debug) {
-		hLog(text, ...extra);
-	}
+    if (config.settings.debug) {
+        hLog(text, ...extra);
+    }
+}
+
+export async function sleep(ms): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+export async function waitUntilReady(executor: () => Promise<boolean>, attempts: number, interval: number, onError: () => void): Promise<void> {
+    let i = 0;
+    while (i < attempts) {
+        if (await executor()) {
+            return;
+        }
+        await sleep(interval);
+        i++;
+    }
+    onError();
 }

helpers/elastic-routes.ts

@@ -145,6 +145,14 @@ function buildResUsageBulk(payloads, messageMap: MMap) {
     });
 }
 
+function buildScheduleBulk(payloads, messageMap: MMap) {
+    return flatMap(payloads, (payload, body) => {
+        const id = `${body.version}`;
+        messageMap.set(id, _.omit(payload, ['content']));
+        return makeScriptedOp(id, body);
+    });
+}
+
 function buildGenTrxBulk(payloads, messageMap: MMap) {
     return flatMap(payloads, (payload, body) => {
         const hash = createHash('sha256')
@@ -183,6 +191,10 @@ const generatorsMap = {
         index_name: 'userres',
         func: buildResUsageBulk
     },
+    schedule: {
+        index_name: 'schedule',
+        func: buildScheduleBulk
+    },
     generated_transaction: {
         index_name: 'gentrx',
         func: buildGenTrxBulk
@@ -193,21 +205,13 @@ const generatorsMap = {
     },
 };
 
-interface IndexDist {
-    index: string;
-    first_block: number;
-    last_block: number;
-}
-
 export class ElasticRoutes {
     public routes: any;
     cm: ConnectionManager;
     chain: string;
     ingestNodeCounters = {};
-    distributionMap: IndexDist[];
 
-    constructor(connectionManager: ConnectionManager, distributionMap: IndexDist[]) {
-        this.distributionMap = distributionMap;
+    constructor(connectionManager: ConnectionManager) {
         this.routes = {generic: this.handleGenericRoute.bind(this)};
         this.cm = connectionManager;
         this.chain = this.cm.chain;
@@ -382,27 +386,27 @@ export class ElasticRoutes {
         return Math.ceil(blockNum / this.cm.config.settings.index_partition_size).toString().padStart(6, '0');
     }
 
-    getIndexNameByBlock(block_num) {
-        if (!this.distributionMap) {
-            return null;
-        }
-        for (let i = this.distributionMap.length - 1; i >= 0; i--) {
-            const test = this.distributionMap[i].first_block <= block_num && this.distributionMap[i].last_block >= block_num;
-            if (test) {
-                return this.distributionMap[i].index;
-            }
-        }
-        return null;
-    }
-
-    addToIndexMap(map, idx, payload) {
-        if (idx) {
-            if (!map[idx]) {
-                map[idx] = [];
-            }
-            map[idx].push(payload);
-        }
-    }
+    // getIndexNameByBlock(block_num) {
+    //     if (!this.distributionMap) {
+    //         return null;
+    //     }
+    //     for (let i = this.distributionMap.length - 1; i >= 0; i--) {
+    //         const test = this.distributionMap[i].first_block <= block_num && this.distributionMap[i].last_block >= block_num;
+    //         if (test) {
+    //             return this.distributionMap[i].index;
+    //         }
+    //     }
+    //     return null;
+    // }
+    //
+    // addToIndexMap(map, idx, payload) {
+    //     if (idx) {
+    //         if (!map[idx]) {
+    //             map[idx] = [];
+    //         }
+    //         map[idx].push(payload);
+    //     }
+    // }
 
     private routeFactory(indexName: string, bulkGenerator, routerFunction) {
         return async (payloads, channel, cb) => {

hyp-repair

@@ -0,0 +1,2 @@
+#!/bin/bash
+node ./scripts/hyp-repair.js "$@"

interfaces/hyperionConfig.ts

@@ -20,6 +20,7 @@ export interface ScalingConfigs {
 }
 
 export interface MainSettings {
+    use_global_agent?: boolean;
     process_prefix?: string;
     ignore_snapshot?: boolean;
     ship_request_rev: string;
@@ -88,6 +89,8 @@ interface CachedRouteConfig {
 }
 
 interface ApiConfigs {
+    stream_scroll_batch?: number;
+    stream_scroll_limit?: number;
     enabled?: boolean;
     pm2_scaling?: number;
     node_max_old_space_size?: number;

interfaces/hyperionConnections.ts

@@ -23,6 +23,7 @@ interface HyperionChainData {
     ship: string;
     WS_ROUTER_PORT: number;
     WS_ROUTER_HOST: string;
+    control_port: number;
 }
 
 interface RedisConfig {

launcher.ts

@@ -13,6 +13,7 @@ const hyperionWorkers = {
 	reader: 'state-reader',
 	deserializer: 'deserializer',
 	continuous_reader: 'state-reader',
+	repair_reader: 'state-reader',
 	ingestor: 'indexer',
 	router: 'ws-router',
 	delta_updater: 'delta-updater'

modules/parsers/1.8-parser.ts

@@ -215,6 +215,7 @@ export default class HyperionParser extends BaseParser {
 						event: 'consumed_block',
 						block_num: result['block_num'],
 						block_id: result['block_id'],
+						block_ts: result['block_ts'],
 						trx_ids: result['trx_ids'],
 						lib: res.last_irreversible.block_num,
 						live: process.env.live_mode

modules/parsers/2.1-parser.ts

@@ -253,6 +253,7 @@ export default class HyperionParser extends BaseParser {
                         event: 'consumed_block',
                         block_num: result['block_num'],
                         block_id: result['block_id'],
+                        block_ts: result['block_ts'],
                         trx_ids: result['trx_ids'],
                         lib: res.last_irreversible.block_num,
                         live: process.env.live_mode

modules/parsers/3.2-parser.ts

@@ -137,6 +137,7 @@ export default class HyperionParser extends BaseParser {
                         event: 'consumed_block',
                         block_num: result['block_num'],
                         block_id: result['block_id'],
+                        block_ts: result['block_ts'],
                         trx_ids: result['trx_ids'],
                         lib: res.last_irreversible.block_num,
                         live: process.env.live_mode

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperion-history",
-  "version": "3.3.9",
+  "version": "3.3.9-6",
   "description": "Scalable Full History API Solution for EOSIO based blockchains",
   "main": "launcher.js",
   "scripts": {
@@ -14,7 +14,7 @@
     "fix-permissions": "node scripts/fix-permissions.js"
   },
   "engines": {
-    "node": "^18"
+    "node": ">=18"
   },
   "author": {
     "name": "EOS Rio",
@@ -26,54 +26,55 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@elastic/elasticsearch": "^7.17.0",
-    "@eosrio/node-abieos": "^2.1.1",
-    "@fastify/autoload": "4.0.1",
-    "@fastify/cors": "7.0.0",
-    "@fastify/formbody": "^6.0.1",
-    "@fastify/rate-limit": "^6.0.1",
-    "@fastify/redis": "^5.0.0",
-    "@fastify/swagger": "6.1.0",
+    "@elastic/elasticsearch": "^8.8.0",
+    "@eosrio/node-abieos": "3.2.0",
+    "@fastify/autoload": "5.7.1",
+    "@fastify/cors": "8.3.0",
+    "@fastify/formbody": "^7.4.0",
+    "@fastify/rate-limit": "^8.0.0",
+    "@fastify/redis": "^6.1.1",
+    "@fastify/swagger": "8.5.1",
     "@pm2/io": "^5.0.0",
     "amqplib": "^0.10.3",
     "async": "^3.2.4",
     "base-x": "^4.0.0",
-    "commander": "^8.3.0",
-    "cross-fetch": "^3.1.5",
+    "cli-progress": "^3.12.0",
+    "commander": "^10.0.1",
+    "cross-fetch": "^3.1.6",
     "eosjs": "^22.1.0",
-    "fast-json-stringify": "2.7.13",
-    "fastify": "3.29.4",
+    "fast-json-stringify": "5.7.0",
+    "fastify": "^4.17.0",
     "fastify-elasticsearch": "^2.0.0",
-    "fastify-plugin": "^3.0.1",
+    "fastify-plugin": "^4.5.0",
     "flatstr": "^1.0.12",
     "global-agent": "^3.0.0",
-    "got": "11.8.5",
-    "ioredis": "^4.28.5",
+    "got": "11.8.6",
+    "ioredis": "^5.3.2",
     "lodash": "^4.17.21",
     "moment": "^2.29.4",
-    "nodemailer": "^6.9.0",
+    "nodemailer": "^6.9.3",
+    "pino-pretty": "^10.0.0",
     "portfinder": "^1.0.32",
-    "pino-pretty": "^9.1.1",
-    "socket.io": "4.5.4",
-    "socket.io-client": "4.5.4",
+    "socket.io": "4.6.2",
+    "socket.io-client": "4.6.2",
     "socket.io-redis": "^6.1.1",
-    "telegraf": "^4.11.2",
-    "typescript": "^4.5.2",
-    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.19.0",
-    "ws": "^8.12.0"
+    "telegraf": "^4.12.3-canary.1",
+    "typescript": "^5.1.3",
+    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.30.0",
+    "ws": "^8.12.1"
   },
   "devDependencies": {
-    "@types/amqplib": "^0.8.2",
-    "@types/async": "^3.2.16",
+    "@types/amqplib": "^0.10.1",
+    "@types/async": "^3.2.18",
     "@types/global-agent": "^2.1.1",
-    "@types/ioredis": "^4.28.10",
-    "@types/lodash": "^4.14.191",
-    "@types/node": "^18.11.18",
-    "@types/nodemailer": "^6.4.7",
+    "@types/ioredis": "^5.0.0",
+    "@types/lodash": "^4.14.195",
+    "@types/node": "^20.2.5",
+    "@types/nodemailer": "^6.4.8",
     "@types/ws": "^8.5.4"
   },
   "optionalDependencies": {
     "bufferutil": "^4.0.7",
-    "utf-8-validate": "^5.0.10"
+    "utf-8-validate": "^6.0.3"
   }
 }

scripts/fix_missing_blocks/fix-missing-blocks.py

@@ -5,6 +5,7 @@ from subprocess import Popen, PIPE, STDOUT
 from sh import tail
 import time
 import sys
+import shutil
 
 
 # Import that you stop the indexer before you start running this script
@@ -41,28 +42,71 @@ live_reader = '.live_reader.*'
 rewrite = '.rewrite.*'
 
 
-shutting_down = r'.*Shutting down master.*'
+#shutting_down = r'.*Shutting down master.*'
+
 offline = r'.*(offline or unexistent)'
 lastblock_indexed = r'.*Last Indexed Block:.* (\d*)'  #group the last block indexed for reference.
 
 
 # Config
-hyperionfolder="/home/charles/hyperion-history-api/"
+hyperionfolder="/home/charles/hyperion/"
 indexer_log_file = "/home/charles/.pm2/logs/wax-indexer-out.log"
+ecosystem_file_location = f'{hyperionfolder}ecosystem.config.js'
 
 chain = "wax"  #set the Chain name eos or wax
 filepath=hyperionfolder+"chains/wax.config.json"
 indexer_stop = ["pm2", "trigger", "wax-indexer", "stop"]
-indexer_start = [hyperionfolder+"./run.sh", "wax-indexer"]
+indexer_start = [f"{hyperionfolder}./run.sh", f'{ecosystem_file_location}', "wax-indexer"]
+#indexer_start = ['/usr/bin/pm2', 'start', {ecosystem_file_location}, '--only', 'wax-indexer', '--update-env', '--silent']
 tail_indexer_logs = ["tail","-f", indexer_log_file]
-hyperion_version = '3.3'  #'3.1' or '3.3'
+hyperion_version = '3.7'  #'3.1' or '3.3'
 
 
 # No blocks being processed differs between 3.3 and 3.1
-if hyperion_version == '3.3':
-    no_blocks_processed = r'.*No blocks are being processed.*'
-else:
+if hyperion_version == '3.7':
+    no_blocks_processed = r'.*-------- BLOCK RANGE COMPLETED -------------.*'
+    shutting_down = r'.*No more active workers, stopping now...*'
+    
+elif hyperion_version == '3.1':
     no_blocks_processed = r'.*No blocks processed.*'
+    shutting_down = r'.*Shutting down master.*'
+else:
+    no_blocks_processed = r'.*-------- BLOCK RANGE COMPLETED -------------.*'
+    shutting_down = r'.*No more active workers, stopping now...*'
+    
+
+def update_run_script():
+    # Copy the original run.sh to run.bak
+    shutil.copy2(f"{hyperionfolder}/run.sh", f"{hyperionfolder}/run.bak")
+    
+    # Update the contents of run.sh with the new command
+    with open("run.sh", "w") as f:
+        f.write(f'''#!/usr/bin/env bash
+if [ $# -lt 2 ]; then
+  echo "Please inform the app name and ecosystem file location. ex: './run.sh indexer /path/to/ecosystem.config.js'"
+  exit 1
+fi
+echo -e "\n-->> Starting $1..."
+(
+  set -x
+  pm2 start "$2" --only "$1" --update-env --silent
+)
+echo -e "\n-->> Saving pm2 state..."
+(
+  set -x
+  pm2 save
+)
+echo -e "\n-->> Reading $2 logs..."
+(
+  set -x
+  pm2 logs --raw --lines 10 "$2"
+)''')
+
+def restore_run_script():
+    # Restore the original run.sh from run.bak
+    shutil.copy2(f"{hyperionfolder}/run.bak", f"{hyperionfolder}/run.sh")
+
+
 
 # Set ES index pattern
 es_index = f'{chain}-block-*'
@@ -208,8 +252,8 @@ def spinning_cursor(run):
       sys.stdout.flush()
 
 def startRewrite(gt,lt):
-    run = Popen(indexer_start,stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    time.sleep(10)
+    Popen(indexer_start,stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    time.sleep(2)
     logtail = Popen(tail_indexer_logs, stdin=PIPE, stdout=PIPE, stderr=STDOUT) 
     completedcount = 0
     for linenum, line in enumerate(logtail.stdout):

scripts/hpm.js

@@ -1,7 +1,11 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];

scripts/hyp-config.js

@@ -1,259 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const commander_1 = require("commander");
-const path_1 = __importDefault(require("path"));
-const promises_1 = require("fs/promises");
-const fs_1 = require("fs");
-const eosjs_1 = require("eosjs");
-const cross_fetch_1 = __importDefault(require("cross-fetch"));
-const ws_1 = __importDefault(require("ws"));
-const program = new commander_1.Command();
-const chainsDir = path_1.default.join(path_1.default.resolve(), 'chains');
-async function getConnections() {
-    const connectionsJsonFile = await (0, promises_1.readFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'));
-    return JSON.parse(connectionsJsonFile.toString());
-}
-async function getExampleConfig() {
-    const exampleChainData = await (0, promises_1.readFile)(path_1.default.join(chainsDir, 'example.config.json'));
-    return JSON.parse(exampleChainData.toString());
-}
-async function listChains(flags) {
-    const dirdata = await (0, promises_1.readdir)(chainsDir);
-    const connections = await getConnections();
-    const exampleChain = await getExampleConfig();
-    const configuredTable = [];
-    const pendingTable = [];
-    for (const file of dirdata.filter(f => f.endsWith('.json'))) {
-        if (file === 'example.config.json') {
-            continue;
-        }
-        try {
-            const fileData = await (0, promises_1.readFile)(path_1.default.join(chainsDir, file));
-            const jsonData = JSON.parse(fileData.toString());
-            const chainName = jsonData.settings.chain;
-            const chainConn = connections.chains[chainName];
-            const fullChainName = jsonData.api.chain_name;
-            // validate configs with example
-            let missingSections = 0;
-            let missingFields = 0;
-            for (const key in exampleChain) {
-                if (!jsonData[key]) {
-                    if (flags.fixMissingFields) {
-                        jsonData[key] = exampleChain[key];
-                    }
-                    else {
-                        console.log(`⚠ Section ${key} missing on ${file}`);
-                        missingSections++;
-                    }
-                }
-                else {
-                    for (const subKey in exampleChain[key]) {
-                        if (!jsonData[key].hasOwnProperty(subKey)) {
-                            if (flags.fixMissingFields) {
-                                jsonData[key][subKey] = exampleChain[key][subKey];
-                            }
-                            else {
-                                console.log(`⚠ Field ${subKey} missing from ${key} on ${file} (default = ${exampleChain[key][subKey]})`);
-                                missingFields++;
-                            }
-                        }
-                    }
-                }
-            }
-            if (flags.fixMissingFields) {
-                await (0, promises_1.writeFile)(path_1.default.join(chainsDir, file), JSON.stringify(jsonData, null, 2));
-            }
-            const common = {
-                full_name: fullChainName,
-                name: chainName,
-                parser: jsonData.settings.parser,
-                abi_scan: jsonData.indexer.abi_scan_mode,
-                live_reader: jsonData.indexer.live_reader,
-                start_on: jsonData.indexer.start_on,
-                stop_on: jsonData.indexer.stop_on
-            };
-            if (chainConn) {
-                configuredTable.push({
-                    ...common,
-                    nodeos_http: chainConn?.http,
-                    nodeos_ship: chainConn?.ship,
-                    chain_id: chainConn?.chain_id.substr(0, 16) + '...'
-                });
-            }
-            else {
-                pendingTable.push(common);
-            }
-        }
-        catch (e) {
-            console.log(`Failed to parse ${file} - ${e.message}`);
-        }
-    }
-    console.log(' >>>> Fully Configured Chains');
-    console.table(configuredTable);
-    if (!flags.valid) {
-        console.log(' >>>> Pending Configuration Chains');
-        console.table(pendingTable);
-    }
-}
-async function newChain(shortName, options) {
-    console.log(`Creating new config for ${shortName}...`);
-    const targetPath = path_1.default.join(chainsDir, `${shortName}.config.json`);
-    if ((0, fs_1.existsSync)(targetPath)) {
-        console.error(`Chain config for ${shortName} already defined!`);
-        process.exit(0);
-    }
-    // read example
-    const exampleChain = await getExampleConfig();
-    // read connections.json
-    const connections = await getConnections();
-    if (connections.chains[shortName]) {
-        console.log('Connections already defined!');
-        console.log(connections.chains[shortName]);
-        process.exit(0);
-    }
-    else {
-        connections.chains[shortName] = {
-            name: '',
-            ship: '',
-            http: '',
-            chain_id: '',
-            WS_ROUTER_HOST: '127.0.0.1',
-            WS_ROUTER_PORT: 7001
-        };
-    }
-    const jsonData = exampleChain;
-    jsonData.settings.chain = shortName;
-    if (options.http) {
-        if (options.http.startsWith('http://') || options.http.startsWith('https://')) {
-            console.log(`Testing connection on ${options.http}`);
-        }
-        else {
-            console.error(`Invalid HTTP Endpoint [${options.http}] - Url must start with either http:// or https://`);
-            process.exit(1);
-        }
-        // test nodeos availability
-        const jsonRpc = new eosjs_1.JsonRpc(options.http, { fetch: cross_fetch_1.default });
-        const info = await jsonRpc.get_info();
-        jsonData.api.chain_api = options.http;
-        connections.chains[shortName].chain_id = info.chain_id;
-        connections.chains[shortName].http = options.http;
-        if (info.server_version_string.includes('2.1')) {
-            jsonData.settings.parser = '2.1';
-        }
-        else {
-            jsonData.settings.parser = '1.8';
-        }
-        console.log(`Parser version set to ${jsonData.settings.parser}`);
-        console.log(`Chain ID: ${info.chain_id}`);
-    }
-    if (options.ship) {
-        if (options.ship.startsWith('ws://') || options.ship.startsWith('wss://')) {
-            console.log(`Testing connection on ${options.ship}`);
-        }
-        else {
-            console.error(`Invalid WS Endpoint [${options.ship}] - Url must start with either ws:// or wss://`);
-            process.exit(1);
-        }
-        // test ship availability
-        const status = await new Promise(resolve => {
-            const ws = new ws_1.default(options.ship);
-            ws.on("message", (data) => {
-                try {
-                    const abi = JSON.parse(data.toString());
-                    if (abi.version) {
-                        console.log(`Received SHIP Abi Version: ${abi.version}`);
-                        resolve(true);
-                    }
-                    else {
-                        resolve(false);
-                    }
-                }
-                catch (e) {
-                    console.log(e.message);
-                    resolve(false);
-                }
-                ws.close();
-            });
-            ws.on("error", err => {
-                console.log(err);
-                ws.close();
-                resolve(false);
-            });
-        });
-        if (status) {
-            connections.chains[shortName].ship = options.ship;
-        }
-        else {
-            console.log(`Invalid SHIP Endpoint [${options.ship}]`);
-        }
-    }
-    const fullNameArr = [];
-    shortName.split('-').forEach((word) => {
-        fullNameArr.push(word[0].toUpperCase() + word.substr(1));
-    });
-    const fullChainName = fullNameArr.join(' ');
-    jsonData.api.chain_name = fullChainName;
-    connections.chains[shortName].name = fullChainName;
-    console.log(connections.chains[shortName]);
-    console.log('Saving connections.json...');
-    await (0, promises_1.writeFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
-    console.log(`Saving chains/${shortName}.config.json...`);
-    await (0, promises_1.writeFile)(targetPath, JSON.stringify(jsonData, null, 2));
-}
-async function rmChain(shortName) {
-    console.log(`Removing config for ${shortName}...`);
-    const targetPath = path_1.default.join(chainsDir, `${shortName}.config.json`);
-    if (!(0, fs_1.existsSync)(targetPath)) {
-        console.error(`Chain config for ${shortName} not found!`);
-        process.exit(0);
-    }
-    // create backups
-    const backupDir = path_1.default.join(path_1.default.resolve(), 'configuration_backups');
-    if (!(0, fs_1.existsSync)(backupDir)) {
-        await (0, promises_1.mkdir)(backupDir);
-    }
-    const connections = await getConnections();
-    try {
-        const chainConn = connections.chains[shortName];
-        const now = Date.now();
-        if (chainConn) {
-            await (0, promises_1.writeFile)(path_1.default.join(backupDir, `${shortName}_${now}_connections.json`), JSON.stringify(chainConn, null, 2));
-        }
-        await (0, promises_1.cp)(targetPath, path_1.default.join(backupDir, `${shortName}_${now}_config.json`));
-    }
-    catch (e) {
-        console.log(e.message);
-        console.log('Failed to create backups! Aborting!');
-        process.exit(1);
-    }
-    console.log(`Deleting ${targetPath}`);
-    await (0, promises_1.rm)(targetPath);
-    delete connections.chains[shortName];
-    console.log('Saving connections.json...');
-    await (0, promises_1.writeFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
-    console.log(`✅ ${shortName} removal completed!`);
-}
-// main program
-(() => {
-    const list = program.command('list').alias('ls');
-    list.command('chains')
-        .option('--valid', 'only show valid chains')
-        .option('--fix-missing-fields', 'set defaults on missing fields')
-        .description('list configured chains')
-        .action(listChains);
-    const newCmd = program.command('new').alias('n');
-    newCmd.command('chain <shortName>')
-        .description('initialize new chain config based on example')
-        .option('--http <http_endpoint>', 'define chain api http endpoint')
-        .option('--ship <ship_endpoint>', 'define state history ws endpoint')
-        .action(newChain);
-    const remove = program.command('remove').alias('rm');
-    remove.command('chain <shortName>')
-        .description('backup and delete chain configuration')
-        .action(rmChain);
-    program.parse(process.argv);
-})();
-//# sourceMappingURL=hyp-config.js.map

scripts/hyp-config.ts

@@ -3,17 +3,28 @@ import path from "path";
 import {cp, mkdir, readdir, readFile, rm, writeFile} from "fs/promises";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
 import {HyperionConnections} from "../interfaces/hyperionConnections";
-import {existsSync} from "fs";
+import {copyFileSync, existsSync, rmSync} from "fs";
 import {JsonRpc} from "eosjs";
 import fetch from "cross-fetch";
 import WebSocket from 'ws';
+import * as readline from "readline";
+import * as amqp from "amqplib";
+import IORedis from "ioredis";
+import {Client} from "@elastic/elasticsearch";
 
 const program = new Command();
 const chainsDir = path.join(path.resolve(), 'chains');
+const connectionsPath = path.join(path.resolve(), 'connections.json');
+const exampleConnectionsPath = path.join(path.resolve(), 'example-connections.json');
+const backupDir = path.join(path.resolve(), 'configuration_backups');
 
 async function getConnections(): Promise<HyperionConnections> {
-    const connectionsJsonFile = await readFile(path.join(path.resolve(), 'connections.json'));
-    return JSON.parse(connectionsJsonFile.toString());
+    if (existsSync(connectionsPath)) {
+        const connectionsJsonFile = await readFile(connectionsPath);
+        return JSON.parse(connectionsJsonFile.toString());
+    } else {
+        return null;
+    }
 }
 
 async function getExampleConfig(): Promise<HyperionConfig> {
@@ -85,7 +96,7 @@ async function listChains(flags) {
                     ...common,
                     nodeos_http: chainConn?.http,
                     nodeos_ship: chainConn?.ship,
-                    chain_id: chainConn?.chain_id.substr(0,16) + '...'
+                    chain_id: chainConn?.chain_id.substr(0, 16) + '...'
                 });
             } else {
                 pendingTable.push(common);
@@ -119,6 +130,11 @@ async function newChain(shortName, options) {
     // read connections.json
     const connections = await getConnections();
 
+    if (!connections) {
+        console.log(`The connections.json file is not present, please run "./hyp-config connections init" to configure it.`);
+        process.exit(1);
+    }
+
     if (connections.chains[shortName]) {
         console.log('Connections already defined!');
         console.log(connections.chains[shortName]);
@@ -130,7 +146,8 @@ async function newChain(shortName, options) {
             http: '',
             chain_id: '',
             WS_ROUTER_HOST: '127.0.0.1',
-            WS_ROUTER_PORT: 7001
+            WS_ROUTER_PORT: 7001,
+            control_port: 7002
         };
     }
 
@@ -147,18 +164,25 @@ async function newChain(shortName, options) {
         }
 
         // test nodeos availability
-        const jsonRpc = new JsonRpc(options.http, {fetch});
-        const info = await jsonRpc.get_info();
-        jsonData.api.chain_api = options.http;
-        connections.chains[shortName].chain_id = info.chain_id;
-        connections.chains[shortName].http = options.http;
-        if (info.server_version_string.includes('2.1')) {
-            jsonData.settings.parser = '2.1';
-        } else {
-            jsonData.settings.parser = '1.8';
+        try {
+            const jsonRpc = new JsonRpc(options.http, {fetch});
+            const info = await jsonRpc.get_info();
+            jsonData.api.chain_api = options.http;
+            connections.chains[shortName].chain_id = info.chain_id;
+            connections.chains[shortName].http = options.http;
+            if (info.server_version_string.includes('v3')) {
+                jsonData.settings.parser = '3.2';
+            } else if (info.server_version_string.includes('v2.1')) {
+                jsonData.settings.parser = '2.1';
+            } else {
+                jsonData.settings.parser = '1.8';
+            }
+            console.log(`Parser version set to ${jsonData.settings.parser}`);
+            console.log(`Chain ID: ${info.chain_id}`);
+        } catch (e) {
+            console.error(`Invalid HTTP Endpoint [${options.http}] - ${e.message}`);
+            process.exit(1);
         }
-        console.log(`Parser version set to ${jsonData.settings.parser}`);
-        console.log(`Chain ID: ${info.chain_id}`)
     }
 
     if (options.ship) {
@@ -204,7 +228,7 @@ async function newChain(shortName, options) {
     const fullNameArr = [];
     shortName.split('-').forEach((word: string) => {
         fullNameArr.push(word[0].toUpperCase() + word.substr(1));
-        });
+    });
     const fullChainName = fullNameArr.join(' ');
 
     jsonData.api.chain_name = fullChainName;
@@ -213,7 +237,7 @@ async function newChain(shortName, options) {
     console.log(connections.chains[shortName]);
 
     console.log('Saving connections.json...');
-    await writeFile(path.join(path.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    await writeFile(connectionsPath, JSON.stringify(connections, null, 2));
     console.log(`Saving chains/${shortName}.config.json...`);
     await writeFile(targetPath, JSON.stringify(jsonData, null, 2));
 }
@@ -228,7 +252,6 @@ async function rmChain(shortName) {
     }
 
     // create backups
-    const backupDir = path.join(path.resolve(), 'configuration_backups');
     if (!existsSync(backupDir)) {
         await mkdir(backupDir);
     }
@@ -254,12 +277,310 @@ async function rmChain(shortName) {
     await rm(targetPath);
     delete connections.chains[shortName];
     console.log('Saving connections.json...');
-    await writeFile(path.join(path.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    await writeFile(connectionsPath, JSON.stringify(connections, null, 2));
     console.log(`✅ ${shortName} removal completed!`);
 }
 
+async function getExampleConnections(): Promise<HyperionConnections | null> {
+    try {
+        const connectionsJsonFile = await readFile(exampleConnectionsPath);
+        return JSON.parse(connectionsJsonFile.toString());
+    } catch (e: any) {
+        return null;
+    }
+}
+
+async function checkAMQP(conn: HyperionConnections): Promise<boolean> {
+    console.log(`\n[info] [RABBIT:AMQP] - Testing rabbitmq amqp connection...`);
+    let frameMaxValue = '0x10000';
+    if (conn.amqp.frameMax) {
+        frameMaxValue = conn.amqp.frameMax;
+    }
+    const u = encodeURIComponent(conn.amqp.user);
+    const p = encodeURIComponent(conn.amqp.pass);
+    const v = encodeURIComponent(conn.amqp.vhost)
+    const amqp_url = `amqp://${u}:${p}@${conn.amqp.host}/${v}?frameMax=${frameMaxValue}`;
+    try {
+        const connection = await amqp.connect(amqp_url);
+        if (connection) {
+            console.log('[info] [RABBIT:AMQP] - Connection established!');
+            await connection.close();
+            console.log(`\n[info] [RABBIT:HTTP] - Testing rabbitmq http api connection...`);
+            const apiUrl = `${conn.amqp.protocol}://${conn.amqp.api}/api/nodes`;
+
+            let headers = new Headers();
+            headers.set('Authorization', 'Basic ' + btoa(conn.amqp.user + ":" + conn.amqp.pass));
+            const data = await fetch(apiUrl, {
+                method: 'GET',
+                headers: headers
+            });
+            const resp = await data.json();
+            if (resp.length > 0) {
+                for (const node of resp) {
+                    console.log(`[info] [RABBIT:HTTP] - Node name: ${node.name} (running: ${node.running})`);
+                    if (!node.running) {
+                        return false;
+                    }
+                }
+                return true;
+            } else {
+                return false;
+            }
+        } else {
+            return false;
+        }
+    } catch (reason: any) {
+        console.log('[error] [RABBIT] - ' + reason.message);
+        return false;
+    }
+}
+
+async function checkES(conn: HyperionConnections): Promise<boolean> {
+    console.log(`\n[info] [ES] - Testing elasticsearch connection...`);
+    let es_url;
+    const _es = conn.elasticsearch;
+    if (!_es.protocol) {
+        _es.protocol = 'http';
+    }
+    if (_es.user !== '') {
+        es_url = `${_es.protocol}://${_es.user}:${_es.pass}@${_es.host}`;
+    } else {
+        es_url = `${_es.protocol}://${_es.host}`
+    }
+    // console.log(`Prepared client: ${es_url}`);
+    const client = new Client({
+        node: es_url,
+        ssl: {rejectUnauthorized: false}
+    });
+    try {
+        const result = await client.cat.health();
+        console.log(`[info] [ES] - ${result.body.trim()}`);
+        console.log('[info] [ES] - Connection established!');
+        return true;
+    } catch (reason: any) {
+        console.log('[error] [ES] - ' + reason.message);
+        return false;
+    }
+}
+
+async function checkRedis(conn: HyperionConnections) {
+    console.log(`\n[info] [REDIS] - Testing redis connection...`);
+    try {
+        const ioRedisClient = new IORedis(conn.redis);
+        const pingResult = await ioRedisClient.ping();
+        if (pingResult === 'PONG') {
+            console.log('[info] [REDIS] - Connection established!');
+            ioRedisClient.disconnect();
+            return true;
+        } else {
+            console.log('[error] [REDIS] - PONG expected, got: ' + pingResult);
+            return false;
+        }
+    } catch (reason: any) {
+        console.log('[error] [REDIS] - ' + reason.message);
+        return false;
+    }
+}
+
+async function initConfig() {
+    const connections = await getConnections();
+
+    if (connections) {
+        console.log('connections.json already created!\n use "./hyp-config connections reset" to revert to default');
+        return;
+    }
+
+    const exampleConn = await getExampleConnections();
+
+    const rl = readline.createInterface({input: process.stdin, output: process.stdout});
+    const prompt = (query: string) => new Promise((resolve) => rl.question(query, resolve));
+
+    const conn = exampleConn;
+
+    if (!conn) {
+        console.log('No example-connections.json to use as reference!');
+        process.exit(1);
+    }
+    conn.chains = {};
+
+    // check rabbitmq
+    let amqp_state = false;
+    while (!amqp_state) {
+        amqp_state = await checkAMQP(conn);
+        if (!amqp_state) {
+
+            const amqp_user = await prompt('\n > Enter the RabbitMQ user (or press ENTER to use "hyperion_user"): ');
+            if (amqp_user) {
+                conn.amqp.user = amqp_user as string;
+            } else {
+                conn.amqp.user = 'hyperion_user'
+            }
+
+            const amqp_pass = await prompt(' > Enter the RabbitMQ password (or press ENTER to use "hyperion_password"): ');
+            if (amqp_pass) {
+                conn.amqp.pass = amqp_pass as string;
+            } else {
+                conn.amqp.pass = 'hyperion_password'
+            }
+
+            const amqp_vhost = await prompt(' > Enter the RabbitMQ vhost (or press ENTER to use "hyperion"): ');
+            if (amqp_vhost) {
+                conn.amqp.vhost = amqp_vhost as string;
+            } else {
+                conn.amqp.vhost = 'hyperion'
+            }
+
+            console.log('\n------ current RabbitMQ config -----');
+            console.log(conn.amqp);
+            console.log('--------------------------------------');
+        }
+    }
+
+    console.log(`\n RabbitMQ ✅`);
+
+    // check elasticsearch
+    let elastic_state = false;
+    while (!elastic_state) {
+        elastic_state = await checkES(conn);
+        if (!elastic_state) {
+
+            const es_user = await prompt('\n > Enter the elasticsearch user (or press ENTER to use "elastic"): ');
+            if (es_user) {
+                conn.elasticsearch.user = es_user as string;
+            } else {
+                conn.elasticsearch.user = 'elastic'
+            }
+
+            const es_pass = await prompt(' > Enter the elasticsearch password: ');
+            if (es_pass) {
+                conn.elasticsearch.pass = es_pass as string;
+            }
+
+            const es_proto = await prompt(' > Do you want to use http or https?\n1 = http\n2 = https (default)\n');
+            if (es_proto === "1") {
+                conn.elasticsearch.protocol = 'http';
+            } else {
+                conn.elasticsearch.protocol = 'https';
+            }
+
+            console.log('\n------ current elasticsearch config -----');
+            console.log(conn.elasticsearch);
+            console.log('-------------------------------------------');
+        }
+    }
+
+    console.log(`\n Elasticsearch ✅`);
+
+    // check redis connection
+    let redis_state = false;
+    while (!redis_state) {
+        redis_state = await checkRedis(conn);
+        if (!redis_state) {
+            const redis_host = await prompt('\n > Enter the Redis Server address (or press ENTER to use "127.0.0.1"): ');
+            if (redis_host) {
+                conn.redis.host = redis_host as string;
+            } else {
+                conn.redis.host = '127.0.0.1';
+            }
+
+            const redis_port = await prompt(' > Enter the Redis Server port (or press ENTER to use "6379"): ');
+            if (redis_port) {
+                conn.redis.port = parseInt(redis_port as string);
+            } else {
+                conn.redis.port = 6379;
+            }
+
+            console.log('\n------ current redis config -----');
+            console.log(conn.redis);
+            console.log('-----------------------------------');
+        }
+    }
+
+    console.log(`\n Redis ✅`);
+
+    console.log('Init completed! Saving configuration...');
+
+    await writeFile(connectionsPath, JSON.stringify(conn, null, 2));
+
+    console.log('✅ ✅');
+    rl.close();
+}
+
+async function testConnections() {
+    if (existsSync(connectionsPath)) {
+        try {
+            const conn = await getConnections();
+            if (conn) {
+                const results = {
+                    redis: await checkRedis(conn),
+                    elasticsearch: await checkES(conn),
+                    rabbitmq: await checkAMQP(conn)
+                }
+                console.log('\n------ Testing Completed -----');
+                console.table(results);
+            } else {
+                console.log(`The connections.json was not found!.`);
+            }
+        } catch (e: any) {
+            console.log(e.message);
+        }
+    } else {
+        console.log(`The connections.json file is not present, please run "./hyp-config connections init" to configure it.`);
+    }
+}
+
+async function resetConnections() {
+    try {
+
+        // create backups
+        if (!existsSync(backupDir)) {
+            await mkdir(backupDir);
+        }
+
+        if (existsSync(connectionsPath)) {
+            const rl = readline.createInterface({input: process.stdin, output: process.stdout});
+            const prompt = (query: string) => new Promise((resolve) => rl.question(query, resolve));
+            const confirmation = await prompt('Are you sure you want to reset the connection configuration? Type "YES" to confirm.\n');
+            if (confirmation === 'YES') {
+                copyFileSync(connectionsPath, path.join(backupDir, 'connections.json.bak'));
+                rmSync(connectionsPath);
+                console.log('connections.json removed, please use "./hyp-config connections init" to reconfigure');
+            } else {
+                console.log('Operation canceled. No files were removed.');
+            }
+            rl.close();
+        } else {
+            console.log(`The connections.json file is not present, please run "./hyp-config connections init" to configure it.`);
+        }
+    } catch (e: any) {
+        console.log(e);
+    }
+}
+
+
 // main program
 (() => {
+
+    const connections = program.command('connections');
+
+    // ./hyp-config connections init
+    connections
+        .command('init')
+        .description('create and test the connections.json file')
+        .action(initConfig);
+
+    // ./hyp-config connections test
+    connections
+        .command('test')
+        .description('test connections to the hyperion infrastructure')
+        .action(testConnections);
+
+    // ./hyp-config connections reset
+    connections
+        .command('reset')
+        .description('remove the connections.json file')
+        .action(resetConnections);
+
     const list = program.command('list').alias('ls');
     list.command('chains')
         .option('--valid', 'only show valid chains')

scripts/hyp-repair.ts

@@ -0,0 +1,622 @@
+import {Command} from 'commander';
+import {readFileSync} from "node:fs";
+import {Client, estypes} from "@elastic/elasticsearch";
+// @ts-ignore
+import cliProgress from "cli-progress";
+import {JsonRpc} from 'eosjs';
+import fetch from 'cross-fetch';
+import {existsSync, mkdirSync, writeFileSync} from "fs";
+import {HyperionBlock} from "./repair-cli/interfaces.js";
+import {
+    getBlocks,
+    getFirstIndexedBlock,
+    getLastIndexedBlock,
+    initESClient,
+    readChainConfig,
+    readConnectionConfig
+} from "./repair-cli/functions.js";
+import {SearchResponse} from "@elastic/elasticsearch/api/types";
+
+import {WebSocket} from 'ws';
+import {block} from "../definitions/index-templates";
+
+const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+const program = new Command();
+const errorRanges: any[] = [];
+let pendingBlock: HyperionBlock | null = null;
+
+const missingBlocks: {
+    start: number,
+    end: number
+}[] = [];
+
+async function run(client: Client, rpc: JsonRpc, indexName: string, blockInit: number, firstBlock: number, qtdTotal: any, loop: any = 1) {
+    let blockInitial = blockInit;
+    let blockFinal = blockInitial - qtdTotal;
+    const tRef = process.hrtime.bigint();
+    progressBar.start(loop, 0);
+    for (let i: any = 1; i <= loop; i++) {
+        if (blockFinal < firstBlock) {
+            blockFinal = firstBlock;
+        }
+        try {
+            let {body} = await getBlocks(client, indexName, blockInitial, blockFinal, qtdTotal);
+            let {hits: {hits}} = body;
+            const blocks = hits.map((obj: any) => obj._source);
+            await findForksOnRange(blocks, rpc);
+            blockInitial = blockFinal;
+            blockFinal = blockInitial - qtdTotal;
+            progressBar.update(i);
+        } catch (e: any) {
+            console.log("Error: ", e);
+            process.exit(1);
+        }
+    }
+    progressBar.stop();
+    console.log("=========== Forked Ranges ===========");
+    console.table(errorRanges);
+    console.log("=========== Missing Ranges ==========");
+    console.table(missingBlocks);
+    const tDiff = Number(process.hrtime.bigint() - tRef) / 1000000;
+    console.log(`Total time: ${Math.round(tDiff / 1000)}s`);
+}
+
+async function findForksOnRange(blocks: HyperionBlock[], rpc: JsonRpc) {
+
+    const removals: Set<string> = new Set();
+    let start = null;
+    let end = null;
+    if (blocks.length > 0 && pendingBlock && blocks[0].block_num !== pendingBlock.block_num) {
+        blocks.unshift(pendingBlock);
+    }
+
+    let i = 0;
+    for (const currentBlock of blocks) {
+        const currentBlockNumber = currentBlock.block_num;
+        // console.log('current -> ', currentBlockNumber);
+        const previousBlock = blocks[i + 1];
+        if (!previousBlock) {
+            pendingBlock = currentBlock;
+            // console.log('pending -> ', pendingBlock.block_num);
+            return;
+        }
+        i++;
+        // console.log('previous -> ', previousBlock.block_num);
+        if (previousBlock.block_num !== currentBlockNumber - 1) {
+            console.log(`\nBlock number mismatch, expected: ${currentBlockNumber - 1} got ${previousBlock.block_num}`);
+            missingBlocks.push({
+                start: previousBlock.block_num + 1,
+                end: currentBlockNumber - 1
+            });
+            continue;
+        }
+
+        if (previousBlock && previousBlock.block_id !== currentBlock.prev_id) {
+            if (start === null) {
+                start = currentBlockNumber - 1;
+                const blockData = await rpc.get_block_info(start);
+                if (blockData && blockData.id !== previousBlock.block_id) {
+                    removals.add(previousBlock.block_id);
+                }
+            }
+        } else {
+            if (start) {
+                const blockData = await rpc.get_block_info(currentBlockNumber);
+                if (blockData) {
+                    if (blockData.id !== currentBlock.block_id) {
+                        removals.add(currentBlock.block_id);
+                    } else {
+                        end = currentBlockNumber + 1;
+                        const range = {start, end, ids: [...removals]};
+                        errorRanges.push(range);
+                        removals.clear();
+                        // console.log(`\n ⚠️⚠️ Forked at ${range.start} to ${range.end}`);
+                        start = null;
+                        end = null;
+                    }
+                }
+            }
+        }
+    }
+}
+
+async function scanChain(chain: string, args: any) {
+    const chainConfig = readChainConfig(chain);
+
+    console.log(chainConfig.settings.index_version);
+
+    const config = readConnectionConfig();
+
+    const client = initESClient(config);
+
+    const jsonRpc = new JsonRpc(config.chains[chain].http, {fetch});
+
+    const ping = await client.ping();
+
+    if (!ping) {
+        console.log('Could not connect to ElasticSearch');
+        process.exit();
+    }
+
+    const blockIndex = `${chain}-block-${chainConfig.settings.index_version}`;
+
+    let firstBlock;
+    let lastBlock;
+
+    if (args.first) {
+        // reduce by 2 to account for the fork end validation
+        firstBlock = args.first - 2;
+    } else {
+        firstBlock = await getFirstIndexedBlock(client, blockIndex);
+    }
+
+    if (args.last) {
+        lastBlock = args.last;
+    } else {
+        lastBlock = await getLastIndexedBlock(client, blockIndex);
+    }
+
+    const totalBlocks = lastBlock - firstBlock;
+
+    let batchSize = 1000;
+    if (args.batch) {
+        batchSize = args.batch;
+    }
+
+    const numberOfBatches = Math.ceil(totalBlocks / batchSize);
+
+    console.log("Range:", firstBlock, lastBlock);
+    console.log("Total Blocks:", totalBlocks);
+    console.log("Batch Size:", batchSize);
+    console.log("Number of Batches:", numberOfBatches);
+
+    await run(client, jsonRpc, blockIndex, lastBlock, firstBlock, batchSize, numberOfBatches);
+    console.log(`Finished checking forked blocks!`);
+
+    if (errorRanges.length > 0 || missingBlocks.length > 0) {
+        if (!existsSync('.repair')) {
+            mkdirSync('.repair');
+        }
+    }
+
+    if (errorRanges.length > 0) {
+        let path = `.repair/${chain}-${firstBlock + 2}-${lastBlock}-forked-blocks.json`;
+        if (args.outFile) {
+            path = args.outFile + '-forked-blocks.json';
+        }
+        writeFileSync(path, JSON.stringify(errorRanges));
+    }
+
+    if (missingBlocks.length > 0) {
+        let path = `.repair/${chain}-${firstBlock + 2}-${lastBlock}-missing-blocks.json`;
+        if (args.outFile) {
+            path = args.outFile + '-missing-blocks.json';
+        }
+        writeFileSync(path, JSON.stringify(missingBlocks));
+    }
+}
+
+async function repairMissing(chain: string, file: string, args: any) {
+    const chainConfig = readChainConfig(chain);
+    const config = readConnectionConfig();
+    const client = initESClient(config);
+    const ping = await client.ping();
+    if (!ping) {
+        console.log('Could not connect to ElasticSearch');
+        process.exit();
+    }
+    await fillMissingBlocksFromFile(args.host, chain, file, args.dry);
+}
+
+async function repairChain(chain: string, file: string, args: any) {
+
+    const chainConfig = readChainConfig(chain);
+    const config = readConnectionConfig();
+    const client = initESClient(config);
+    const ping = await client.ping();
+    if (!ping) {
+        console.log('Could not connect to ElasticSearch');
+        process.exit();
+    }
+
+    if (args.checkTasks) {
+        const tasks = await client.tasks.list();
+        if (tasks.body && tasks.body.nodes) {
+            for (let node in tasks.body.nodes) {
+                const nodeTasks = tasks.body.nodes[node].tasks;
+                console.log(nodeTasks);
+            }
+        }
+        process.exit(0);
+    }
+
+    const forkedBlocks = JSON.parse(readFileSync(file).toString());
+    const blockIndex = `${chain}-block-${chainConfig.settings.index_version}`;
+    let deleteBlocks = 0;
+    let deleteActions = 0;
+    let deleteDeltas = 0;
+    let deleteAbis = 0;
+    let deleteAccounts = 0;
+    let deleteVoters = 0;
+    let deleteProposals = 0;
+    let deleteLinks = 0;
+    let deletePermissions = 0;
+
+    for (const range of forkedBlocks) {
+
+        // ACTIONS
+        const searchActions = {
+            index: `${chain}-action-${chainConfig.settings.index_version}-*`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
+                    }
+                }
+            }
+        };
+
+        if (args.dry) {
+            const resultActions = await client.search<any>(searchActions);
+            if ((resultActions.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
+                deleteActions += (resultActions.body.hits.total as estypes.SearchTotalHits)?.value;
+            }
+        } else {
+            delete searchActions.size;
+            const deletedActionsResult = await client.deleteByQuery(searchActions);
+            if (deletedActionsResult && deletedActionsResult.body.deleted && deletedActionsResult.body.deleted > 0) {
+                deleteActions += deletedActionsResult.body.deleted;
+            }
+        }
+
+        // DELTAS
+        const searchDeltas = {
+            index: `${chain}-delta-${chainConfig.settings.index_version}-*`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
+                    }
+                }
+            }
+        }
+
+        if (args.dry) {
+            const resultDeltas = await client.search<any>(searchDeltas);
+            if ((resultDeltas.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
+                deleteDeltas += (resultDeltas.body.hits.total as estypes.SearchTotalHits)?.value;
+            }
+        } else {
+            delete searchDeltas.size;
+            const deletedDeltasResult = await client.deleteByQuery(searchDeltas);
+            if (deletedDeltasResult && deletedDeltasResult.body.deleted && deletedDeltasResult.body.deleted > 0) {
+                deleteDeltas += deletedDeltasResult.body.deleted;
+            }
+        }
+
+        // ABIS
+        const searchAbis = {
+            index: `${chain}-abi-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [{range: {block: {lte: range.start, gte: range.end}}}]
+                    }
+                }
+            }
+        }
+
+        if (args.dry) {
+            const resultAbis = await client.search<any>(searchAbis);
+            if ((resultAbis.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
+                deleteAbis += (resultAbis.body.hits.total as estypes.SearchTotalHits)?.value;
+                console.log('ABIs', {lte: range.start, gte: range.end});
+            }
+        } else {
+            delete searchAbis.size;
+            const deletedAbisResult = await client.deleteByQuery(searchAbis);
+            if (deletedAbisResult && deletedAbisResult.body.deleted && deletedAbisResult.body.deleted > 0) {
+                deleteAbis += deletedAbisResult.body.deleted;
+            }
+        }
+
+        // ACCOUNTS
+        const searchAccounts = {
+            index: `${chain}-table-accounts-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
+                    }
+                }
+            }
+        }
+
+        if (args.dry) {
+            const resultAccounts = await client.search<any>(searchAccounts);
+            if ((resultAccounts.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
+                console.log('[WARNING]', (resultAccounts.body.hits.total as estypes.SearchTotalHits)?.value, 'accounts needs to be updated');
+                deleteAccounts += (resultAccounts.body.hits.total as estypes.SearchTotalHits)?.value;
+            }
+        } else {
+            delete searchAccounts.size;
+            const deletedAccountsResult = await client.deleteByQuery(searchAccounts);
+            if (deletedAccountsResult && deletedAccountsResult.body.deleted && deletedAccountsResult.body.deleted > 0) {
+                deleteAccounts += deletedAccountsResult.body.deleted;
+            }
+        }
+
+        // VOTERS
+        const searchVoters = {
+            index: `${chain}-table-voters-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
+                    }
+                }
+            }
+        }
+
+        if (args.dry) {
+            const resultVoters = await client.search<any>(searchVoters);
+            if ((resultVoters.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
+                console.log('[WARNING]', (resultVoters.body.hits.total as estypes.SearchTotalHits)?.value, 'voters needs to be updated');
+                deleteVoters += (resultVoters.body.hits.total as estypes.SearchTotalHits)?.value;
+            }
+        } else {
+            delete searchVoters.size;
+            const deletedVotersResult = await client.deleteByQuery(searchVoters);
+            if (deletedVotersResult && deletedVotersResult.body.deleted && deletedVotersResult.body.deleted > 0) {
+                deleteVoters += deletedVotersResult.body.deleted;
+            }
+        }
+
+        // PROPOSALS
+        const searchProposals = {
+            index: `${chain}-table-proposals-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
+                    }
+                }
+            }
+        }
+
+        if (args.dry) {
+            const resultProposals = await client.search<any>(searchProposals);
+            if ((resultProposals.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
+                console.log('[WARNING]', (resultProposals.body.hits.total as estypes.SearchTotalHits)?.value, 'proposals needs to be updated');
+                deleteProposals += (resultProposals.body.hits.total as estypes.SearchTotalHits)?.value;
+            }
+        } else {
+            delete searchProposals.size;
+            const deletedProposalsResult = await client.deleteByQuery(searchProposals);
+            if (deletedProposalsResult && deletedProposalsResult.body.deleted && deletedProposalsResult.body.deleted > 0) {
+                deleteProposals += deletedProposalsResult.body.deleted;
+            }
+        }
+
+        // LINKS
+        const searchLinks = {
+            index: `${chain}-link-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
+                    }
+                }
+            }
+        }
+
+        if (args.dry) {
+            const resultLinks = await client.search<any>(searchLinks);
+            if ((resultLinks.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
+                console.log('[WARNING]', (resultLinks.body.hits.total as estypes.SearchTotalHits)?.value, 'links needs to be updated');
+                deleteLinks += (resultLinks.body.hits.total as estypes.SearchTotalHits)?.value;
+            }
+        } else {
+            delete searchLinks.size;
+            const deletedLinksResult = await client.deleteByQuery(searchLinks);
+            if (deletedLinksResult && deletedLinksResult.body.deleted && deletedLinksResult.body.deleted > 0) {
+                deleteLinks += deletedLinksResult.body.deleted;
+            }
+        }
+
+        // PERMISSIONS
+        const searchPermissions = {
+            index: `${chain}-perm-${chainConfig.settings.index_version}`,
+            size: 0,
+            track_total_hits: true,
+            body: {
+                query: {
+                    bool: {
+                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
+                    }
+                }
+            }
+        }
+
+        if (args.dry) {
+            const resultPermissions = await client.search<any>(searchPermissions);
+            if ((resultPermissions.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
+                console.log('[WARNING]', (resultPermissions.body.hits.total as estypes.SearchTotalHits)?.value, 'permissions needs to be updated');
+                console.log({lte: range.start, gte: range.end});
+                deletePermissions += (resultPermissions.body.hits.total as estypes.SearchTotalHits)?.value;
+            }
+        } else {
+            delete searchPermissions.size;
+            const deletedPermissionsResult = await client.deleteByQuery(searchPermissions);
+            if (deletedPermissionsResult && deletedPermissionsResult.body.deleted && deletedPermissionsResult.body.deleted > 0) {
+                deletePermissions += deletedPermissionsResult.body.deleted;
+            }
+        }
+
+        for (const id of range.ids) {
+            const searchBlocks = {
+                index: blockIndex,
+                body: {
+                    query: {bool: {must: [{term: {block_id: {value: id}}}]}}
+                }
+            };
+            if (args.dry) {
+                console.log(`Deleting block ${id}...`);
+                const resultBlocks = await client.search<SearchResponse<HyperionBlock>>(searchBlocks);
+                if (resultBlocks.body.hits.hits.length > 0 && resultBlocks.body.hits.hits[0]._source) {
+                    deleteBlocks++;
+                }
+            } else {
+                const result = await client.deleteByQuery(searchBlocks);
+                if (result && result.body.deleted && result.body.deleted > 0) {
+                    deleteBlocks += result.body.deleted;
+                }
+            }
+        }
+    }
+
+    if (args.dry) {
+        console.log(`DRY-RUN: Would have deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`);
+        console.dir({
+            blocks: deleteBlocks,
+            actions: deleteActions,
+            deltas: deleteDeltas,
+            abis: deleteAbis,
+            accounts: deleteAccounts,
+            voters: deleteVoters,
+            proposals: deleteProposals,
+            links: deleteLinks,
+            permissions: deletePermissions
+        });
+    } else {
+        console.log(`Deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`);
+    }
+
+    await fillMissingBlocksFromFile(args.host, chain, file, args.dry);
+}
+
+async function fillMissingBlocksFromFile(host: string, chain: string, file: string, dryRun: string) {
+    const config = readConnectionConfig();
+    const controlPort = config.chains[chain].control_port;
+    let hyperionIndexer = `ws://localhost:${controlPort}`;
+    if (host) {
+        hyperionIndexer = 'ws://' + host + ':' + controlPort;
+    }
+    const controller = new WebSocket(hyperionIndexer + '/local');
+    controller.on('open', async () => {
+        console.log('Connected to Hyperion Controller');
+        const parsedFile = JSON.parse(readFileSync(file).toString());
+        if (!dryRun) {
+            const payload = {
+                "event": "fill_missing_blocks",
+                "data": parsedFile
+            };
+            controller.send(JSON.stringify(payload));
+        } else {
+            console.log('Dry run, skipping repair');
+            controller.close();
+        }
+    });
+    controller.on('message', (data) => {
+        const parsed = JSON.parse(data.toString());
+        if (parsed.event === 'repair_completed') {
+            console.log(`Hyperion repair completed!`);
+            controller.close();
+        }
+    });
+    controller.on('close', () => {
+        console.log('Disconnected from Hyperion Controller');
+    });
+    controller.on('error', (err) => {
+        console.log(err);
+    });
+}
+
+function viewFile(file: string) {
+    const data = readFileSync(file, 'utf8');
+    const parsed = JSON.parse(data);
+    console.table(parsed);
+}
+
+// Commander Logic
+
+program
+    .name('Hyperion Repair CLI')
+    .description('CLI to find and repair forked and missing blocks on Hyperion')
+    .version('0.2.2');
+
+
+program.command('scan <chain>')
+    .description('scan for forked blocks')
+    .option('-d, --dry', 'dry-run, do not delete or repair blocks')
+    .option('-o, --out-file <file>', 'forked-blocks.json output file')
+    .option('-f, --first <number>', 'initial block to start validation')
+    .option('-l, --last <number>', 'last block to validate')
+    .option('-b, --batch <number>', 'batch size to process')
+    .action(scanChain);
+
+program.command('repair <chain> <file>')
+    .description('repair forked blocks')
+    .option('-h, --host <host>', 'Hyperion local control api')
+    .option('-d, --dry', 'dry-run, do not delete or repair blocks')
+    .option('-t, --check-tasks', 'check for running tasks')
+    .action(repairChain);
+
+program.command('fill-missing <chain> <file>')
+    .description('write missing blocks')
+    .option('-h, --host <host>', 'Hyperion local control api')
+    .option('-d, --dry', 'dry-run, do not delete or repair blocks')
+    .action(repairMissing);
+
+program.command('view <file>')
+    .description('view forked blocks')
+    .action(viewFile);
+
+program.command('connect')
+    .option('-h, --host <host>', 'Hyperion local control api')
+    .action(async (args) => {
+        let hyperionIndexer = 'ws://localhost:4321';
+        let valid = false;
+        if (args.host) {
+            hyperionIndexer = args.host;
+        }
+        try {
+            const controller = new WebSocket(hyperionIndexer + '/local');
+            controller.on('open', () => {
+                valid = true;
+                controller.close();
+            });
+            controller.on('close', () => {
+                if (valid) {
+                    console.log(`✅  Hyperion Indexer Online - ${hyperionIndexer}`);
+                }
+            });
+            controller.on('error', (err) => {
+                console.log("Error:", err.message);
+                console.log(`Failed to connect on Hyperion Indexer at ${hyperionIndexer}, please use "--host ws://ADDRESS:PORT" to specify a remote indexer connection`);
+            });
+        } catch (e) {
+            console.log(e);
+        }
+    });
+
+program.parse();
+
+
+
+
+

scripts/repair-cli/functions.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getBlocks = exports.getLastIndexedBlock = exports.getFirstIndexedBlock = exports.readChainConfig = exports.initESClient = exports.readConnectionConfig = void 0;
+const node_fs_1 = require("node:fs");
+const elasticsearch_1 = require("@elastic/elasticsearch");
+function readConnectionConfig() {
+    const file = (0, node_fs_1.readFileSync)('connections.json', 'utf8');
+    return JSON.parse(file);
+}
+exports.readConnectionConfig = readConnectionConfig;
+function initESClient(config) {
+    const node = `${config.elasticsearch.protocol}://${config.elasticsearch.host}`;
+    return new elasticsearch_1.Client({
+        nodes: [node],
+        auth: {
+            username: config.elasticsearch.user,
+            password: config.elasticsearch.pass
+        },
+        ssl: {
+            rejectUnauthorized: false
+        }
+    });
+}
+exports.initESClient = initESClient;
+function readChainConfig(chain) {
+    const file = (0, node_fs_1.readFileSync)(`chains/${chain}.config.json`, 'utf8');
+    return JSON.parse(file);
+}
+exports.readChainConfig = readChainConfig;
+async function getFirstIndexedBlock(client, blockIndex) {
+    const { body } = await client.search({
+        index: blockIndex,
+        size: 1,
+        body: {
+            sort: [{ block_num: { order: 'asc' } }]
+        }
+    });
+    if (body.hits.hits[0]._source) {
+        return body.hits.hits[0]._source.block_num;
+    }
+    else {
+        console.log('No blocks indexed yet');
+        process.exit();
+    }
+}
+exports.getFirstIndexedBlock = getFirstIndexedBlock;
+async function getLastIndexedBlock(client, blockIndex) {
+    const { body } = await client.search({
+        index: blockIndex,
+        size: 1,
+        body: {
+            sort: [{ block_num: { order: 'desc' } }]
+        }
+    });
+    if (body.hits.hits[0]._source) {
+        return body.hits.hits[0]._source.block_num;
+    }
+    else {
+        console.log('No blocks indexed yet');
+        process.exit();
+    }
+}
+exports.getLastIndexedBlock = getLastIndexedBlock;
+async function getBlocks(client, indexName, blocoInicial, blocoFinal, size) {
+    return await client.search({
+        index: indexName,
+        size: size,
+        body: {
+            sort: [{ block_num: { order: "desc" } }],
+            query: {
+                range: {
+                    block_num: {
+                        gte: blocoFinal,
+                        lte: blocoInicial,
+                    },
+                },
+            },
+        }
+    });
+}
+exports.getBlocks = getBlocks;
+//# sourceMappingURL=functions.js.map

scripts/repair-cli/functions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"functions.js","sourceRoot":"","sources":["functions.ts"],"names":[],"mappings":";;;AAAA,qCAAqC;AACrC,0DAAuD;AAGvD,SAAgB,oBAAoB;IAChC,MAAM,IAAI,GAAG,IAAA,sBAAY,EAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAHD,oDAGC;AAED,SAAgB,YAAY,CAAC,MAAW;IACpC,MAAM,IAAI,GAAG,GAAG,MAAM,CAAC,aAAa,CAAC,QAAQ,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IAC/E,OAAO,IAAI,sBAAM,CAAC;QACd,KAAK,EAAE,CAAC,IAAI,CAAC;QACb,IAAI,EAAE;YACF,QAAQ,EAAE,MAAM,CAAC,aAAa,CAAC,IAAI;YACnC,QAAQ,EAAE,MAAM,CAAC,aAAa,CAAC,IAAI;SACtC;QACD,GAAG,EAAE;YACD,kBAAkB,EAAE,KAAK;SAC5B;KACJ,CAAC,CAAC;AACP,CAAC;AAZD,oCAYC;AAED,SAAgB,eAAe,CAAC,KAAa;IACzC,MAAM,IAAI,GAAG,IAAA,sBAAY,EAAC,UAAU,KAAK,cAAc,EAAE,MAAM,CAAC,CAAC;IACjE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC;AAHD,0CAGC;AAEM,KAAK,UAAU,oBAAoB,CAAC,MAAc,EAAE,UAAkB;IACzE,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,MAAM,CAAC,MAAM,CAAwC;QACtE,KAAK,EAAE,UAAU;QACjB,IAAI,EAAE,CAAC;QACP,IAAI,EAAE;YACF,IAAI,EAAE,CAAC,EAAC,SAAS,EAAE,EAAC,KAAK,EAAE,KAAK,EAAC,EAAC,CAAC;SACtC;KACJ,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;KAC9C;SAAM;QACH,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,EAAE,CAAC;KAClB;AACL,CAAC;AAdD,oDAcC;AAEM,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,UAAkB;IACxE,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,MAAM,CAAC,MAAM,CAAwC;QACtE,KAAK,EAAE,UAAU;QACjB,IAAI,EAAE,CAAC;QACP,IAAI,EAAE;YACF,IAAI,EAAE,CAAC,EAAC,SAAS,EAAE,EAAC,KAAK,EAAE,MAAM,EAAC,EAAC,CAAC;SACvC;KACJ,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE;QAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;KAC9C;SAAM;QACH,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,EAAE,CAAC;KAClB;AACL,CAAC;AAdD,kDAcC;AAEM,KAAK,UAAU,SAAS,CAAC,MAAW,EAAE,SAAiB,EAAE,YAAiB,EAAE,UAAe,EAAE,IAAS;IACzG,OAAO,MAAM,MAAM,CAAC,MAAM,CAAC;QACvB,KAAK,EAAE,SAAS;QAChB,IAAI,EAAE,IAAI;QACV,IAAI,EAAE;YACF,IAAI,EAAE,CAAC,EAAC,SAAS,EAAE,EAAC,KAAK,EAAE,MAAM,EAAC,EAAC,CAAC;YACpC,KAAK,EAAE;gBACH,KAAK,EAAE;oBACH,SAAS,EAAE;wBACP,GAAG,EAAE,UAAU;wBACf,GAAG,EAAE,YAAY;qBACpB;iBACJ;aACJ;SACJ;KACJ,CAAC,CAAC;AACP,CAAC;AAhBD,8BAgBC"}

scripts/repair-cli/functions.ts

@@ -0,0 +1,78 @@
+import {readFileSync} from "node:fs";
+import {Client, estypes} from "@elastic/elasticsearch";
+import {HyperionBlock} from "./interfaces.js";
+
+export function readConnectionConfig() {
+    const file = readFileSync('connections.json', 'utf8');
+    return JSON.parse(file);
+}
+
+export function initESClient(config: any) {
+    const node = `${config.elasticsearch.protocol}://${config.elasticsearch.host}`;
+    return new Client({
+        nodes: [node],
+        auth: {
+            username: config.elasticsearch.user,
+            password: config.elasticsearch.pass
+        },
+        ssl: {
+            rejectUnauthorized: false
+        }
+    });
+}
+
+export function readChainConfig(chain: string) {
+    const file = readFileSync(`chains/${chain}.config.json`, 'utf8');
+    return JSON.parse(file);
+}
+
+export async function getFirstIndexedBlock(client: Client, blockIndex: string) {
+    const {body} = await client.search<estypes.SearchResponse<HyperionBlock>>({
+        index: blockIndex,
+        size: 1,
+        body: {
+            sort: [{block_num: {order: 'asc'}}]
+        }
+    });
+    if (body.hits.hits[0]._source) {
+        return body.hits.hits[0]._source.block_num;
+    } else {
+        console.log('No blocks indexed yet');
+        process.exit();
+    }
+}
+
+export async function getLastIndexedBlock(client: Client, blockIndex: string) {
+    const {body} = await client.search<estypes.SearchResponse<HyperionBlock>>({
+        index: blockIndex,
+        size: 1,
+        body: {
+            sort: [{block_num: {order: 'desc'}}]
+        }
+    });
+    if (body.hits.hits[0]._source) {
+        return body.hits.hits[0]._source.block_num;
+    } else {
+        console.log('No blocks indexed yet');
+        process.exit();
+    }
+}
+
+export async function getBlocks(client: any, indexName: string, blocoInicial: any, blocoFinal: any, size: any) {
+    return await client.search({
+        index: indexName,
+        size: size,
+        body: {
+            sort: [{block_num: {order: "desc"}}],
+            query: {
+                range: {
+                    block_num: {
+                        gte: blocoFinal,
+                        lte: blocoInicial,
+                    },
+                },
+            },
+        }
+    });
+}
+

scripts/repair-cli/interfaces.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=interfaces.js.map

scripts/repair-cli/interfaces.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["interfaces.ts"],"names":[],"mappings":""}

scripts/repair-cli/interfaces.ts

@@ -0,0 +1,11 @@
+export interface HyperionBlock {
+    "@timestamp": string,
+    "block_num": number,
+    "block_id": string,
+    "producer": string,
+    "new_producers"?: any,
+    "schedule_version": number,
+    "cpu_usage": number,
+    "net_usage": number,
+    "prev_id": string
+}

tsconfig.json

@@ -6,6 +6,7 @@
     "allowSyntheticDefaultImports": true,
     "noImplicitAny": false,
     "moduleResolution": "node",
+    "noEmitOnError": true,
     "sourceMap": true,
     "allowJs": false,
     "typeRoots": [

workers/deserializer.ts

@@ -11,36 +11,36 @@ import flatstr from 'flatstr';
 
 const FJS = require('fast-json-stringify');
 
-const lightBlockSerializer = FJS({
-    title: 'Light Block',
-    type: 'object',
-    properties: {
-        '@timestamp': {type: 'string'},
-        block_num: {type: 'integer'},
-        block_id: {type: 'string'},
-        prev_id: {type: 'string'},
-        producer: {type: 'string'},
-        new_producers: {
-            type: 'object',
-            nullable: true,
-            properties: {
-                version: {type: 'integer'},
-                producers: {
-                    type: 'array',
-                    items: {
-                        properties: {
-                            block_signing_key: {type: 'string'},
-                            producer_name: {type: 'string'}
-                        }
-                    }
-                }
-            }
-        },
-        schedule_version: {type: 'integer'},
-        cpu_usage: {type: 'integer'},
-        net_usage: {type: 'integer'}
-    }
-});
+// const lightBlockSerializer = FJS({
+//     title: 'Light Block',
+//     type: 'object',
+//     properties: {
+//         '@timestamp': {type: 'string'},
+//         block_num: {type: 'integer'},
+//         block_id: {type: 'string'},
+//         prev_id: {type: 'string'},
+//         producer: {type: 'string'},
+//         new_producers: {
+//             type: 'object',
+//             nullable: true,
+//             properties: {
+//                 version: {type: 'integer'},
+//                 producers: {
+//                     type: 'array',
+//                     items: {
+//                         properties: {
+//                             block_signing_key: {type: 'string'},
+//                             producer_name: {type: 'string'}
+//                         }
+//                     }
+//                 }
+//             }
+//         },
+//         schedule_version: {type: 'integer'},
+//         cpu_usage: {type: 'integer'},
+//         net_usage: {type: 'integer'}
+//     }
+// });
 
 const index_queues = require('../definitions/index-queues').index_queues;
 const {AbiDefinitions} = require("../definitions/abi_def");
@@ -410,6 +410,16 @@ export default class MainDSWorker extends HyperionWorker {
                         live: process.env.live_mode
                     });
                 }
+
+                // stream light block
+                if (this.allowStreaming) {
+                    this.ch.publish('', this.chain + ':stream', Buffer.from(JSON.stringify(light_block)), {
+                        headers: {
+                            event: 'block',
+                            blockNum: light_block.block_num
+                        }
+                    });
+                }
             }
 
             // Process Delta Traces (must be done first to catch ABI updates)
@@ -495,6 +505,7 @@ export default class MainDSWorker extends HyperionWorker {
             return {
                 block_num: res['this_block']['block_num'],
                 block_id: res['this_block']['block_id'],
+                block_ts,
                 trx_ids: onBlockTransactions,
                 size: _traces.length
             };
@@ -1149,6 +1160,7 @@ export default class MainDSWorker extends HyperionWorker {
                 let jsonRow = await this.processContractRowNative(payload, block_num);
 
                 if (jsonRow?.value && !jsonRow['_blacklisted']) {
+                    console.log(jsonRow);
                     debugLog('Delta DS failed ->>', jsonRow);
                     jsonRow = await this.processContractRowNative(payload, block_num - 1);
                     debugLog('Retry with previous ABI ->>', jsonRow);
@@ -1389,9 +1401,33 @@ export default class MainDSWorker extends HyperionWorker {
         },
 
         // Global Chain configuration update
-        // "global_property": async (global_property, block_num, block_ts, row, block_id) => {
-        //     hLog(block_num, global_property);
-        // },
+        "global_property": async (global_property, block_num, block_ts) => {
+            if (global_property.proposed_schedule.version !== 0) {
+                hLog("Proposed Schedule version: " + global_property.proposed_schedule.version + " at block: " + global_property.proposed_schedule_block_num);
+                try {
+                    const payload = {
+                        block_num: global_property.proposed_schedule_block_num,
+                        '@timestamp': block_ts,
+                        version: global_property.proposed_schedule.version,
+                        producers: global_property.proposed_schedule.producers
+                    };
+                    payload.producers.forEach((producer: any) => {
+                        producer.name = producer.producer_name;
+                        delete producer.producer_name;
+                        if (producer.authority[0] === 'block_signing_authority_v0') {
+                            producer.authority = producer.authority[1];
+                            producer.keys = producer.authority.keys.map((key: any) => {
+                                return key.key;
+                            });
+                            delete producer.authority;
+                        }
+                    });
+                    await this.pushToIndexQueue(payload, 'schedule');
+                } catch (e) {
+                    hLog("Failed to parse proposed schedule: " + e.message);
+                }
+            }
+        },
 
         // Activated Protocol features
         // "protocol_state": async (protocol_state, block_num, block_ts, row, block_id) => {
@@ -1458,6 +1494,8 @@ export default class MainDSWorker extends HyperionWorker {
                                 await this.deltaStructHandlers[key](data[1], block_num, block_ts, row, block_id);
                             } catch (e) {
                                 hLog(`Delta struct [${key}] processing error: ${e.message}`);
+                                hLog(e);
+                                console.log(data[1]);
                             }
                         }
                     }
@@ -1602,41 +1640,46 @@ export default class MainDSWorker extends HyperionWorker {
 
         this.tableHandlers[EOSIO_ALIAS + ':producers'] = (delta) => {
             const data = delta['data'];
-            delta['@producers'] = {
-                total_votes: parseFloat(data['total_votes']),
-                is_active: data['is_active'],
-                unpaid_blocks: data['unpaid_blocks']
-            };
-            delete delta['data'];
+            if (data) {
+                delta['@producers'] = {
+                    total_votes: parseFloat(data['total_votes']),
+                    is_active: data['is_active'],
+                    unpaid_blocks: data['unpaid_blocks']
+                };
+                delete delta['data'];
+            }
         };
 
         this.tableHandlers[EOSIO_ALIAS + ':userres'] = (delta) => {
             const data = delta['data'];
-            const net = parseFloat(data['net_weight'].split(" ")[0]);
-            const cpu = parseFloat(data['cpu_weight'].split(" ")[0]);
-            delta['@userres'] = {
-                owner: data['owner'],
-                net_weight: net,
-                cpu_weight: cpu,
-                total_weight: parseFloat((net + cpu).toFixed(4)),
-                ram_bytes: parseInt(data['ram_bytes'])
-            };
-            delete delta['data'];
+            if (data['net_weight'] && data['cpu_weight']) {
+                const net = parseFloat(data['net_weight'].split(" ")[0]);
+                const cpu = parseFloat(data['cpu_weight'].split(" ")[0]);
+                delta['@userres'] = {
+                    owner: data['owner'],
+                    net_weight: net,
+                    cpu_weight: cpu,
+                    total_weight: parseFloat((net + cpu).toFixed(4)),
+                    ram_bytes: parseInt(data['ram_bytes'])
+                };
+                delete delta['data'];
+            }
         };
 
         this.tableHandlers[EOSIO_ALIAS + ':delband'] = (delta) => {
             const data = delta['data'];
-            const net = parseFloat(data['net_weight'].split(" ")[0]);
-            const cpu = parseFloat(data['cpu_weight'].split(" ")[0]);
-            delta['@delband'] = {
-                from: data['from'],
-                to: data['to'],
-                net_weight: net,
-                cpu_weight: cpu,
-                total_weight: parseFloat((net + cpu).toFixed(4))
-            };
-            delete delta['data'];
-            // hLog(delta);
+            if (data['net_weight'] && data['cpu_weight']) {
+                const net = parseFloat(data['net_weight'].split(" ")[0]);
+                const cpu = parseFloat(data['cpu_weight'].split(" ")[0]);
+                delta['@delband'] = {
+                    from: data['from'],
+                    to: data['to'],
+                    net_weight: net,
+                    cpu_weight: cpu,
+                    total_weight: parseFloat((net + cpu).toFixed(4))
+                };
+                delete delta['data'];
+            }
         };
 
         this.tableHandlers[EOSIO_ALIAS + '.msig:proposal'] = async (delta) => {

workers/indexer.ts

@@ -11,20 +11,11 @@ export default class IndexerWorker extends HyperionWorker {
     private temp_indexed_count = 0;
 
     esRoutes: ElasticRoutes;
-    distributionMap;
 
     constructor() {
         super();
 
-        if (process.env.distribution) {
-            try {
-                this.distributionMap = JSON.parse(process.env.distribution);
-            } catch {
-                hLog('Failed to parse distribution map');
-            }
-        }
-
-        this.esRoutes = new ElasticRoutes(this.manager, this.distributionMap);
+        this.esRoutes = new ElasticRoutes(this.manager);
         this.indexQueue = cargo((payload: Message[], callback) => {
             if (this.ch_ready && payload) {
                 if (this.esRoutes.routes[process.env.type]) {

workers/state-reader.ts

@@ -22,6 +22,7 @@ export default class StateReader extends HyperionWorker {
     private completionMonitoring: NodeJS.Timeout;
     private types: Map<string, Type>;
     private local_block_num = parseInt(process.env.first_block, 10) - 1;
+    private local_block_id = '';
     private queueSizeCheckInterval = 5000;
     private local_distributed_count = 0;
     private lastPendingCount = 0;
@@ -41,6 +42,9 @@ export default class StateReader extends HyperionWorker {
 
     private forkedBlocks = new Map<string, number>();
     private idle = true;
+    private repairMode = false;
+    private internalPendingRanges = [];
+
 
     constructor() {
         super();
@@ -190,17 +194,56 @@ export default class StateReader extends HyperionWorker {
 
     newRange(data: any) {
         debugLog(`new_range [${data.first_block},${data.last_block}]`);
+
+        if (this.repairMode) {
+            hLog(`Received Repair request [${data.first_block},${data.last_block}]`);
+        }
+
         this.local_distributed_count = 0;
         clearInterval(this.completionMonitoring);
         this.completionMonitoring = null;
         this.completionSignaled = false;
         this.local_last_block = data.last_block;
+
+        // make sure no data is requested from before the first indexed block on ship
+        if (parseInt(data.first_block) < this.shipInitStatus['trace_begin_block']) {
+            data.first_block = this.shipInitStatus['trace_begin_block'];
+            hLog("Impossible to repair requested range, first block is before the first indexed block on ship, trimming to " + data.first_block);
+        }
+
         this.range_size = parseInt(data.last_block) - parseInt(data.first_block);
-        if (this.allowRequests) {
-            this.requestBlockRange(data.first_block, data.last_block);
-            this.pendingRequest = null;
+
+        // if range size is not positive we need to immediately signal completion to master
+        if (this.range_size <= 0) {
+            this.emitCompletedSignal();
+        }
+
+        // make sure the range doesn't exceed the batch size
+        if (this.range_size > this.conf.scaling.batch_size) {
+            // split the range into multiple ranges
+            const ranges = [];
+            let start = parseInt(data.first_block);
+            let end = parseInt(data.last_block);
+            while (start < end) {
+                const s = start;
+                const e = start + this.conf.scaling.batch_size;
+                ranges.push({
+                    first_block: s,
+                    last_block: e > end ? end : e
+                });
+                start = e + 1;
+            }
+            console.log(`Splitting range [${data.first_block},${data.last_block}] into ${ranges.length} ranges`);
+            this.internalPendingRanges = ranges;
+            const next = this.internalPendingRanges.shift();
+            this.newRange(next);
         } else {
-            this.pendingRequest = [data.first_block, data.last_block];
+            if (this.allowRequests) {
+                this.requestBlockRange(data.first_block, data.last_block);
+                this.pendingRequest = null;
+            } else {
+                this.pendingRequest = [data.first_block, data.last_block];
+            }
         }
     }
 
@@ -270,10 +313,18 @@ export default class StateReader extends HyperionWorker {
                 if (pending === 0) {
                     debugLog(`Reader completed - ${this.range_size} - ${this.local_distributed_count}`);
                     clearInterval(this.completionMonitoring);
-                    process.send({
-                        event: 'completed',
-                        id: process.env['worker_id']
-                    });
+
+                    // check if there are any pending ranges, then signal completion to master
+                    if (this.internalPendingRanges.length === 0) {
+                        process.send({
+                            event: 'completed',
+                            id: process.env['worker_id']
+                        });
+                    } else {
+                        // process next range in queue
+                        const next = this.internalPendingRanges.shift();
+                        this.newRange(next);
+                    }
                 }
             }, 200);
         }
@@ -285,7 +336,7 @@ export default class StateReader extends HyperionWorker {
                 try {
                     await this.onMessage(block);
                 } catch (e) {
-                    console.log(e);
+                    hLog(e);
                 }
             }
             this.ackBlockRange(block_array.length);
@@ -308,7 +359,7 @@ export default class StateReader extends HyperionWorker {
         }
 
         if (!process.env.worker_role) {
-            console.log("[FATAL ERROR] undefined role! Exiting now!");
+            hLog("[FATAL ERROR] undefined role! Exiting now!");
             this.ship.close(false);
             process.exit(1);
             return;
@@ -356,6 +407,15 @@ export default class StateReader extends HyperionWorker {
                             break;
                         }
 
+                        case 'repair_reader': {
+                            this.repairMode = true;
+                            hLog('Waiting for operations...');
+                            process.send({
+                                event: 'repair_reader_ready'
+                            });
+                            break;
+                        }
+
                     }
                 } else {
                     this.ship.close(true);
@@ -370,10 +430,17 @@ export default class StateReader extends HyperionWorker {
                 if (res['this_block']) {
                     this.idle = false;
                     const blk_num = res['this_block']['block_num'];
+                    const blk_id = res['this_block']['block_id'];
+
                     const lib = res['last_irreversible'];
                     const task_payload = {num: blk_num, content: data};
 
                     if (res.block && res.traces && res.deltas) {
+
+                        if (this.repairMode) {
+                            hLog("Repaired block: " + blk_num);
+                        }
+
                         debugLog(`block_num: ${blk_num}, block_size: ${res.block.length}, traces_size: ${res.traces.length}, deltas_size: ${res.deltas.length}`);
                     } else {
                         if (!res.traces) {
@@ -390,18 +457,35 @@ export default class StateReader extends HyperionWorker {
                     if (this.isLiveReader) {
 
                         // LIVE READER MODE
-                        if (blk_num !== this.local_block_num + 1) {
-                            hLog(`Expected: ${this.local_block_num + 1}, received: ${blk_num}`);
+                        let prev_id;
+                        if (res['prev_block'] && res['prev_block']['block_id']) {
+                            prev_id = res['prev_block']['block_id'];
+                        }
+
+                        // // 2% chance to modify the previous block id to simulate a fork
+                        // if (Math.random() < 0.02) {
+                        //     // prev_id = prev_id + 'a';
+                        //     this.local_block_num = this.local_block_num - 1;
+                        // }
+
+                        if ((this.local_block_id && prev_id && this.local_block_id !== prev_id) || (blk_num !== this.local_block_num + 1)) {
+
+                            console.log((this.local_block_id && prev_id && this.local_block_id !== prev_id), (blk_num !== this.local_block_num + 1));
+                            console.log(blk_num, this.local_block_num + 1);
+
+                            hLog(`Unlinked block at ${prev_id} with previous block ${this.local_block_id}`);
+                            hLog(`Forked block: ${blk_num}`);
                             try {
                                 // delete all previously stored data for the forked blocks
                                 await this.handleFork(res);
                             } catch (e) {
                                 hLog(`Failed to handle fork during live reading! - Error: ${e.message}`);
                             }
-                        } else {
-                            this.local_block_num = blk_num;
                         }
 
+                        this.local_block_num = blk_num;
+                        this.local_block_id = blk_id;
+
                         if (lib.block_num > this.local_lib) {
                             this.local_lib = lib.block_num;
                             // emit lib update event
@@ -409,22 +493,32 @@ export default class StateReader extends HyperionWorker {
                         }
 
                         await this.stageOneDistQueue.push(task_payload);
+
                         return 1;
+
                     } else {
 
                         // Detect skipped first block
                         if (!this.receivedFirstBlock) {
                             if (blk_num !== this.local_block_num + 1) {
-                                hLog(`WARNING: First block received was #${blk_num}, but #${this.local_block_num + 1} was expected!`);
-                                hLog(`Make sure the block.log file contains the requested range, check with "eosio-blocklog --smoke-test"`);
+                                // Special case for block 2 that is actually the first block on ship
+                                if (blk_num !== 2) {
+                                    hLog(`WARNING: First block received was #${blk_num}, but #${this.local_block_num + 1} was expected!`);
+                                    hLog(`Make sure the block.log file contains the requested range, check with "eosio-blocklog --smoke-test"`);
+                                }
                                 this.local_block_num = blk_num - 1;
+                                this.local_distributed_count++;
+                                process.send({
+                                    event: 'skipped_block',
+                                    block_num: this.local_block_num + 1
+                                })
                             }
                             this.receivedFirstBlock = true;
                         }
 
                         // BACKLOG MODE
                         if (this.future_block !== 0 && this.future_block === blk_num) {
-                            console.log('Missing block ' + blk_num + ' received!');
+                            hLog('Missing block ' + blk_num + ' received!');
                         } else {
                             this.future_block = 0;
                         }
@@ -449,6 +543,9 @@ export default class StateReader extends HyperionWorker {
                         }
                     }
                 } else {
+                    if (this.repairMode) {
+                        hLog("No data on requested block!");
+                    }
                     this.idle = true;
                     // hLog(`Reader is idle! - Head at: ${result[1].head.block_num}`);
                     // this.ship.close(true);
@@ -588,11 +685,14 @@ export default class StateReader extends HyperionWorker {
                 id: targetBlock
             });
             targetBlock++;
-            console.log(blockData);
             if (blockData.body) {
                 const targetBlockId = blockData.body._source.block_id;
                 this.forkedBlocks.set(targetBlockId, Date.now());
-                this.deleteForkedBlock(targetBlockId).catch(console.log);
+                try {
+                    await this.deleteForkedBlock(targetBlockId);
+                } catch (e: any) {
+                    hLog(`Error deleting forked block ${targetBlockId}: ${e.message}`);
+                }
             }
         }
     }
@@ -616,41 +716,41 @@ export default class StateReader extends HyperionWorker {
         this.send(['get_blocks_ack_request_v0', {num_messages: size}]);
     }
 
-    private startQueueWatcher() {
-        setInterval(() => {
-            if (this.isLiveReader) {
-                this.manager.checkQueueSize(this.chain + ":live_blocks").then(value => {
-                    if (value > this.conf.scaling.block_queue_limit) {
-                        this.allowRequests = false;
-                    } else {
-                        this.allowRequests = true;
-                        if (this.pendingRequest) {
-                            this.processPending();
-                        }
-                    }
-                });
-            } else {
-                let checkArr = [];
-                for (let i = 0; i < this.conf.scaling.ds_queues; i++) {
-                    const q = this.chain + ":blocks:" + (i + 1);
-                    checkArr.push(this.manager.checkQueueSize(q));
-                }
-                Promise.all(checkArr).then(data => {
-                    if (data.some(el => el > this.conf.scaling.block_queue_limit)) {
-                        if (this.allowRequests) {
-                            hLog('ship reader paused!', data);
-                        }
-                        this.allowRequests = false;
-                    } else {
-                        this.allowRequests = true;
-                        if (this.pendingRequest) {
-                            this.processPending();
-                        }
-                    }
-                });
-            }
-        }, this.queueSizeCheckInterval);
-    }
+    // private startQueueWatcher() {
+    //     setInterval(() => {
+    //         if (this.isLiveReader) {
+    //             this.manager.checkQueueSize(this.chain + ":live_blocks").then(value => {
+    //                 if (value > this.conf.scaling.block_queue_limit) {
+    //                     this.allowRequests = false;
+    //                 } else {
+    //                     this.allowRequests = true;
+    //                     if (this.pendingRequest) {
+    //                         this.processPending();
+    //                     }
+    //                 }
+    //             });
+    //         } else {
+    //             let checkArr = [];
+    //             for (let i = 0; i < this.conf.scaling.ds_queues; i++) {
+    //                 const q = this.chain + ":blocks:" + (i + 1);
+    //                 checkArr.push(this.manager.checkQueueSize(q));
+    //             }
+    //             Promise.all(checkArr).then(data => {
+    //                 if (data.some(el => el > this.conf.scaling.block_queue_limit)) {
+    //                     if (this.allowRequests) {
+    //                         hLog('ship reader paused!', data);
+    //                     }
+    //                     this.allowRequests = false;
+    //                 } else {
+    //                     this.allowRequests = true;
+    //                     if (this.pendingRequest) {
+    //                         this.processPending();
+    //                     }
+    //                 }
+    //             });
+    //         }
+    //     }, this.queueSizeCheckInterval);
+    // }
 
     private processPending() {
         this.requestBlockRange(this.pendingRequest[0], this.pendingRequest[1]);
@@ -669,6 +769,7 @@ export default class StateReader extends HyperionWorker {
         );
     }
 
+
     private handleLostConnection() {
         this.recovery = true;
         this.ship.close(false);
@@ -680,4 +781,11 @@ export default class StateReader extends HyperionWorker {
             this.startWS();
         }, 5000);
     }
+
+    private emitCompletedSignal() {
+        process.send({
+            event: 'completed',
+            id: process.env['worker_id']
+        });
+    }
 }

workers/ws-router.ts

@@ -1,7 +1,7 @@
 import {HyperionWorker} from "./hyperionWorker";
 
 import {Server, Socket} from "socket.io";
-import {checkFilter, hLog} from "../helpers/common_functions";
+import {checkDeltaFilter, checkFilter, hLog} from "../helpers/common_functions";
 import {createServer} from "http";
 
 const greylist = ['eosio.token'];
@@ -34,21 +34,21 @@ export default class WSRouter extends HyperionWorker {
         this.ch.consume(this.q, this.onConsume.bind(this));
     }
 
+    appendIdAndEmit(event, data) {
+        this.io.emit(event, {
+            chain_id: this.manager.conn.chains[this.chain]?.chain_id,
+            ...data
+        });
+    };
 
     onIpcMessage(msg: any): void {
         switch (msg.event) {
             case 'lib_update': {
-                this.io.emit('lib_update', {
-                    chain_id: this.manager.conn.chains[this.chain]?.chain_id,
-                    ...msg.data
-                });
+                this.appendIdAndEmit('lib_update', msg.data);
                 break;
             }
             case 'fork_event': {
-                this.io.emit('fork_event', {
-                    chain_id: this.manager.conn.chains[this.chain]?.chain_id,
-                    ...msg.data
-                });
+                this.appendIdAndEmit('fork_event', msg.data);
                 break;
             }
         }
@@ -68,7 +68,17 @@ export default class WSRouter extends HyperionWorker {
         // push to plugin handlers
         this.mLoader.processStreamEvent(msg);
 
+        // process incoming live messages
         switch (msg.properties.headers.event) {
+            case 'block': {
+                // forward block events to all APIs
+                this.io.of('/').emit('block', {
+                    serverTime: Date.now(),
+                    blockNum: msg.properties.headers.blockNum,
+                    content: msg.content
+                });
+                break;
+            }
             case 'trace': {
                 const actHeader = msg.properties.headers;
                 const code = actHeader.account;
@@ -146,7 +156,7 @@ export default class WSRouter extends HyperionWorker {
             }
 
             default: {
-                console.log('Unindentified message!');
+                console.log('Unidentified message!');
                 console.log(msg);
             }
         }
@@ -206,7 +216,10 @@ export default class WSRouter extends HyperionWorker {
     addActionRequest(data, id) {
         const req = data.request;
         if (typeof req.account !== 'string') {
-            return {status: 'FAIL', reason: 'invalid request'};
+            return {
+                status: 'FAIL',
+                reason: 'invalid request'
+            };
         }
         if (greylist.indexOf(req.contract) !== -1) {
             if (req.account === '' || req.account === req.contract) {
@@ -219,6 +232,7 @@ export default class WSRouter extends HyperionWorker {
         const link = {
             type: 'action',
             relay: id,
+            reqUUID: data.reqUUID,
             client: data.client_socket,
             filters: req.filters,
             account: req.account,
@@ -230,7 +244,10 @@ export default class WSRouter extends HyperionWorker {
             if (req.account !== '') {
                 this.appendToL1Map(this.notifiedMap, req.account, link);
             } else {
-                return {status: 'FAIL', reason: 'invalid request'};
+                return {
+                    status: 'FAIL',
+                    reason: 'invalid request'
+                };
             }
         }
         this.addToClientIndex(data, id, [req.contract, req.action, req.account]);
@@ -257,6 +274,7 @@ export default class WSRouter extends HyperionWorker {
         const link = {
             type: 'delta',
             relay: id,
+            reqUUID: data.reqUUID,
             client: data.client_socket,
             filters: data.request.filters,
             payer: data.request.payer,
@@ -268,7 +286,10 @@ export default class WSRouter extends HyperionWorker {
             if (req.payer !== '' && req.payer !== '*') {
                 this.appendToL1Map(this.payerMap, req.payer, link);
             } else {
-                return {status: 'FAIL', reason: 'invalid request'};
+                return {
+                    status: 'FAIL',
+                    reason: 'invalid request'
+                };
             }
         }
         this.addToClientIndex(data, id, [req.code, req.table, req.payer]);
@@ -340,7 +361,7 @@ export default class WSRouter extends HyperionWorker {
                         if (result.status === 'OK') {
                             callback(result);
                         } else {
-                            callback(result.reason);
+                            callback(result);
                         }
                         break;
                     }
@@ -349,7 +370,7 @@ export default class WSRouter extends HyperionWorker {
                         if (result.status === 'OK') {
                             callback(result);
                         } else {
-                            callback(result.reason);
+                            callback(result);
                         }
                         break;
                     }
@@ -412,7 +433,7 @@ export default class WSRouter extends HyperionWorker {
                 });
             }
             if (allow) {
-                relay.emit('trace', {client: link.client, message: msg});
+                relay.emit('trace', {client: link.client, req: link.reqUUID, message: msg});
                 this.totalRoutedMessages++;
             }
         }
@@ -427,15 +448,15 @@ export default class WSRouter extends HyperionWorker {
             } else {
                 allow = true;
             }
-            // if (link.filters?.length > 0) {
-            //     // check filters
-            //     const _parsedMsg = JSON.parse(msg);
-            //     allow = link.filters.every(filter => {
-            //         return checkDeltaFilter(filter, _parsedMsg);
-            //     });
-            // }
+            if (link.filters?.length > 0) {
+                // check filters
+                const _parsedMsg = JSON.parse(msg);
+                allow = link.filters.every(filter => {
+                    return checkDeltaFilter(filter, _parsedMsg);
+                });
+            }
             if (allow) {
-                relay.emit('delta', {client: link.client, message: msg});
+                relay.emit('delta', {client: link.client, req: link.reqUUID, message: msg});
                 this.totalRoutedMessages++;
             }
         }