Merge pull request #114 from bensig/v3.3.5

Helpful Scripts

.gitignore

@@ -1,3 +1,5 @@
+# elastic
+elastic_pass.txt
 # Logs
 logs
 *.log
@@ -31,7 +33,6 @@ node_modules/
 
 .idea
 *.pem
-ecosystem.config.js
 package-lock.json
 connections.json
 connections.json2
@@ -75,6 +76,12 @@ modules/**/*.js.map
 addons/**/*.js
 addons/**/*.js.map
 
+scripts/hpm.js
+scripts/hpm.js.map
+
+scripts/hyp-config.js
+scripts/hyp-config.js.map
+
 plugins/.state.json
 plugins/repos
 plugins/*.js

addons/eosjs-native/eosjs-numeric.js

@@ -308,6 +308,7 @@ var KeyType;
 (function (KeyType) {
     KeyType[KeyType["k1"] = 0] = "k1";
     KeyType[KeyType["r1"] = 1] = "r1";
+    KeyType[KeyType["wa"] = 2] = "wa";
 })(KeyType = exports.KeyType || (exports.KeyType = {}));
 /** Public key data size, excluding type field */
 exports.publicKeyDataSize = 33;
@@ -459,6 +460,8 @@ function signatureToString(signature) {
         return keyToString(signature, 'K1', 'SIG_K1_');
     } else if (signature.type === KeyType.r1) {
         return keyToString(signature, 'R1', 'SIG_R1_');
+    } else if (signature.type === KeyType.wa) {
+	return keyToString(signature, 'WA', 'SIG_WA_');
     } else {
         throw new Error('unrecognized signature format');
     }

api/config/open_api.ts

@@ -4,6 +4,7 @@ export function generateOpenApiConfig(config: HyperionConfig) {
     const packageData = require('../../package');
     const health_link = `https://${config.api.server_name}/v2/health`;
     const explorer_link = `https://${config.api.server_name}/v2/explore`;
+
     let description = `
 <img height="64" src="https://eosrio.io/hyperion.png">
 ### Scalable Full History API Solution for EOSIO based blockchains
@@ -13,9 +14,15 @@ export function generateOpenApiConfig(config: HyperionConfig) {
 #### Provided by [${config.api.provider_name}](${config.api.provider_url})
 #### Health API: <a target="_blank" href="${health_link}">${health_link}</a>
 `;
-    if(config.api.enable_explorer) {
-        description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
+
+    if (config.plugins) {
+        if (config.plugins.explorer) {
+            if (config.plugins.explorer.enabled) {
+                description += `#### Integrated Explorer: <a target="_blank" href="${explorer_link}">${explorer_link}</a>`
+            }
+        }
     }
+
     return {
         routePrefix: '/v2/docs',
         exposeRoute: true,

api/helpers/cacheManager.ts

@@ -0,0 +1,86 @@
+import {HyperionConfig} from "../../interfaces/hyperionConfig";
+import {createHash} from "crypto";
+import {FastifyRequest} from "fastify";
+
+export interface CacheConfig {
+    ttl: number;
+}
+
+export interface CachedEntry {
+    data: string;
+    exp: number;
+}
+
+export class CacheManager {
+
+    v1CacheConfigs: Map<string, CacheConfig> = new Map<string, CacheConfig>();
+    v1Caches: Map<string, Map<string, CachedEntry>> = new Map<string, Map<string, CachedEntry>>();
+
+    constructor(conf: HyperionConfig) {
+        if (conf.api.v1_chain_cache) {
+            conf.api.v1_chain_cache.forEach(value => {
+                this.v1CacheConfigs.set(value.path, {
+                    ttl: value.ttl
+                });
+                this.v1Caches.set(value.path, new Map<string, CachedEntry>());
+            });
+        }
+        setInterval(() => {
+            try {
+                // remove expired entries
+                let removeCount = 0;
+                const now = Date.now();
+                this.v1Caches.forEach((pathCacheMap: Map<string, CachedEntry>, pathKey: string) => {
+                    pathCacheMap.forEach((cache: CachedEntry, entryKey: string, map: Map<string, CachedEntry>) => {
+                        if (cache.exp < now) {
+                            map.delete(entryKey);
+                            removeCount++;
+                        }
+                    });
+                });
+                if (removeCount > 0) {
+                    console.log(`${removeCount} expired cache entries removed`);
+                }
+            } catch (e) {
+                console.log(e);
+            }
+        }, 5000);
+    }
+
+    hashPayload(input: string): string {
+        return createHash('sha256').update(input).digest().toString('hex');
+    }
+
+    setCachedData(hash: string, path: string, payload: any): void {
+        if (this.v1CacheConfigs.has(path) && this.v1Caches.has(path)) {
+            this.v1Caches.get(path).set(hash, {
+                data: payload,
+                exp: this.v1CacheConfigs.get(path).ttl + Date.now()
+            });
+        }
+    }
+
+    getCachedData(request: FastifyRequest): [string | null, string, string] {
+        const urlParts = request.url.split("?");
+        const pathComponents = urlParts[0].split('/');
+        const path = pathComponents.at(-1);
+        let payload = '';
+        if (request.method === 'POST') {
+            payload = JSON.stringify(request.body);
+        } else if (request.method === 'GET') {
+            payload = request.url;
+        }
+        const hashedString = this.hashPayload(payload);
+        if (this.v1Caches.has(path)) {
+            const entry = this.v1Caches.get(path).get(hashedString);
+            if (entry && entry.exp > Date.now()) {
+                return [entry.data, hashedString, path];
+            } else {
+                return [null, hashedString, path];
+            }
+        } else {
+            return [null, hashedString, path];
+        }
+    }
+
+}

api/helpers/functions.ts

@@ -4,436 +4,456 @@ import {FastifyInstance, FastifyReply, FastifyRequest, FastifySchema, HTTPMethod
 import got from "got";
 
 export function extendResponseSchema(responseProps: any) {
-	const props = {
-		query_time_ms: {type: "number"},
-		cached: {type: "boolean"},
-		hot_only: {type: "boolean"},
-		lib: {type: "number"},
-		total: {
-			type: "object",
-			properties: {
-				value: {type: "number"},
-				relation: {type: "string"}
-			}
-		}
-	};
-	for (const p in responseProps) {
-		if (responseProps.hasOwnProperty(p)) {
-			props[p] = responseProps[p];
-		}
-	}
-	return {
-		200: {
-			type: 'object',
-			properties: props
-		}
-	};
+    const props = {
+        query_time_ms: {type: "number"},
+        cached: {type: "boolean"},
+        hot_only: {type: "boolean"},
+        lib: {type: "number"},
+        total: {
+            type: "object",
+            properties: {
+                value: {type: "number"},
+                relation: {type: "string"}
+            }
+        }
+    };
+    for (const p in responseProps) {
+        if (responseProps.hasOwnProperty(p)) {
+            props[p] = responseProps[p];
+        }
+    }
+    return {
+        200: {
+            type: 'object',
+            properties: props
+        }
+    };
 }
 
 export function extendQueryStringSchema(queryParams: any, required?: string[]) {
-	const params = {
-		limit: {
-			description: 'limit of [n] results per page',
-			type: 'integer',
-			minimum: 1
-		},
-		skip: {
-			description: 'skip [n] results',
-			type: 'integer',
-			minimum: 0
-		}
-	};
-	for (const p in queryParams) {
-		if (queryParams.hasOwnProperty(p)) {
-			params[p] = queryParams[p];
-		}
-	}
-	const schema = {
-		type: 'object',
-		properties: params
-	}
-	if (required && required.length > 0) {
-		schema["required"] = required;
-	}
-	return schema;
+    const params = {
+        limit: {
+            description: 'limit of [n] results per page',
+            type: 'integer',
+            minimum: 1
+        },
+        skip: {
+            description: 'skip [n] results',
+            type: 'integer',
+            minimum: 0
+        }
+    };
+    for (const p in queryParams) {
+        if (queryParams.hasOwnProperty(p)) {
+            params[p] = queryParams[p];
+        }
+    }
+    const schema = {
+        type: 'object',
+        properties: params
+    }
+    if (required && required.length > 0) {
+        schema["required"] = required;
+    }
+    return schema;
 }
 
 export async function getCacheByHash(redis, key, chain) {
-	const hash = createHash('sha256');
-	const query_hash = hash.update(chain + "-" + key).digest('hex');
-	return [await redis.get(query_hash), query_hash];
+    const hash = createHash('sha256');
+    const query_hash = hash.update(chain + "-" + key).digest('hex');
+    return [await redis.get(query_hash), query_hash];
 }
 
 export function mergeActionMeta(action) {
-	const name = action.act.name;
-	if (action['@' + name]) {
-		action['act']['data'] = _.merge(action['@' + name], action['act']['data']);
-		delete action['@' + name];
-	}
-	action['timestamp'] = action['@timestamp'];
-	// delete action['@timestamp'];
+    const name = action.act.name;
+    if (action['@' + name]) {
+        action['act']['data'] = _.merge(action['@' + name], action['act']['data']);
+        delete action['@' + name];
+    }
+    action['timestamp'] = action['@timestamp'];
+    // delete action['@timestamp'];
 }
 
 export function mergeDeltaMeta(delta: any) {
-	const name = delta.table;
-	if (delta["@" + name]) {
-		delta['data'] = _.merge(delta['@' + name], delta['data']);
-		delete delta['@' + name];
-	}
-	delta['timestamp'] = delta['@timestamp'];
-	delete delta['@timestamp'];
-	return delta;
+    const name = delta.table;
+    if (delta["@" + name]) {
+        delta['data'] = _.merge(delta['@' + name], delta['data']);
+        delete delta['@' + name];
+    }
+    delta['timestamp'] = delta['@timestamp'];
+    delete delta['@timestamp'];
+    return delta;
 }
 
 export function setCacheByHash(fastify, hash, response, expiration?: number) {
-	if (fastify.manager.config.api.enable_caching) {
-		let exp;
-		if (expiration) {
-			exp = expiration;
-		} else {
-			exp = fastify.manager.config.api.cache_life;
-		}
-		fastify.redis.set(hash, JSON.stringify(response), 'EX', exp).catch(console.log);
-	}
+    if (fastify.manager.config.api.enable_caching) {
+        let exp;
+        if (expiration) {
+            exp = expiration;
+        } else {
+            exp = fastify.manager.config.api.cache_life;
+        }
+        fastify.redis.set(hash, JSON.stringify(response), 'EX', exp).catch(console.log);
+    }
 }
 
 export function getRouteName(filename: string) {
-	const arr = filename.split("/");
-	return arr[arr.length - 2];
+    const arr = filename.split("/");
+    return arr[arr.length - 2];
 }
 
 export function addApiRoute(
-	fastifyInstance: FastifyInstance,
-	method: HTTPMethods | HTTPMethods[],
-	routeName: string,
-	routeBuilder: (fastify: FastifyInstance, route: string) => (request: FastifyRequest, reply: FastifyReply) => Promise<void>,
-	schema: FastifySchema
+    fastifyInstance: FastifyInstance,
+    method: HTTPMethods | HTTPMethods[],
+    routeName: string,
+    routeBuilder: (fastify: FastifyInstance, route: string) => (request: FastifyRequest, reply: FastifyReply) => Promise<void>,
+    schema: FastifySchema
 ) {
-	fastifyInstance.route({
-		url: '/' + routeName,
-		method,
-		handler: routeBuilder(fastifyInstance, routeName),
-		schema
-	});
+    fastifyInstance.route({
+        url: '/' + routeName,
+        method,
+        handler: routeBuilder(fastifyInstance, routeName),
+        schema
+    });
 }
 
 export async function getCachedResponse(server: FastifyInstance, route: string, key: any) {
-	const chain = server.manager.chain;
-	let resp, hash;
-	if (server.manager.config.api.enable_caching) {
-		[resp, hash] = await getCacheByHash(server.redis, route + JSON.stringify(key), chain);
-		if (resp) {
-			resp = JSON.parse(resp);
-			resp['cached'] = true;
-			return [resp, hash];
-		} else {
-			return [null, hash];
-		}
-	} else {
-		return [null, null];
-	}
+    const chain = server.manager.chain;
+    let resp, hash;
+    if (server.manager.config.api.enable_caching) {
+        [resp, hash] = await getCacheByHash(server.redis, route + JSON.stringify(key), chain);
+        if (resp) {
+            resp = JSON.parse(resp);
+            resp['cached'] = true;
+            return [resp, hash];
+        } else {
+            return [null, hash];
+        }
+    } else {
+        return [null, null];
+    }
 }
 
 export function getTrackTotalHits(query) {
-	let trackTotalHits: number | boolean = 10000;
-	if (query?.track) {
-		if (query.track === 'true') {
-			trackTotalHits = true;
-		} else if (query.track === 'false') {
-			trackTotalHits = false;
-		} else {
-			const parsed = parseInt(query.track, 10);
-			if (parsed > 0) {
-				trackTotalHits = parsed;
-			} else {
-				throw new Error('failed to parse track param');
-			}
-		}
-	}
-	return trackTotalHits;
+    let trackTotalHits: number | boolean = 10000;
+    if (query?.track) {
+        if (query.track === 'true') {
+            trackTotalHits = true;
+        } else if (query.track === 'false') {
+            trackTotalHits = false;
+        } else {
+            const parsed = parseInt(query.track, 10);
+            if (parsed > 0) {
+                trackTotalHits = parsed;
+            } else {
+                throw new Error('failed to parse track param');
+            }
+        }
+    }
+    return trackTotalHits;
 }
 
 function bigint2Milliseconds(input: bigint) {
-	return parseFloat((parseInt(input.toString()) / 1000000).toFixed(3));
+    return parseFloat((parseInt(input.toString()) / 1000000).toFixed(3));
 }
 
 const defaultRouteCacheMap = {
-	get_resource_usage: 3600,
-	get_creator: 3600 * 24
+    get_resource_usage: 3600,
+    get_creator: 3600 * 24
 }
 
 export async function timedQuery(
-	queryFunction: (fastify: FastifyInstance, request: FastifyRequest) => Promise<any>,
-	fastify: FastifyInstance, request: FastifyRequest, route: string): Promise<any> {
+    queryFunction: (fastify: FastifyInstance, request: FastifyRequest) => Promise<any>,
+    fastify: FastifyInstance, request: FastifyRequest, route: string): Promise<any> {
 
-	// get reference time in nanoseconds
-	const t0 = process.hrtime.bigint();
+    // get reference time in nanoseconds
+    const t0 = process.hrtime.bigint();
 
-	// check for cached data, return the response hash if caching is enabled
-	const [cachedResponse, hash] = await getCachedResponse(
-		fastify,
-		route,
-		request.method === 'POST' ? request.body : request.query
-	);
+    // check for cached data, return the response hash if caching is enabled
+    const [cachedResponse, hash] = await getCachedResponse(
+        fastify,
+        route,
+        request.method === 'POST' ? request.body : request.query
+    );
 
-	if (cachedResponse && !request.query["ignoreCache"]) {
-		// add cached query time
-		cachedResponse['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
-		return cachedResponse;
-	}
+    if (cachedResponse && !request.query["ignoreCache"]) {
+        // add cached query time
+        cachedResponse['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
+        return cachedResponse;
+    }
 
-	// call query function
-	const response = await queryFunction(fastify, request);
+    // call query function
+    const response = await queryFunction(fastify, request);
 
-	// save response to cash
-	if (hash) {
-		let EX = null;
-		if (defaultRouteCacheMap[route]) {
-			EX = defaultRouteCacheMap[route];
-		}
-		setCacheByHash(fastify, hash, response, EX);
-	}
+    // save response to cash
+    if (hash) {
+        let EX = null;
+        if (defaultRouteCacheMap[route]) {
+            EX = defaultRouteCacheMap[route];
+        }
+        setCacheByHash(fastify, hash, response, EX);
+    }
 
-	// add normal query time
-	if (response) {
-		response['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
-		return response;
-	} else {
-		return {};
-	}
+    // add normal query time
+    if (response) {
+        response['query_time_ms'] = bigint2Milliseconds(process.hrtime.bigint() - t0);
+        return response;
+    } else {
+        return {};
+    }
 }
 
 export function chainApiHandler(fastify: FastifyInstance) {
-	return async (request: FastifyRequest, reply: FastifyReply) => {
-		await handleChainApiRedirect(request, reply, fastify);
-	}
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        // check cache
+        const [cachedData, hash, path] = fastify.cacheManager.getCachedData(request);
+        if (cachedData) {
+            // console.log('cache hit:', path, hash);
+            reply.headers({'hyperion-cached': true}).send(cachedData);
+        } else {
+            // call actual request
+            const apiResponse = await handleChainApiRedirect(request, reply, fastify);
+            // console.log('cache miss:', path, hash);
+            fastify.cacheManager.setCachedData(hash, path, apiResponse);
+        }
+    }
 }
 
 export async function handleChainApiRedirect(
-	request: FastifyRequest,
-	reply: FastifyReply,
-	fastify: FastifyInstance
-) {
-	const urlParts = request.url.split("?");
-	let reqUrl = fastify.chain_api + urlParts[0];
+    request: FastifyRequest,
+    reply: FastifyReply,
+    fastify: FastifyInstance
+): Promise<string> {
+    const urlParts = request.url.split("?");
+    let reqUrl = fastify.chain_api + urlParts[0];
 
-	if (urlParts[0] === '/v1/chain/push_transaction' && fastify.push_api && fastify.push_api !== "") {
-		reqUrl = fastify.push_api + urlParts[0];
-	}
+    // const pathComponents = urlParts[0].split('/');
+    // const path = pathComponents.at(-1);
 
-	const opts = {};
 
-	if (request.method === 'POST') {
-		if (request.body) {
-			if (typeof request.body === 'string') {
-				opts['body'] = request.body;
-			} else if (typeof request.body === 'object') {
-				opts['body'] = JSON.stringify(request.body);
-			}
-		} else {
-			opts['body'] = "";
-		}
-	} else if (request.method === 'GET') {
-		opts['json'] = request.query;
-	}
+    if (urlParts[0] === '/v1/chain/push_transaction' && fastify.push_api && fastify.push_api !== "") {
+        reqUrl = fastify.push_api + urlParts[0];
+    }
 
-	try {
-		const apiResponse = await got.post(reqUrl, opts);
-		reply.headers({"Content-Type": "application/json"});
-		if (request.method === 'HEAD') {
-			reply.headers({"Content-Length": apiResponse.body.length});
-			reply.send("");
-		} else {
-			reply.send(apiResponse.body);
-		}
-	} catch (error) {
-		if (error.response) {
-			reply.status(error.response.statusCode).send(error.response.body);
-		} else {
-			console.log(error);
-			reply.status(500).send();
-		}
-		if (fastify.manager.config.api.chain_api_error_log) {
-			try {
-				if (error.response) {
-					const error_msg = JSON.parse(error.response.body).error.details[0].message;
-					console.log(`endpoint: ${request.raw.url} | status: ${error.response.statusCode} | error: ${error_msg}`);
-				} else {
-					console.log(error);
-				}
-				// if (request.req.url === '/v1/chain/push_transaction') {
-				//     const packedTrx = JSON.parse(opts['body']).packed_trx;
-				//     const trxBuffer = Buffer.from(packedTrx, 'hex');
-				//     const trxData = await fastify.eosjs.api.deserializeTransactionWithActions(trxBuffer);
-				//     console.log(trxData);
-				// }
-			} catch (e) {
-				console.log(e);
-			}
-		}
-	}
+    const opts = {};
+
+    if (request.method === 'POST') {
+        if (request.body) {
+            if (typeof request.body === 'string') {
+                opts['body'] = request.body;
+            } else if (typeof request.body === 'object') {
+                opts['body'] = JSON.stringify(request.body);
+            }
+        } else {
+            opts['body'] = "";
+        }
+    } else if (request.method === 'GET') {
+        opts['json'] = request.query;
+    }
+
+    try {
+        const apiResponse = await got.post(reqUrl, opts);
+        reply.headers({"Content-Type": "application/json"});
+        if (request.method === 'HEAD') {
+            reply.headers({"Content-Length": apiResponse.body.length});
+            reply.send("");
+            return '';
+        } else {
+            reply.send(apiResponse.body);
+            return apiResponse.body;
+        }
+    } catch (error) {
+
+        if (error.response) {
+            reply.status(error.response.statusCode).send(error.response.body);
+        } else {
+            console.log(error);
+            reply.status(500).send();
+        }
+
+        if (fastify.manager.config.api.chain_api_error_log) {
+            try {
+                if (error.response) {
+                    const error_msg = JSON.parse(error.response.body).error.details[0].message;
+                    console.log(`endpoint: ${request.raw.url} | status: ${error.response.statusCode} | error: ${error_msg}`);
+                } else {
+                    console.log(error);
+                }
+                // if (request.req.url === '/v1/chain/push_transaction') {
+                //     const packedTrx = JSON.parse(opts['body']).packed_trx;
+                //     const trxBuffer = Buffer.from(packedTrx, 'hex');
+                //     const trxData = await fastify.eosjs.api.deserializeTransactionWithActions(trxBuffer);
+                //     console.log(trxData);
+                // }
+            } catch (e) {
+                console.log(e);
+            }
+        }
+
+        return '';
+    }
 }
 
 export function addChainApiRoute(fastify: FastifyInstance, routeName, description, props?, required?) {
-	const baseSchema = {
-		description: description,
-		summary: description,
-		tags: ['chain']
-	};
-	addApiRoute(
-		fastify,
-		['GET', 'HEAD'],
-		routeName,
-		chainApiHandler,
-		{
-			...baseSchema,
-			querystring: props ? {
-				type: 'object',
-				properties: props,
-				required: required
-			} : undefined
-		}
-	);
-	addApiRoute(
-		fastify,
-		'POST',
-		routeName,
-		chainApiHandler,
-		{
-			...baseSchema,
-			body: props ? {
-				type: ['object', 'string'],
-				properties: props,
-				required: required
-			} : undefined
-		}
-	);
+    const baseSchema = {
+        description: description,
+        summary: description,
+        tags: ['chain']
+    };
+    addApiRoute(
+        fastify,
+        ['GET', 'HEAD'],
+        routeName,
+        chainApiHandler,
+        {
+            ...baseSchema,
+            querystring: props ? {
+                type: 'object',
+                properties: props,
+                required: required
+            } : undefined
+        }
+    );
+    addApiRoute(
+        fastify,
+        'POST',
+        routeName,
+        chainApiHandler,
+        {
+            ...baseSchema,
+            body: props ? {
+                type: ['object', 'string'],
+                properties: props,
+                required: required
+            } : undefined
+        }
+    );
 }
 
 export function addSharedSchemas(fastify: FastifyInstance) {
-	fastify.addSchema({
-		$id: "WholeNumber",
-		title: "Integer",
-		description: "Integer or String",
-		anyOf: [
-			{
-				type: "string",
-				pattern: "^\\d+$"
-			},
-			{
-				type: "integer"
-			}
-		],
-	});
+    fastify.addSchema({
+        $id: "WholeNumber",
+        title: "Integer",
+        description: "Integer or String",
+        anyOf: [
+            {
+                type: "string",
+                pattern: "^\\d+$"
+            },
+            {
+                type: "integer"
+            }
+        ],
+    });
 
-	fastify.addSchema({
-		$id: "Symbol",
-		type: "string",
-		description: "A symbol composed of capital letters between 1-7.",
-		pattern: "^([A-Z]{1,7})$",
-		title: "Symbol"
-	});
+    fastify.addSchema({
+        $id: "Symbol",
+        type: "string",
+        description: "A symbol composed of capital letters between 1-7.",
+        pattern: "^([A-Z]{1,7})$",
+        title: "Symbol"
+    });
 
-	fastify.addSchema({
-		$id: "Signature",
-		type: "string",
-		description: "String representation of an EOSIO compatible cryptographic signature",
-		pattern: "^SIG_([RK]1|WA)_[1-9A-HJ-NP-Za-km-z]+$",
-		title: "Signature"
-	})
+    fastify.addSchema({
+        $id: "Signature",
+        type: "string",
+        description: "String representation of an EOSIO compatible cryptographic signature",
+        pattern: "^SIG_([RK]1|WA)_[1-9A-HJ-NP-Za-km-z]+$",
+        title: "Signature"
+    })
 
-	fastify.addSchema({
-		$id: "AccountName",
-		description: "String representation of an EOSIO compatible account name",
-		"anyOf": [
-			{
-				"type": "string",
-				"description": "String representation of privileged EOSIO name type",
-				"pattern": "^(eosio[\\.][a-z1-5]{1,6})([a-j]{1})?$",
-				"title": "NamePrivileged"
-			},
-			{
-				"type": "string",
-				"description": "String representation of basic EOSIO name type, must be 12 characters and contain only a-z and 0-5",
-				"pattern": "^([a-z]{1}[a-z1-5]{11})([a-j]{1})?$",
-				"title": "NameBasic"
-			},
-			{
-				"type": "string",
-				"description": "String representation of EOSIO bid name type, 1-12 characters and only a-z and 0-5 are allowed",
-				"pattern": "^([a-z1-5]{1,12})([a-j]{1})?$",
-				"title": "NameBid"
-			},
-			{
-				"type": "string",
-				"description": "String representation of EOSIO name type",
-				"pattern": "^([a-z1-5]{1}[a-z1-5\\.]{0,10}[a-z1-5]{1})([a-j]{1})?$",
-				"title": "NameCatchAll"
-			}
-		],
-		"title": "Name"
-	});
+    fastify.addSchema({
+        $id: "AccountName",
+        description: "String representation of an EOSIO compatible account name",
+        "anyOf": [
+            {
+                "type": "string",
+                "description": "String representation of privileged EOSIO name type",
+                "pattern": "^(eosio[\\.][a-z1-5]{1,6})([a-j]{1})?$",
+                "title": "NamePrivileged"
+            },
+            {
+                "type": "string",
+                "description": "String representation of basic EOSIO name type, must be 12 characters and contain only a-z and 0-5",
+                "pattern": "^([a-z]{1}[a-z1-5]{11})([a-j]{1})?$",
+                "title": "NameBasic"
+            },
+            {
+                "type": "string",
+                "description": "String representation of EOSIO bid name type, 1-12 characters and only a-z and 0-5 are allowed",
+                "pattern": "^([a-z1-5]{1,12})([a-j]{1})?$",
+                "title": "NameBid"
+            },
+            {
+                "type": "string",
+                "description": "String representation of EOSIO name type",
+                "pattern": "^([a-z1-5]{1}[a-z1-5\\.]{0,10}[a-z1-5]{1})([a-j]{1})?$",
+                "title": "NameCatchAll"
+            }
+        ],
+        "title": "Name"
+    });
 
-	fastify.addSchema({
-		$id: "Expiration",
-		"description": "Time that transaction must be confirmed by.",
-		"type": "string",
-		"pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$",
-		"title": "DateTime"
-	});
+    fastify.addSchema({
+        $id: "Expiration",
+        "description": "Time that transaction must be confirmed by.",
+        "type": "string",
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$",
+        "title": "DateTime"
+    });
 
-	fastify.addSchema({
-		$id: "BlockExtensions",
-		"type": "array",
-		"items": {
-			"anyOf": [
-				{"type": "integer"},
-				{"type": "string"}
-			]
-		},
-		"title": "Extension"
-	})
+    fastify.addSchema({
+        $id: "BlockExtensions",
+        "type": "array",
+        "items": {
+            "anyOf": [
+                {"type": "integer"},
+                {"type": "string"}
+            ]
+        },
+        "title": "Extension"
+    })
 
-	fastify.addSchema({
-		$id: "ActionItems",
-		"type": "object",
-		"additionalProperties": false,
-		"minProperties": 5,
-		"required": [
-			"account",
-			"name",
-			"authorization",
-			"data",
-			"hex_data"
-		],
-		"properties": {
-			"account": {$ref: 'AccountName#'},
-			"name": {$ref: 'AccountName#'},
-			"authorization": {
-				"type": "array",
-				"items": {
-					"type": "object",
-					"additionalProperties": false,
-					"minProperties": 2,
-					"required": [
-						"actor",
-						"permission"
-					],
-					"properties": {
-						"actor": {$ref: 'AccountName#'},
-						"permission": {$ref: 'AccountName#'}
-					},
-					"title": "Authority"
-				}
-			},
-			"data": {
-				"type": "object",
-				"additionalProperties": true
-			},
-			"hex_data": {
-				"type": "string"
-			}
-		},
-		"title": "Action"
-	});
+    fastify.addSchema({
+        $id: "ActionItems",
+        "type": "object",
+        "additionalProperties": false,
+        "minProperties": 5,
+        "required": [
+            "account",
+            "name",
+            "authorization",
+            "data",
+            "hex_data"
+        ],
+        "properties": {
+            "account": {$ref: 'AccountName#'},
+            "name": {$ref: 'AccountName#'},
+            "authorization": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "minProperties": 2,
+                    "required": [
+                        "actor",
+                        "permission"
+                    ],
+                    "properties": {
+                        "actor": {$ref: 'AccountName#'},
+                        "permission": {$ref: 'AccountName#'}
+                    },
+                    "title": "Authority"
+                }
+            },
+            "data": {
+                "type": "object",
+                "additionalProperties": true
+            },
+            "hex_data": {
+                "type": "string"
+            }
+        },
+        "title": "Action"
+    });
 }

api/routes.ts

@@ -6,114 +6,119 @@ import autoLoad from 'fastify-autoload';
 import got from "got";
 
 function addRedirect(server: FastifyInstance, url: string, redirectTo: string) {
-	server.route({
-		url,
-		method: 'GET',
-		schema: {
-			hide: true
-		},
-		handler: async (request: FastifyRequest, reply: FastifyReply) => {
-			reply.redirect(redirectTo);
-		}
-	});
+    server.route({
+        url,
+        method: 'GET',
+        schema: {
+            hide: true
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
+            reply.redirect(redirectTo);
+        }
+    });
 }
 
 function addRoute(server: FastifyInstance, handlersPath: string, prefix: string) {
-	server.register(autoLoad, {
-		dir: join(__dirname, 'routes', handlersPath),
-		ignorePattern: /.*(handler|schema).js/,
-		dirNameRoutePrefix: false,
-		options: {prefix}
-	});
+    server.register(autoLoad, {
+        dir: join(__dirname, 'routes', handlersPath),
+        ignorePattern: /.*(handler|schema).js/,
+        dirNameRoutePrefix: false,
+        options: {prefix}
+    });
 }
 
 export function registerRoutes(server: FastifyInstance) {
 
-	// build internal map of routes
-	const routeSet = new Set<string>();
-	server.decorate('routeSet', routeSet);
-	const ignoreList = [
-		'/v2',
-		'/v2/history',
-		'/v2/state',
-		'/v1/chain/*',
-		'/v1/chain'
-	];
-	server.addHook('onRoute', opts => {
-		if (!ignoreList.includes(opts.url)) {
-			if (opts.url.startsWith('/v')) {
-				routeSet.add(opts.url);
-			}
-		}
-	});
+    // build internal map of routes
+    const routeSet = new Set<string>();
+    server.decorate('routeSet', routeSet);
+    const ignoreList = [
+        '/v2',
+        '/v2/history',
+        '/v2/state',
+        '/v1/chain/*',
+        '/v1/chain'
+    ];
+    server.addHook('onRoute', opts => {
+        if (!ignoreList.includes(opts.url)) {
+            if (opts.url.startsWith('/v')) {
+                routeSet.add(opts.url);
+            }
+        }
+    });
 
-	// Register fastify api routes
-	addRoute(server, 'v2', '/v2');
-	addRoute(server, 'v2-history', '/v2/history');
-	addRoute(server, 'v2-state', '/v2/state');
-	addRoute(server, 'v2-stats', '/v2/stats');
+    // Register fastify api routes
+    addRoute(server, 'v2', '/v2');
+    addRoute(server, 'v2-history', '/v2/history');
+    addRoute(server, 'v2-state', '/v2/state');
+    addRoute(server, 'v2-stats', '/v2/stats');
 
-	// legacy routes
-	addRoute(server, 'v1-history', '/v1/history');
-	addRoute(server, 'v1-trace', '/v1/trace_api');
+    // legacy routes
+    addRoute(server, 'v1-history', '/v1/history');
+    addRoute(server, 'v1-trace', '/v1/trace_api');
 
-	addSharedSchemas(server);
+    addSharedSchemas(server);
 
-	// chain api redirects
-	addRoute(server, 'v1-chain', '/v1/chain');
-	server.route({
-		url: '/v1/chain/*',
-		method: ["GET", "POST"],
-		schema: {
-			summary: "Wildcard chain api handler",
-			tags: ["chain"]
-		},
-		handler: async (request: FastifyRequest, reply: FastifyReply) => {
-			await handleChainApiRedirect(request, reply, server);
-		}
-	});
+    // chain api redirects
+    addRoute(server, 'v1-chain', '/v1/chain');
 
-	// /v1/node/get_supported_apis
-	server.route({
-		url: '/v1/node/get_supported_apis',
-		method: ["GET"],
-		schema: {
-			summary: "Get list of supported APIs",
-			tags: ["node"]
-		},
-		handler: async (request: FastifyRequest, reply: FastifyReply) => {
-			const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
-			if (data.apis && data.apis.length > 0) {
-				const apiSet = new Set(server.routeSet);
-				data.apis.forEach((a) => apiSet.add(a));
-				reply.send({apis: [...apiSet]});
-			} else {
-				reply.send({apis: [...server.routeSet], error: 'nodeos did not send any data'});
-			}
-		}
-	});
+	// other v1 requests
+    server.route({
+        url: '/v1/chain/*',
+        method: ["GET", "POST"],
+        schema: {
+            summary: "Wildcard chain api handler",
+            tags: ["chain"]
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
 
-	server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
-		console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
-		done();
-	});
+            console.log(request.url);
 
-	if (server.manager.config.features.streaming) {
-		// steam client lib
-		server.get(
-			'/stream-client.js',
-			{schema: {tags: ['internal']}},
-			(request: FastifyRequest, reply: FastifyReply) => {
-				const stream = createReadStream('./hyperion-stream-client.js');
-				reply.type('application/javascript').send(stream);
-			});
-	}
+            await handleChainApiRedirect(request, reply, server);
+        }
+    });
 
-	// Redirect routes to documentation
-	addRedirect(server, '/v2', '/v2/docs');
-	addRedirect(server, '/v2/history', '/v2/docs/index.html#/history');
-	addRedirect(server, '/v2/state', '/v2/docs/index.html#/state');
-	addRedirect(server, '/v1/chain', '/v2/docs/index.html#/chain');
-	addRedirect(server, '/explorer', '/v2/explore');
-	addRedirect(server, '/explore', '/v2/explore');
+    // /v1/node/get_supported_apis
+    server.route({
+        url: '/v1/node/get_supported_apis',
+        method: ["GET"],
+        schema: {
+            summary: "Get list of supported APIs",
+            tags: ["node"]
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply) => {
+            const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
+            if (data.apis && data.apis.length > 0) {
+                const apiSet = new Set(server.routeSet);
+                data.apis.forEach((a) => apiSet.add(a));
+                reply.send({apis: [...apiSet]});
+            } else {
+                reply.send({apis: [...server.routeSet], error: 'nodeos did not send any data'});
+            }
+        }
+    });
+
+    server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
+        console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
+        done();
+    });
+
+    if (server.manager.config.features.streaming) {
+        // steam client lib
+        server.get(
+            '/stream-client.js',
+            {schema: {tags: ['internal']}},
+            (request: FastifyRequest, reply: FastifyReply) => {
+                const stream = createReadStream('./hyperion-stream-client.js');
+                reply.type('application/javascript').send(stream);
+            });
+    }
+
+    // Redirect routes to documentation
+    addRedirect(server, '/v2', '/v2/docs');
+    addRedirect(server, '/v2/history', '/v2/docs/index.html#/history');
+    addRedirect(server, '/v2/state', '/v2/docs/index.html#/state');
+    addRedirect(server, '/v1/chain', '/v2/docs/index.html#/chain');
+    addRedirect(server, '/explorer', '/v2/explore');
+    addRedirect(server, '/explore', '/v2/explore');
 }

api/routes/v1-chain/get_table_rows/index.ts

@@ -36,7 +36,8 @@ export default function (fastify: FastifyInstance, opts: any, next) {
             "lower_bound": {
                 "type": "string"
             }
-        }
+        },
+        ["code", "table", "scope"]
     );
     next();
 }

api/routes/v2-history/get_actions/index.ts

@@ -36,7 +36,8 @@ export const getActionResponseSchema = {
 	"producer": {type: "string"},
 	"parent": {type: "number"},
 	"action_ordinal": {type: 'number'},
-	"creator_action_ordinal": {type: 'number'}
+	"creator_action_ordinal": {type: 'number'},
+	"signatures": {type: "array", items: {type: 'string'}}
 };
 
 export default function (fastify: FastifyInstance, opts: any, next) {

api/server.ts

@@ -8,243 +8,256 @@ import {registerPlugins} from "./plugins";
 import {AddressInfo} from "net";
 import {registerRoutes} from "./routes";
 import {generateOpenApiConfig} from "./config/open_api";
-import {createWriteStream, existsSync, mkdirSync} from "fs";
+import {createWriteStream, existsSync, mkdirSync, readFileSync} from "fs";
 import {SocketManager} from "./socketManager";
 import {HyperionModuleLoader} from "../modules/loader";
 import {extendedActions} from "./routes/v2-history/get_actions/definitions";
 import {io, Socket} from "socket.io-client";
+import {CacheManager} from "./helpers/cacheManager";
+
+import {bootstrap} from 'global-agent';
+bootstrap();
 
 class HyperionApiServer {
 
-	private conf: HyperionConfig;
-	private readonly manager: ConnectionManager;
-	mLoader: HyperionModuleLoader;
-	private readonly fastify;
-	private readonly chain: string;
-	socketManager: SocketManager;
+    private hub: Socket;
+    private readonly fastify;
+    private readonly chain: string;
+    private readonly conf: HyperionConfig;
+    private readonly manager: ConnectionManager;
+    private readonly cacheManager: CacheManager;
 
-	private hub: Socket;
+    socketManager: SocketManager;
+    mLoader: HyperionModuleLoader;
 
-	constructor() {
-		const cm = new ConfigurationModule();
-		this.conf = cm.config;
-		this.chain = this.conf.settings.chain;
-		process.title = `hyp-${this.chain}-api`;
-		this.manager = new ConnectionManager(cm);
-		this.manager.calculateServerHash();
-		this.mLoader = new HyperionModuleLoader(cm);
+    constructor() {
 
-		if (!existsSync('./logs/' + this.chain)) {
-			mkdirSync('./logs/' + this.chain, {recursive: true});
-		}
+        const package_json = JSON.parse(readFileSync('./package.json').toString());
+        hLog(`--------- Hyperion API ${package_json.version} ---------`);
 
-		const logStream = createWriteStream('./logs/' + this.chain + '/api.access.log');
+        const cm = new ConfigurationModule();
+        this.conf = cm.config;
+        this.chain = this.conf.settings.chain;
+        process.title = `hyp-${this.chain}-api`;
+        this.manager = new ConnectionManager(cm);
+        this.manager.calculateServerHash();
+        this.mLoader = new HyperionModuleLoader(cm);
+        this.cacheManager = new CacheManager(this.conf);
 
-		const loggerOpts = {
-			stream: logStream,
-			redact: ['req.headers.authorization'],
-			level: 'info',
-			prettyPrint: true,
-			serializers: {
-				res: (reply) => {
-					return {
-						statusCode: reply.statusCode
-					};
-				},
-				req: (request) => {
-					return {
-						method: request.method,
-						url: request.url,
-						ip: request.headers['x-real-ip']
-					}
-				}
-			}
-		};
+        if (!existsSync('./logs/' + this.chain)) {
+            mkdirSync('./logs/' + this.chain, {recursive: true});
+        }
 
-		this.fastify = fastify({
-			ignoreTrailingSlash: false,
-			trustProxy: true,
-			pluginTimeout: 5000,
-			logger: this.conf.api.access_log ? loggerOpts : false
-		});
+        const logStream = createWriteStream('./logs/' + this.chain + '/api.access.log');
 
-		this.fastify.decorate('manager', this.manager);
+        const loggerOpts = {
+            stream: logStream,
+            redact: ['req.headers.authorization'],
+            level: 'info',
+            prettyPrint: true,
+            serializers: {
+                res: (reply) => {
+                    return {
+                        statusCode: reply.statusCode
+                    };
+                },
+                req: (request) => {
+                    return {
+                        method: request.method,
+                        url: request.url,
+                        ip: request.headers['x-real-ip']
+                    }
+                }
+            }
+        };
 
-		// import get_actions query params from custom modules
-		const extendedActionsSet: Set<string> = new Set([...extendedActions]);
-		for (const qPrefix of this.mLoader.extendedActions) {
-			extendedActionsSet.add(qPrefix);
-		}
-		this.fastify.decorate('allowedActionQueryParamSet', extendedActionsSet);
+        this.fastify = fastify({
+            ignoreTrailingSlash: false,
+            trustProxy: true,
+            pluginTimeout: 5000,
+            logger: this.conf.api.access_log ? loggerOpts : false
+        });
 
-		// define chain api url for /v1/chain/ redirects
-		let chainApiUrl: string = this.conf.api.push_api;
-		if (chainApiUrl === null || chainApiUrl === "") {
-			chainApiUrl = this.manager.conn.chains[this.chain].http;
-		}
-		this.fastify.decorate('chain_api', chainApiUrl);
+        this.fastify.decorate('cacheManager', this.cacheManager);
 
-		// define optional push api url for /v1/chain/push_transaction
-		if (this.conf.api.push_api) {
-			this.fastify.decorate('push_api', this.conf.api.push_api);
-		}
+        this.fastify.decorate('manager', this.manager);
 
-		hLog(`Chain API URL: "${this.fastify.chain_api}" | Push API URL: "${this.fastify.push_api}"`);
+        // import get_actions query params from custom modules
+        const extendedActionsSet: Set<string> = new Set([...extendedActions]);
+        for (const qPrefix of this.mLoader.extendedActions) {
+            extendedActionsSet.add(qPrefix);
+        }
+        this.fastify.decorate('allowedActionQueryParamSet', extendedActionsSet);
 
-		const ioRedisClient = new IORedis(this.manager.conn.redis);
+        // define chain api url for /v1/chain/ redirects
+        let chainApiUrl: string = this.conf.api.chain_api;
+        if (chainApiUrl === null || chainApiUrl === "") {
+            chainApiUrl = this.manager.conn.chains[this.chain].http;
+        }
+        this.fastify.decorate('chain_api', chainApiUrl);
 
-		const pluginParams = {
-			fastify_elasticsearch: {
-				client: this.manager.elasticsearchClient
-			},
-			fastify_redis: this.manager.conn.redis,
-			fastify_eosjs: this.manager,
-		} as any;
+        // define optional push api url for /v1/chain/push_transaction
+        if (this.conf.api.push_api) {
+            this.fastify.decorate('push_api', this.conf.api.push_api);
+        }
 
-		if(!this.conf.api.disable_rate_limit) {
-			let rateLimiterWhitelist = ['127.0.0.1'];
-			if (this.conf.api.rate_limit_allow && this.conf.api.rate_limit_allow.length > 0) {
-				const tempSet = new Set<string>(['127.0.0.1', ...this.conf.api.rate_limit_allow]);
-				rateLimiterWhitelist = [...tempSet];
-			}
-			let rateLimiterRPM = 1000;
-			if (this.conf.api.rate_limit_rpm) {
-				rateLimiterRPM = this.conf.api.rate_limit_rpm;
-			}
-			pluginParams.fastify_rate_limit = {
-				max: rateLimiterRPM,
-				allowList: rateLimiterWhitelist,
-				timeWindow: '1 minute',
-				redis: ioRedisClient
-			}
-		}
+        hLog(`Chain API URL: "${this.fastify.chain_api}" | Push API URL: "${this.fastify.push_api}"`);
 
-		if (this.conf.features.streaming.enable) {
-			this.activateStreaming();
-		}
+        const ioRedisClient = new IORedis(this.manager.conn.redis);
 
-		const docsConfig = generateOpenApiConfig(this.manager.config);
-		if(docsConfig) {
-			pluginParams.fastify_swagger = docsConfig;
-		}
+        const pluginParams = {
+            fastify_elasticsearch: {
+                client: this.manager.elasticsearchClient
+            },
+            fastify_redis: this.manager.conn.redis,
+            fastify_eosjs: this.manager,
+        } as any;
 
-		registerPlugins(this.fastify, pluginParams);
+        if (!this.conf.api.disable_rate_limit) {
+            let rateLimiterWhitelist = ['127.0.0.1'];
+            if (this.conf.api.rate_limit_allow && this.conf.api.rate_limit_allow.length > 0) {
+                const tempSet = new Set<string>(['127.0.0.1', ...this.conf.api.rate_limit_allow]);
+                rateLimiterWhitelist = [...tempSet];
+            }
+            let rateLimiterRPM = 1000;
+            if (this.conf.api.rate_limit_rpm) {
+                rateLimiterRPM = this.conf.api.rate_limit_rpm;
+            }
+            pluginParams.fastify_rate_limit = {
+                max: rateLimiterRPM,
+                allowList: rateLimiterWhitelist,
+                timeWindow: '1 minute',
+                redis: ioRedisClient
+            }
+        }
 
-		this.addGenericTypeParsing();
-	}
+        if (this.conf.features.streaming.enable) {
+            this.activateStreaming();
+        }
 
-	activateStreaming() {
-		console.log('Importing stream module');
-		import('./socketManager').then((mod) => {
-			const connOpts = this.manager.conn.chains[this.chain];
+        const docsConfig = generateOpenApiConfig(this.manager.config);
+        if (docsConfig) {
+            pluginParams.fastify_swagger = docsConfig;
+        }
 
-			let _port = 57200;
-			if (connOpts.WS_ROUTER_PORT) {
-				_port = connOpts.WS_ROUTER_PORT;
-			}
+        registerPlugins(this.fastify, pluginParams);
 
-			let _host = "127.0.0.1";
-			if (connOpts.WS_ROUTER_HOST) {
-				_host = connOpts.WS_ROUTER_HOST;
-			}
+        this.addGenericTypeParsing();
+    }
 
-			this.socketManager = new mod.SocketManager(
-				this.fastify,
-				`http://${_host}:${_port}`,
-				this.manager.conn.redis
-			);
-			this.socketManager.startRelay();
-		});
-	}
+    activateStreaming() {
+        console.log('Importing stream module');
+        import('./socketManager').then((mod) => {
+            const connOpts = this.manager.conn.chains[this.chain];
 
-	private addGenericTypeParsing() {
-		this.fastify.addContentTypeParser('*', (request, payload, done) => {
-			let data = '';
-			payload.on('data', chunk => {
-				data += chunk;
-			});
-			payload.on('end', () => {
-				done(null, data);
-			});
-			payload.on('error', (err) => {
-				console.log('---- Content Parsing Error -----');
-				console.log(err);
-			});
-		});
-	}
+            let _port = 57200;
+            if (connOpts.WS_ROUTER_PORT) {
+                _port = connOpts.WS_ROUTER_PORT;
+            }
 
-	async init() {
+            let _host = "127.0.0.1";
+            if (connOpts.WS_ROUTER_HOST) {
+                _host = connOpts.WS_ROUTER_HOST;
+            }
 
-		await this.mLoader.init();
+            this.socketManager = new mod.SocketManager(
+                this.fastify,
+                `http://${_host}:${_port}`,
+                this.manager.conn.redis
+            );
+            this.socketManager.startRelay();
+        });
+    }
 
-		// add custom plugin routes
-		for (const plugin of this.mLoader.plugins) {
-			if (plugin.hasApiRoutes) {
-				plugin.addRoutes(this.fastify);
-				plugin.chainName = this.chain;
-			}
-		}
+    private addGenericTypeParsing() {
+        this.fastify.addContentTypeParser('*', (request, payload, done) => {
+            let data = '';
+            payload.on('data', chunk => {
+                data += chunk;
+            });
+            payload.on('end', () => {
+                done(null, data);
+            });
+            payload.on('error', (err) => {
+                console.log('---- Content Parsing Error -----');
+                console.log(err);
+            });
+        });
+    }
 
-		registerRoutes(this.fastify);
+    async init() {
 
-		// register documentation when ready
-		this.fastify.ready().then(async () => {
-			await this.fastify.swagger();
-		}, (err) => {
-			hLog('an error happened', err)
-		});
+        await this.mLoader.init();
 
-		try {
-			await this.fastify.listen({
-				host: this.conf.api.server_addr,
-				port: this.conf.api.server_port
-			});
-			hLog(`${this.chain} hyperion api ready and listening on port ${(this.fastify.server.address() as AddressInfo).port}`);
-			this.startHyperionHub();
-		} catch (err) {
-			hLog(err);
-			process.exit(1)
-		}
-	}
+        // add custom plugin routes
+        for (const plugin of this.mLoader.plugins) {
+            if (plugin.hasApiRoutes) {
+                hLog(`Adding routes for plugin: ${plugin.internalPluginName}`);
+                plugin.addRoutes(this.fastify);
+                plugin.chainName = this.chain;
+            }
+        }
 
-	startHyperionHub() {
-		if (this.conf.hub) {
-			const url = this.conf.hub.inform_url;
-			hLog(`Connecting API to Hyperion Hub`);
-			this.hub = io(url, {
-				query: {
-					key: this.conf.hub.publisher_key,
-					client_mode: 'false'
-				}
-			});
-			this.hub.on('connect', () => {
-				hLog(`Hyperion Hub connected!`);
-				this.emitHubApiUpdate();
-			});
-		}
-	}
+        registerRoutes(this.fastify);
 
-	private emitHubApiUpdate() {
-		this.hub.emit('hyp_info', {
-			type: 'api',
-			production: this.conf.hub.production,
-			location: this.conf.hub.location,
-			chainId: this.manager.conn.chains[this.chain].chain_id,
-			providerName: this.conf.api.provider_name,
-			explorerEnabled: this.conf.api.enable_explorer,
-			providerUrl: this.conf.api.provider_url,
-			providerLogo: this.conf.api.provider_logo,
-			chainLogo: this.conf.api.chain_logo_url,
-			chainCodename: this.chain,
-			chainName: this.conf.api.chain_name,
-			endpoint: this.conf.api.server_name,
-			features: this.conf.features,
-			filters: {
-				blacklists: this.conf.blacklists,
-				whitelists: this.conf.whitelists
-			}
-		});
-	}
+        // register documentation when ready
+        this.fastify.ready().then(async () => {
+            await this.fastify.swagger();
+        }, (err) => {
+            hLog('an error happened', err)
+        });
+
+        try {
+            await this.fastify.listen({
+                host: this.conf.api.server_addr,
+                port: this.conf.api.server_port
+            });
+            hLog(`${this.chain} hyperion api ready and listening on port ${(this.fastify.server.address() as AddressInfo).port}`);
+            this.startHyperionHub();
+        } catch (err) {
+            hLog(err);
+            process.exit(1)
+        }
+    }
+
+    startHyperionHub() {
+        if (this.conf.hub) {
+            const url = this.conf.hub.inform_url;
+            hLog(`Connecting API to Hyperion Hub`);
+            this.hub = io(url, {
+                query: {
+                    key: this.conf.hub.publisher_key,
+                    client_mode: 'false'
+                }
+            });
+            this.hub.on('connect', () => {
+                hLog(`Hyperion Hub connected!`);
+                this.emitHubApiUpdate();
+            });
+        }
+    }
+
+    private emitHubApiUpdate() {
+        this.hub.emit('hyp_info', {
+            type: 'api',
+            production: this.conf.hub.production,
+            location: this.conf.hub.location,
+            chainId: this.manager.conn.chains[this.chain].chain_id,
+            providerName: this.conf.api.provider_name,
+            explorerEnabled: this.conf.plugins.explorer?.enabled,
+            providerUrl: this.conf.api.provider_url,
+            providerLogo: this.conf.api.provider_logo,
+            chainLogo: this.conf.api.chain_logo_url,
+            chainCodename: this.chain,
+            chainName: this.conf.api.chain_name,
+            endpoint: this.conf.api.server_name,
+            features: this.conf.features,
+            filters: {
+                blacklists: this.conf.blacklists,
+                whitelists: this.conf.whitelists
+            }
+        });
+    }
 }
 
 const server = new HyperionApiServer();

chains/example.config.json

@@ -2,6 +2,7 @@
   "api": {
     "enabled": true,
     "pm2_scaling": 1,
+    "node_max_old_space_size": 1024,
     "chain_name": "EXAMPLE Chain",
     "server_addr": "127.0.0.1",
     "server_port": 7000,
@@ -21,7 +22,6 @@
       "get_trx_actions": 200
     },
     "access_log": false,
-    "enable_explorer": false,
     "chain_api_error_log": false,
     "custom_core_token": "",
     "enable_export_action": false,
@@ -29,10 +29,21 @@
     "rate_limit_rpm": 1000,
     "rate_limit_allow": [],
     "disable_tx_cache": false,
-    "tx_cache_expiration_sec": 3600
+    "tx_cache_expiration_sec": 3600,
+    "v1_chain_cache": [
+      {
+        "path": "get_block",
+        "ttl": 3000
+      },
+      {
+        "path": "get_info",
+        "ttl": 500
+      }
+    ]
   },
   "indexer": {
     "enabled": true,
+    "node_max_old_space_size": 4096,
     "start_on": 0,
     "stop_on": 0,
     "rewrite": false,
@@ -66,7 +77,8 @@
     "hot_warm_policy": false,
     "custom_policy": "",
     "bypass_index_map": false,
-    "index_partition_size": 10000000
+    "index_partition_size": 10000000,
+    "es_replicas": 0
   },
   "blacklists": {
     "actions": [],
@@ -79,21 +91,21 @@
     "root_only": false
   },
   "scaling": {
-    "readers":            1,
-    "ds_queues":          1,
-    "ds_threads":         1,
-    "ds_pool_size":       1,
-    "indexing_queues":    1,
-    "ad_idx_queues":      1,
-    "dyn_idx_queues":     1,
-    "max_autoscale":      4,
-    "batch_size":         5000,
-    "resume_trigger":     5000,
+    "readers": 1,
+    "ds_queues": 1,
+    "ds_threads": 1,
+    "ds_pool_size": 1,
+    "indexing_queues": 1,
+    "ad_idx_queues": 1,
+    "dyn_idx_queues": 1,
+    "max_autoscale": 4,
+    "batch_size": 5000,
+    "resume_trigger": 5000,
     "auto_scale_trigger": 20000,
-    "block_queue_limit":  10000,
-    "max_queue_limit":    100000,
-    "routing_mode":       "round_robin",
-    "polling_interval":   10000
+    "block_queue_limit": 10000,
+    "max_queue_limit": 100000,
+    "routing_mode": "round_robin",
+    "polling_interval": 10000
   },
   "features": {
     "streaming": {

definitions/ecosystem_settings.js

@@ -1,52 +1,51 @@
-// max js heap in MB for each indexer subprocess
-const idx_js_heap = 4096;
-
-// max js heap in MB for each api process
-const api_js_heap = 1024;
-
 function interpreterArgs(heap) {
-  const arr = ['--max-old-space-size=' + heap, '--trace-deprecation', '--trace-warnings'];
-  if (process.env.INSPECT) {
-    arr.push('--inspect');
-  }
-  return arr;
+    const arr = ['--trace-deprecation', '--trace-warnings'];
+    if (heap) {
+        arr.push('--max-old-space-size=' + heap);
+    } else {
+        arr.push('--max-old-space-size=2048');
+    }
+    if (process.env.INSPECT) {
+        arr.push('--inspect');
+    }
+    return arr;
 }
 
-function addIndexer(chainName) {
-  return {
-    script: './launcher.js',
-    name: chainName + '-indexer',
-    namespace: chainName,
-    interpreter: 'node',
-    interpreter_args: interpreterArgs(idx_js_heap),
-    autorestart: false,
-    kill_timeout: 3600,
-    watch: false,
-    time: true,
-    env: {
-      CONFIG_JSON: 'chains/' + chainName + '.config.json',
-      TRACE_LOGS: 'false',
-    },
-  };
+function addIndexer(chainName, heap) {
+    return {
+        script: './launcher.js',
+        name: chainName + '-indexer',
+        namespace: chainName,
+        interpreter: 'node',
+        interpreter_args: interpreterArgs(heap),
+        autorestart: false,
+        kill_timeout: 3600,
+        watch: false,
+        time: true,
+        env: {
+            CONFIG_JSON: 'chains/' + chainName + '.config.json',
+            TRACE_LOGS: 'false',
+        },
+    };
 }
 
-function addApiServer(chainName, threads) {
-  return {
-    script: './api/server.js',
-    name: chainName + '-api',
-    namespace: chainName,
-    node_args: interpreterArgs(api_js_heap),
-    exec_mode: 'cluster',
-    merge_logs: true,
-    instances: threads,
-    autorestart: true,
-    exp_backoff_restart_delay: 100,
-    watch: false,
-    time: true,
-    env: {
-      CONFIG_JSON: 'chains/' + chainName + '.config.json',
-    },
-  };
+function addApiServer(chainName, threads, heap) {
+    return {
+        script: './api/server.js',
+        name: chainName + '-api',
+        namespace: chainName,
+        node_args: interpreterArgs(heap),
+        exec_mode: 'cluster',
+        merge_logs: true,
+        instances: threads,
+        autorestart: true,
+        exp_backoff_restart_delay: 100,
+        watch: false,
+        time: true,
+        env: {
+            CONFIG_JSON: 'chains/' + chainName + '.config.json',
+        },
+    };
 }
 
 module.exports = {addIndexer, addApiServer};

definitions/index-templates.ts

@@ -1,9 +1,7 @@
 import {ConfigurationModule} from "../modules/config";
 
 const shards = 2;
-const replicas = 0;
 const refresh = "1s";
-let defaultLifecyclePolicy = "200G";
 
 export * from './index-lifecycle-policies';
 
@@ -15,12 +13,10 @@ const compression = "best_compression";
 const cm = new ConfigurationModule();
 const chain = cm.config.settings.chain;
 
-if (cm.config.settings.hot_warm_policy) {
-	defaultLifecyclePolicy = "hyperion-rollover";
-}
-
-if (cm.config.settings.custom_policy) {
-	defaultLifecyclePolicy = cm.config.settings.custom_policy;
+// update number of replicas if set to larger than 0
+let replicas = 0;
+if (cm.config.settings.es_replicas) {
+	replicas = cm.config.settings.es_replicas;
 }
 
 const defaultIndexSettings = {

ecosystem.config.js

@@ -0,0 +1,23 @@
+const {addApiServer, addIndexer} = require('./definitions/ecosystem_settings');
+const {readdirSync, readFileSync} = require("fs");
+const path = require('path');
+
+const apps = [];
+const chainsRoot = path.join(path.resolve(), 'chains');
+readdirSync(chainsRoot)
+    .filter(f => f.endsWith('.config.json'))
+    .forEach(value => {
+        const configFile = readFileSync(path.join(chainsRoot, value))
+        const config = JSON.parse(configFile.toString());
+        const chainName = config.settings.chain;
+        if (config.api.enabled) {
+            const apiHeap = config.api.node_max_old_space_size;
+            apps.push(addApiServer(chainName, config.api.pm2_scaling, apiHeap));
+        }
+        if (config.indexer.enabled) {
+            const indexerHeap = config.indexer.node_max_old_space_size;
+            apps.push(addIndexer(chainName, indexerHeap));
+        }
+    });
+
+module.exports = {apps};

example-ecosystem.config.js

@@ -1,14 +0,0 @@
-const {addApiServer, addIndexer} = require("./definitions/ecosystem_settings");
-
-module.exports = {
-    apps: [
-        // launch indexer
-        addIndexer('eos'),
-
-        // launch a single threaded api
-        addApiServer('eos', 1)
-
-        // example: launching a 4-node cluster of apis using pm2 cluster
-        // addApiServer('eos', 4)
-    ]
-};

hpm

@@ -0,0 +1,3 @@
+#!/bin/bash
+node ./scripts/hpm.js "$@"
+# HPM_DEBUG=1 node ./scripts/hpm.js "$@"

hyp-config

@@ -0,0 +1,2 @@
+#!/bin/bash
+node ./scripts/hyp-config.js "$@"

install.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash -
+curl -sL https://deb.nodesource.com/setup_16.x | sudo -E bash -
 sudo apt-get install -y nodejs
 sudo npm install pm2 -g
 npm install

install_env.sh

@@ -128,16 +128,31 @@ install_keys_sources() {
     echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
   fi
 
-  PPA="http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang"
-  if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
-    curl -fsSL https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc | sudo apt-key add -
-    echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang" | sudo tee /etc/apt/sources.list.d/bintray.rabbitmq.list
-    curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | sudo bash
-  fi
+  sudo apt-get install curl gnupg apt-transport-https -y
 
-  PPA="https://deb.nodesource.com/node_13.x bionic main"
+  ## Team RabbitMQ's main signing key
+  curl -1sLf "https://keys.openpgp.org/vks/v1/by-fingerprint/0A9AF2115F4687BD29803A206B73A36E6026DFCA" | sudo gpg --dearmor | sudo tee /usr/share/keyrings/com.rabbitmq.team.gpg > /dev/null
+  ## Cloudsmith: modern Erlang repository
+  curl -1sLf https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/gpg.E495BB49CC4BBE5B.key | sudo gpg --dearmor | sudo tee /usr/share/keyrings/io.cloudsmith.rabbitmq.E495BB49CC4BBE5B.gpg > /dev/null
+  ## Cloudsmith: RabbitMQ repository
+  curl -1sLf https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/gpg.9F4587F226208342.key | sudo gpg --dearmor | sudo tee /usr/share/keyrings/io.cloudsmith.rabbitmq.9F4587F226208342.gpg > /dev/null
+
+  ## Add apt repositories maintained by Team RabbitMQ
+  sudo tee /etc/apt/sources.list.d/rabbitmq.list <<EOF
+## Provides modern Erlang/OTP releases
+##
+deb [signed-by=/usr/share/keyrings/io.cloudsmith.rabbitmq.E495BB49CC4BBE5B.gpg] https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/ubuntu bionic main
+deb-src [signed-by=/usr/share/keyrings/io.cloudsmith.rabbitmq.E495BB49CC4BBE5B.gpg] https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-erlang/deb/ubuntu bionic main
+
+## Provides RabbitMQ
+##
+deb [signed-by=/usr/share/keyrings/io.cloudsmith.rabbitmq.9F4587F226208342.gpg] https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu bionic main
+deb-src [signed-by=/usr/share/keyrings/io.cloudsmith.rabbitmq.9F4587F226208342.gpg] https://dl.cloudsmith.io/public/rabbitmq/rabbitmq-server/deb/ubuntu bionic main
+EOF
+
+  PPA="https://deb.nodesource.com/node_16.x bionic main"
   if ! grep -q "^deb .*$PPA" /etc/apt/sources.list /etc/apt/sources.list.d/*; then
-    curl -sL "https://deb.nodesource.com/setup_13.x" | sudo -E bash -
+    curl -sL "https://deb.nodesource.com/setup_16.x" | sudo -E bash -
   fi
 
   sudo apt update -y
@@ -197,7 +212,17 @@ rabbit_credentials() {
 
 install_rabittmq() {
   echo -e "\n\n${COLOR_BLUE}Installing rabbit-mq...${COLOR_NC}\n\n"
-  sudo apt install -y rabbitmq-server
+
+  ## Install Erlang packages
+  sudo apt-get install -y erlang-base \
+                          erlang-asn1 erlang-crypto erlang-eldap erlang-ftp erlang-inets \
+                          erlang-mnesia erlang-os-mon erlang-parsetools erlang-public-key \
+                          erlang-runtime-tools erlang-snmp erlang-ssl \
+                          erlang-syntax-tools erlang-tftp erlang-tools erlang-xmerl
+
+  ## Install rabbitmq-server and its dependencies
+  sudo apt-get install rabbitmq-server -y --fix-missing
+
   #enable web gui
   sudo rabbitmq-plugins enable rabbitmq_management
   sudo rabbitmqctl add_vhost /hyperion
@@ -218,9 +243,11 @@ install_elastic() {
   check_ram
   if [ "$RAM" -lt 32 ]; then
     (( RAM=RAM/2 ))
-    sudo sed -ie 's/-Xms1g/-Xms'"$RAM"'g/; s/-Xmx1g/-Xmx'"$RAM"'g/' /etc/elasticsearch/jvm.options
+    sudo bash -c 'echo -e "-Xms'"$RAM"'g\n-Xmx'"$RAM"'g\n" > /etc/elasticsearch/jvm.options.d/ram.options'
+  #  sudo sed -ie 's/-Xms1g/-Xms'"$RAM"'g/; s/-Xmx1g/-Xmx'"$RAM"'g/' /etc/elasticsearch/jvm.options
   else
-    sudo sed -ie 's/-Xms1g/-Xms16g/; s/-Xmx1g/-Xmx16g/' /etc/elasticsearch/jvm.options
+    sudo bash -c 'echo -e "-Xms16g\n-Xmx16g\n" > /etc/elasticsearch/jvm.options.d/ram.options'
+  #  sudo sed -ie 's/-Xms1g/-Xms16g/; s/-Xmx1g/-Xmx16g/' /etc/elasticsearch/jvm.options
   fi
 
   sudo bash -c 'echo "xpack.security.enabled: true" >> /etc/elasticsearch/elasticsearch.yml'
@@ -243,7 +270,7 @@ install_kibana() {
 
   KIBANA_PASSWORD=$(awk <elastic_pass.txt '/PASSWORD kibana =/ {print $4}')
 
-  sudo sed -ie 's/#elasticsearch.password: "pass"/elasticsearch.password: '"$KIBANA_PASSWORD"'/; s/#elasticsearch.username: "kibana"/elasticsearch.username: "kibana"/' /etc/kibana/kibana.yml
+  sudo sed -ie 's/#elasticsearch.password: "pass"/elasticsearch.password: '"$KIBANA_PASSWORD"'/; s/elasticsearch.username: "kibana"/elasticsearch.username: "kibana"/' /etc/kibana/kibana.yml
 
   sudo systemctl enable kibana
   sudo systemctl start kibana

interfaces/hyperionConfig.ts

@@ -1,174 +1,189 @@
 import {AlertManagerOptions} from "../modules/alertsManager";
 
 export interface ScalingConfigs {
-	polling_interval: number;
-	resume_trigger: number;
-	max_queue_limit: number;
-	block_queue_limit: number;
-	routing_mode: string;
-	batch_size: number;
-	queue_limit: number;
-	readers: number;
-	ds_queues: number;
-	ds_threads: number;
-	ds_pool_size: number;
-	indexing_queues: number;
-	ad_idx_queues: number;
-	dyn_idx_queues: number;
-	max_autoscale: number;
-	auto_scale_trigger: number;
+    polling_interval: number;
+    resume_trigger: number;
+    max_queue_limit: number;
+    block_queue_limit: number;
+    routing_mode: string;
+    batch_size: number;
+    queue_limit: number;
+    readers: number;
+    ds_queues: number;
+    ds_threads: number;
+    ds_pool_size: number;
+    indexing_queues: number;
+    ad_idx_queues: number;
+    dyn_idx_queues: number;
+    max_autoscale: number;
+    auto_scale_trigger: number;
 }
 
 export interface MainSettings {
-	process_prefix?: string;
-	index_partition_size: number;
-	ignore_snapshot?: boolean;
-	ship_request_rev: string;
-	custom_policy: string;
-	bypass_index_map: boolean;
-	hot_warm_policy: boolean;
-	auto_mode_switch: boolean;
-	ds_profiling: boolean;
-	max_ws_payload_mb: number;
-	ipc_debug_rate?: number;
-	bp_monitoring?: boolean;
-	preview: boolean;
-	chain: string;
-	eosio_alias: string;
-	parser: string;
-	auto_stop: number;
-	index_version: string;
-	debug: boolean;
-	rate_monitoring: boolean;
-	bp_logs: boolean;
-	dsp_parser: boolean;
-	allow_custom_abi: boolean;
+    process_prefix?: string;
+    ignore_snapshot?: boolean;
+    ship_request_rev: string;
+    custom_policy: string;
+    bypass_index_map: boolean;
+    hot_warm_policy: boolean;
+    auto_mode_switch: boolean;
+    ds_profiling: boolean;
+    max_ws_payload_mb: number;
+    ipc_debug_rate?: number;
+    bp_monitoring?: boolean;
+    preview: boolean;
+    chain: string;
+    eosio_alias: string;
+    parser: string;
+    auto_stop: number;
+    index_version: string;
+    debug: boolean;
+    rate_monitoring: boolean;
+    bp_logs: boolean;
+    dsp_parser: boolean;
+    allow_custom_abi: boolean;
+    index_partition_size: number;
+    es_replicas: number;
 }
 
 export interface IndexerConfigs {
-	fill_state: boolean;
-	start_on: number;
-	stop_on: number;
-	rewrite: boolean;
-	purge_queues: boolean;
-	live_reader: boolean;
-	live_only_mode: boolean;
-	abi_scan_mode: boolean;
-	fetch_block: boolean;
-	fetch_traces: boolean;
-	fetch_deltas: boolean;
-	disable_reading: boolean;
-	disable_indexing: boolean;
-	process_deltas: boolean;
-	repair_mode: boolean;
-	max_inline: number;
-	disable_delta_rm?: boolean;
+    enabled?: boolean;
+    node_max_old_space_size?: number;
+    fill_state: boolean;
+    start_on: number;
+    stop_on: number;
+    rewrite: boolean;
+    purge_queues: boolean;
+    live_reader: boolean;
+    live_only_mode: boolean;
+    abi_scan_mode: boolean;
+    fetch_block: boolean;
+    fetch_traces: boolean;
+    fetch_deltas: boolean;
+    disable_reading: boolean;
+    disable_indexing: boolean;
+    process_deltas: boolean;
+    repair_mode: boolean;
+    max_inline: number;
+    disable_delta_rm?: boolean;
 }
 
 interface ApiLimits {
-	get_links?: number;
-	get_actions?: number;
-	get_blocks?: number;
-	get_created_accounts?: number;
-	get_deltas?: number;
-	get_key_accounts?: number;
-	get_proposals?: number;
-	get_tokens?: number;
-	get_transfers?: number;
-	get_voters?: number;
-	get_trx_actions?: number;
+    get_links?: number;
+    get_actions?: number;
+    get_blocks?: number;
+    get_created_accounts?: number;
+    get_deltas?: number;
+    get_key_accounts?: number;
+    get_proposals?: number;
+    get_tokens?: number;
+    get_transfers?: number;
+    get_voters?: number;
+    get_trx_actions?: number;
+}
+
+interface CachedRouteConfig {
+    path: string;
+    ttl: number
 }
 
 interface ApiConfigs {
+    enabled?: boolean;
+    pm2_scaling?: number;
+    node_max_old_space_size?: number;
     disable_rate_limit?: boolean;
-	disable_tx_cache?: boolean;
-	tx_cache_expiration_sec?: number | string;
-	rate_limit_rpm?: number;
-	rate_limit_allow?: string[];
-	custom_core_token?: string;
-	chain_api_error_log?: boolean;
-	chain_api?: string;
-	push_api?: string;
-	enable_explorer?: boolean;
-	access_log: boolean;
-	chain_name: string;
-	server_port: number;
-	server_addr: string;
-	server_name: string;
-	provider_name: string;
-	provider_url: string;
-	provider_logo: string;
-	chain_logo_url: string;
-	enable_caching: boolean,
-	cache_life: number;
-	limits: ApiLimits
+    disable_tx_cache?: boolean;
+    tx_cache_expiration_sec?: number | string;
+    rate_limit_rpm?: number;
+    rate_limit_allow?: string[];
+    custom_core_token?: string;
+    chain_api_error_log?: boolean;
+    chain_api?: string;
+    push_api?: string;
+    access_log: boolean;
+    chain_name: string;
+    server_port: number;
+    server_addr: string;
+    server_name: string;
+    provider_name: string;
+    provider_url: string;
+    provider_logo: string;
+    chain_logo_url: string;
+    enable_caching: boolean,
+    cache_life: number;
+    limits: ApiLimits,
+    v1_chain_cache?: CachedRouteConfig[]
 }
 
 interface HubLocation {
-	city: string,
-	country: string,
-	lat: number,
-	lon: number
+    city: string,
+    country: string,
+    lat: number,
+    lon: number
 }
 
 interface HyperionHubConfigs {
-	location: HubLocation;
-	production: boolean;
-	publisher_key: string;
-	inform_url: string;
+    location: HubLocation;
+    production: boolean;
+    publisher_key: string;
+    inform_url: string;
 }
 
 export interface HyperionConfig {
-	hub: HyperionHubConfigs;
-	settings: MainSettings;
-	scaling: ScalingConfigs;
-	indexer: IndexerConfigs;
+    api: ApiConfigs;
 
-	api: ApiConfigs;
+    settings: MainSettings;
 
-	blacklists: {
-		actions: string[],
-		deltas: string[]
-	};
+    hub: HyperionHubConfigs;
+    scaling: ScalingConfigs;
 
-	whitelists: {
-		max_depth: number;
-		root_only: boolean,
-		actions: string[],
-		deltas: string[]
-	};
+    indexer: IndexerConfigs;
 
-	features: {
-		streaming: {
-			enable: boolean,
-			traces: boolean,
-			deltas: boolean
-		},
-		tables: {
-			proposals: boolean,
-			accounts: boolean,
-			voters: boolean,
-			userres: boolean,
-			delband: boolean
-		},
-		index_deltas: boolean,
-		index_transfer_memo: boolean,
-		index_all_deltas: boolean,
-		deferred_trx: boolean,
-		failed_trx: boolean,
-		resource_usage: boolean,
-		resource_limits: boolean,
-	};
+    blacklists: {
+        actions: string[],
+        deltas: string[]
+    };
 
-	prefetch: {
-		read: number,
-		block: number,
-		index: number
-	};
+    whitelists: {
+        max_depth: number;
+        root_only: boolean,
+        actions: string[],
+        deltas: string[]
+    };
 
-	experimental: any;
+    features: {
+        streaming: {
+            enable: boolean,
+            traces: boolean,
+            deltas: boolean
+        },
+        tables: {
+            proposals: boolean,
+            accounts: boolean,
+            voters: boolean,
+            userres: boolean,
+            delband: boolean
+        },
+        index_deltas: boolean,
+        index_transfer_memo: boolean,
+        index_all_deltas: boolean,
+        deferred_trx: boolean,
+        failed_trx: boolean,
+        resource_usage: boolean,
+        resource_limits: boolean,
+    };
 
-	plugins: any;
+    prefetch: {
+        read: number,
+        block: number,
+        index: number
+    };
 
-	alerts: AlertManagerOptions;
+    experimental: any;
+
+    plugins: {
+        [key: string]: any;
+    };
+
+    alerts: AlertManagerOptions;
 }

interfaces/hyperionConnections.ts

@@ -20,6 +20,7 @@ interface HyperionChainData {
     name: string;
     chain_id: string;
     http: string;
+    ship: string;
     WS_ROUTER_PORT: number;
     WS_ROUTER_HOST: string;
 }

modules/loader.ts

@@ -5,189 +5,217 @@ import {HyperionConnections} from "../interfaces/hyperionConnections";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
 import {BaseParser} from "./parsers/base-parser";
 import {hLog} from "../helpers/common_functions";
-import {HyperionPlugin} from "../plugins/hyperion-plugin";
+import {HyperionPlugin, HyperionStreamHandler} from "../plugins/hyperion-plugin";
 
 export class HyperionModuleLoader {
 
-	private handledActions = new Map();
-	private handledDeltas = new Map();
-	chainMappings = new Map();
-	extendedActions: Set<string> = new Set();
-	extraMappings = [];
-	chainID;
-	private conn: HyperionConnections;
-	private config: HyperionConfig;
-	public parser: BaseParser;
-	public plugins: HyperionPlugin[];
+    private handledActions = new Map();
+    private handledDeltas = new Map();
+    private streamHandlers: HyperionStreamHandler[] = [];
 
-	constructor(private cm: ConfigurationModule) {
-		this.plugins = [];
-		this.conn = cm.connections;
-		this.config = cm.config;
-		if (!this.conn.chains[this.config.settings.chain]) {
-			console.log('Chain  ' + this.config.settings.chain + ' not defined on connections.json!');
-			process.exit(0);
-		}
-		this.chainID = this.conn.chains[this.config.settings.chain].chain_id;
-		this.loadActionHandlers();
-	}
+    chainMappings = new Map();
+    extendedActions: Set<string> = new Set();
+    extraMappings = [];
+    chainID;
+    private conn: HyperionConnections;
+    private config: HyperionConfig;
+    public parser: BaseParser;
+    public plugins: HyperionPlugin[];
 
-	async loadParser() {
-		const path = join(__dirname, 'parsers', this.config.settings.parser + "-parser");
-		const mod = (await import(path)).default;
-		this.parser = new mod(this.cm) as BaseParser;
-	}
+    constructor(private cm: ConfigurationModule) {
+        this.plugins = [];
+        this.conn = cm.connections;
+        this.config = cm.config;
+        if (!this.conn.chains[this.config.settings.chain]) {
+            console.log('Chain  ' + this.config.settings.chain + ' not defined on connections.json!');
+            process.exit(0);
+        }
+        this.chainID = this.conn.chains[this.config.settings.chain].chain_id;
+        this.loadActionHandlers();
+    }
 
-	processActionData(action) {
-		const wildcard = this.handledActions.get('*');
-		if (wildcard && wildcard.has(action.act.name)) {
-			wildcard.get(action.act.name)(action);
-		}
-		if (this.handledActions && this.handledActions.has(action.act.account)) {
-			const _c = this.handledActions.get(action.act.account);
-			if (_c.has(action.act.name)) {
-				_c.get(action.act.name)(action);
-			}
-		}
-	}
+    async loadParser() {
+        const path = join(__dirname, 'parsers', this.config.settings.parser + "-parser");
+        const mod = (await import(path)).default;
+        this.parser = new mod(this.cm) as BaseParser;
+    }
 
-	async processDeltaData(delta): Promise<void> {
-		if (this.handledDeltas.has(delta.code)) {
-			const _c = this.handledDeltas.get(delta.code);
-			if (_c.has(delta.table)) {
-				await _c.get(delta.table)(delta);
-			}
-		}
-	}
+    processActionData(action) {
+        const wildcard = this.handledActions.get('*');
+        if (wildcard && wildcard.has(action.act.name)) {
+            wildcard.get(action.act.name)(action);
+        }
+        if (this.handledActions && this.handledActions.has(action.act.account)) {
+            const _c = this.handledActions.get(action.act.account);
+            if (_c.has(action.act.name)) {
+                _c.get(action.act.name)(action);
+            }
+        }
+    }
 
-	includeActionModule(_module) {
-		if (this.handledActions.has(_module.contract)) {
-			const existing = this.handledActions.get(_module.contract);
-			existing.set(_module.action, _module.handler);
-		} else {
-			const _map = new Map();
-			_map.set(_module.action, _module.handler);
-			this.handledActions.set(_module.contract, _map);
-		}
-		if (_module.mappings) {
-			this.extraMappings.push(_module.mappings);
-		}
-		if (_module.defineQueryPrefix) {
-			this.extendedActions.add(_module.defineQueryPrefix);
-		}
-	}
+    async processDeltaData(delta): Promise<void> {
+        if (this.handledDeltas.has(delta.code)) {
+            const _c = this.handledDeltas.get(delta.code);
+            if (_c.has(delta.table)) {
+                await _c.get(delta.table)(delta);
+            }
+        }
+    }
 
-	includeDeltaModule(deltaModule) {
-		if (this.handledDeltas.has(deltaModule.contract)) {
-			const existing = this.handledDeltas.get(deltaModule.contract);
-			existing.set(deltaModule.table, deltaModule.handler);
-		} else {
-			const _map = new Map();
-			_map.set(deltaModule.table, deltaModule.handler);
-			this.handledDeltas.set(deltaModule.contract, _map);
-		}
-		if (deltaModule.mappings) {
-			this.extraMappings.push(deltaModule.mappings);
-		}
-	}
 
-	loadActionHandlers() {
-		const files = readdirSync('modules/action_data/');
-		for (const plugin of files) {
-			const _module = require(join(__dirname, 'action_data', plugin)).hyperionModule;
-			if (_module.parser_version.includes(this.config.settings.parser)) {
-				if (_module.chain === this.chainID || _module.chain === '*') {
-					const key = `${_module.contract}::${_module.action}`;
-					if (this.chainMappings.has(key)) {
-						if (this.chainMappings.get(key) === '*' && _module.chain === this.chainID) {
-							this.includeActionModule(_module);
-							this.chainMappings.set(key, _module.chain);
-						}
-					} else {
-						this.includeActionModule(_module);
-						this.chainMappings.set(key, _module.chain);
-					}
-				}
-			}
-		}
-	}
+    includeActionModule(_module) {
+        if (this.handledActions.has(_module.contract)) {
+            const existing = this.handledActions.get(_module.contract);
+            existing.set(_module.action, _module.handler);
+        } else {
+            const _map = new Map();
+            _map.set(_module.action, _module.handler);
+            this.handledActions.set(_module.contract, _map);
+        }
+        if (_module.mappings) {
+            this.extraMappings.push(_module.mappings);
+        }
+        if (_module.defineQueryPrefix) {
+            this.extendedActions.add(_module.defineQueryPrefix);
+        }
+    }
 
-	async init() {
-		try {
-			await this.loadParser();
-			await this.loadPlugins();
-		} catch (e) {
-			console.log(e);
-			process.exit(1);
-		}
-	}
+    includeDeltaModule(deltaModule) {
+        if (this.handledDeltas.has(deltaModule.contract)) {
+            const existing = this.handledDeltas.get(deltaModule.contract);
+            existing.set(deltaModule.table, deltaModule.handler);
+        } else {
+            const _map = new Map();
+            _map.set(deltaModule.table, deltaModule.handler);
+            this.handledDeltas.set(deltaModule.contract, _map);
+        }
+        if (deltaModule.mappings) {
+            this.extraMappings.push(deltaModule.mappings);
+        }
+    }
 
-	// main loader function for plugin modules
-	private async loadPlugins() {
-		const base = join(__dirname, '..', 'plugins');
-		if (!existsSync(base)) {
-			return;
-		}
-		const repos = join(base, 'repos');
-		if (!existsSync(repos)) {
-			return;
-		}
-		const state = join(base, '.state.json');
-		if (!existsSync(state)) {
-			return;
-		}
+    loadActionHandlers() {
+        const files = readdirSync('modules/action_data/');
+        for (const plugin of files) {
+            const _module = require(join(__dirname, 'action_data', plugin)).hyperionModule;
+            if (_module.parser_version.includes(this.config.settings.parser)) {
+                if (_module.chain === this.chainID || _module.chain === '*') {
+                    const key = `${_module.contract}::${_module.action}`;
+                    if (this.chainMappings.has(key)) {
+                        if (this.chainMappings.get(key) === '*' && _module.chain === this.chainID) {
+                            this.includeActionModule(_module);
+                            this.chainMappings.set(key, _module.chain);
+                        }
+                    } else {
+                        this.includeActionModule(_module);
+                        this.chainMappings.set(key, _module.chain);
+                    }
+                }
+            }
+        }
+    }
 
-		let pluginState;
-		try {
-			pluginState = JSON.parse(readFileSync(state).toString());
-		} catch (e) {
-			hLog('Failed to read plugin state');
-			return;
-		}
+    async init() {
+        try {
+            await this.loadParser();
+            await this.loadPlugins();
+        } catch (e) {
+            console.log(e);
+            process.exit(1);
+        }
+    }
 
-		for (const pluginName of Object.keys(pluginState)) {
-			if (pluginState[pluginName] && this.config.plugins[pluginName]) {
-				if (pluginState[pluginName].enabled && this.config.plugins[pluginName].enabled) {
-					try {
-						const pluginModule = (await import(join(repos, pluginName))).default;
-						const pl = new pluginModule(this.config.plugins[pluginName]);
-						if (pl.actionHandlers) {
-							this.loadPluginActionHandlers(pl.actionHandlers);
-						}
-						if (pl.deltaHandlers) {
-							this.loadPluginDeltaHandlers(pl.deltaHandlers);
-						}
-						this.plugins.push(pl);
-					} catch (e) {
-						hLog(`Plugin "${pluginName}" failed to load: ${e.message}`);
-					}
-				}
-			}
-		}
-	}
+    // main loader function for plugin modules
+    private async loadPlugins() {
+        const base = join(__dirname, '..', 'plugins');
+        if (!existsSync(base)) {
+            return;
+        }
+        const repos = join(base, 'repos');
+        if (!existsSync(repos)) {
+            return;
+        }
+        const state = join(base, '.state.json');
+        if (!existsSync(state)) {
+            return;
+        }
 
-	appendDynamicContracts(allowedDynamicContracts: Set<string>) {
-		for (const plugin of this.plugins) {
-			if (plugin.dynamicContracts) {
-				if (plugin.dynamicContracts.length > 0) {
-					plugin.dynamicContracts.forEach(value => {
-						allowedDynamicContracts.add(value);
-					});
-				}
-			}
-		}
-	}
 
-	private loadPluginActionHandlers(actionHandlers: any) {
-		for (const handler of actionHandlers) {
-			this.includeActionModule(handler);
-		}
-	}
+        let pState;
+        try {
+            const stateFile = JSON.parse(readFileSync(state).toString());
+            pState = stateFile.plugins;
+        } catch (e) {
+            hLog('Failed to read plugin state');
+            return;
+        }
 
-	private loadPluginDeltaHandlers(deltaHandlers: any) {
-		for (const handler of deltaHandlers) {
-			this.includeDeltaModule(handler);
-		}
-	}
+        for (const key in this.config.plugins) {
+            if (this.config.plugins.hasOwnProperty(key)) {
+                if (pState[key] && pState[key].enabled && this.config.plugins[key].enabled) {
+                    try {
+                        const pMod = (await import(join(repos, key))).default;
+                        const pl = new pMod(this.config.plugins[key]);
+                        if (pl.actionHandlers) {
+                            this.loadPluginActionHandlers(pl.actionHandlers);
+                        }
+                        if (pl.deltaHandlers) {
+                            this.loadPluginDeltaHandlers(pl.deltaHandlers);
+                        }
+                        if (pl.streamHandlers) {
+                            this.loadPluginStreamHandlers(pl.streamHandlers);
+                        }
+                        this.plugins.push(pl);
+                    } catch (e) {
+                        hLog(`Plugin "${key}" failed to load: ${e.message}`);
+                    }
+                }
+            }
+        }
+    }
 
+    appendDynamicContracts(allowedDynamicContracts: Set<string>) {
+        for (const plugin of this.plugins) {
+            if (plugin.dynamicContracts) {
+                if (plugin.dynamicContracts.length > 0) {
+                    plugin.dynamicContracts.forEach(value => {
+                        allowedDynamicContracts.add(value);
+                    });
+                }
+            }
+        }
+    }
+
+
+    private loadPluginActionHandlers(actionHandlers: any) {
+        for (const handler of actionHandlers) {
+            this.includeActionModule(handler);
+        }
+    }
+
+    private loadPluginDeltaHandlers(deltaHandlers: any) {
+        for (const handler of deltaHandlers) {
+            this.includeDeltaModule(handler);
+        }
+    }
+
+    private loadPluginStreamHandlers(streamHandlers: any) {
+        for (const handler of streamHandlers) {
+            this.includeStreamModule(handler);
+        }
+    }
+
+    includeStreamModule(_module) {
+        this.streamHandlers.push(_module);
+    }
+
+    processStreamEvent(msg) {
+        if (this.streamHandlers.length > 0) {
+            this.streamHandlers.forEach(sth => {
+                sth.handler(msg).catch(reason => {
+                    hLog(`Stream processing failed:`, reason);
+                });
+            });
+        }
+    }
 }

modules/parsers/base-parser.ts

@@ -10,211 +10,211 @@ import {HyperionActionAct} from "../../interfaces/hyperion-action";
 
 export abstract class BaseParser {
 
-	txDec = new TextDecoder();
-	txEnc = new TextEncoder();
-	configModule: ConfigurationModule;
-	filters: Filters;
-	private readonly chain: string;
-	flatten: boolean = false;
-	private actionReinterpretMap: Map<string, (act: HyperionActionAct) => any>;
+    txDec = new TextDecoder();
+    txEnc = new TextEncoder();
+    configModule: ConfigurationModule;
+    filters: Filters;
+    private readonly chain: string;
+    flatten: boolean = false;
+    private actionReinterpretMap: Map<string, (act: HyperionActionAct) => any>;
 
-	protected constructor(cm: ConfigurationModule) {
-		this.configModule = cm;
-		this.filters = this.configModule.filters;
-		this.chain = this.configModule.config.settings.chain;
-		this.actionReinterpretMap = new Map();
-		this.addCustomHandlers();
-	}
+    protected constructor(cm: ConfigurationModule) {
+        this.configModule = cm;
+        this.filters = this.configModule.filters;
+        this.chain = this.configModule.config.settings.chain;
+        this.actionReinterpretMap = new Map();
+        this.addCustomHandlers();
+    }
 
-	private anyFromCode(act) {
-		return this.chain + '::' + act['account'] + '::*';
-	}
+    private anyFromCode(act) {
+        return this.chain + '::' + act['account'] + '::*';
+    }
 
-	private anyFromName(act) {
-		return this.chain + '::*::' + act['name'];
-	}
+    private anyFromName(act) {
+        return this.chain + '::*::' + act['name'];
+    }
 
-	private codeActionPair(act) {
-		return this.chain + '::' + act['account'] + '::' + act['name'];
-	}
+    private codeActionPair(act) {
+        return this.chain + '::' + act['account'] + '::' + act['name'];
+    }
 
-	protected checkBlacklist(act) {
+    protected checkBlacklist(act) {
 
-		// test action blacklist for chain::code::*
-		if (this.filters.action_blacklist.has(this.anyFromCode(act))) {
-			return true;
-		}
+        // test action blacklist for chain::code::*
+        if (this.filters.action_blacklist.has(this.anyFromCode(act))) {
+            return true;
+        }
 
-		// test action blacklist for chain::*::name
-		if (this.filters.action_blacklist.has(this.anyFromName(act))) {
-			return true;
-		}
+        // test action blacklist for chain::*::name
+        if (this.filters.action_blacklist.has(this.anyFromName(act))) {
+            return true;
+        }
 
-		// test action blacklist for chain::code::name
-		return this.filters.action_blacklist.has(this.codeActionPair(act));
-	}
+        // test action blacklist for chain::code::name
+        return this.filters.action_blacklist.has(this.codeActionPair(act));
+    }
 
-	protected checkWhitelist(act) {
+    protected checkWhitelist(act) {
 
-		// test action whitelist for chain::code::*
-		if (this.filters.action_whitelist.has(this.anyFromCode(act))) {
-			return true;
-		}
+        // test action whitelist for chain::code::*
+        if (this.filters.action_whitelist.has(this.anyFromCode(act))) {
+            return true;
+        }
 
-		// test action whitelist for chain::*::name
-		if (this.filters.action_whitelist.has(this.anyFromName(act))) {
-			return true;
-		}
+        // test action whitelist for chain::*::name
+        if (this.filters.action_whitelist.has(this.anyFromName(act))) {
+            return true;
+        }
 
-		// test action whitelist for chain::code::name
-		return this.filters.action_whitelist.has(this.codeActionPair(act));
-	}
+        // test action whitelist for chain::code::name
+        return this.filters.action_whitelist.has(this.codeActionPair(act));
+    }
 
-	protected extendFirstAction(worker: DSPoolWorker, action: ActionTrace, trx_data: TrxMetadata, full_trace: any, usageIncluded) {
-		action.cpu_usage_us = trx_data.cpu_usage_us;
-		action.net_usage_words = trx_data.net_usage_words;
-		action.signatures = trx_data.signatures;
-		if (full_trace.action_traces.length > 1) {
-			action.inline_count = trx_data.inline_count - 1;
-			action.inline_filtered = trx_data.filtered;
-			if (action.inline_filtered) {
-				action.max_inline = worker.conf.indexer.max_inline;
-			}
-		} else {
-			action.inline_count = 0;
-		}
-		usageIncluded.status = true;
-	}
+    protected extendFirstAction(worker: DSPoolWorker, action: ActionTrace, trx_data: TrxMetadata, full_trace: any, usageIncluded) {
+        action.cpu_usage_us = trx_data.cpu_usage_us;
+        action.net_usage_words = trx_data.net_usage_words;
+        action.signatures = trx_data.signatures;
+        if (full_trace.action_traces.length > 1) {
+            action.inline_count = trx_data.inline_count - 1;
+            action.inline_filtered = trx_data.filtered;
+            if (action.inline_filtered) {
+                action.max_inline = worker.conf.indexer.max_inline;
+            }
+        } else {
+            action.inline_count = 0;
+        }
+        usageIncluded.status = true;
+    }
 
-	protected createSerialBuffer(data: string) {
-		return new SerialBuffer({
-			textDecoder: this.txDec,
-			textEncoder: this.txEnc,
-			array: Buffer.from(data, 'hex')
-		});
-	}
+    protected createSerialBuffer(data: string) {
+        return new SerialBuffer({
+            textDecoder: this.txDec,
+            textEncoder: this.txEnc,
+            array: Buffer.from(data, 'hex')
+        });
+    }
 
-	protected addCustomHandlers() {
-		// simple assets
-		this.actionReinterpretMap.set('*::saecreate', (act) => {
-			const _sb = this.createSerialBuffer(act.data);
-			const result = {owner: "", assetid: null};
-			result.owner = _sb.getName();
-			result.assetid = _sb.getUint64AsNumber()
-			return result;
-		});
+    protected addCustomHandlers() {
+        // simple assets
+        this.actionReinterpretMap.set('*::saecreate', (act) => {
+            const _sb = this.createSerialBuffer(act.data);
+            const result = {owner: "", assetid: null};
+            result.owner = _sb.getName();
+            result.assetid = _sb.getUint64AsNumber()
+            return result;
+        });
 
-		this.actionReinterpretMap.set('*::saetransfer', (act) => {
-			const _sb = this.createSerialBuffer(act.data);
-			const result = {from: "", to: "", assetids: [], memo: ""};
-			result.from = _sb.getName();
-			result.to = _sb.getName();
-			const len = _sb.getVaruint32();
-			for (let i = 0; i < len; i++) {
-				result.assetids.push(_sb.getUint64AsNumber());
-			}
-			result.memo = _sb.getString();
-			return result;
-		});
+        this.actionReinterpretMap.set('*::saetransfer', (act) => {
+            const _sb = this.createSerialBuffer(act.data);
+            const result = {from: "", to: "", assetids: [], memo: ""};
+            result.from = _sb.getName();
+            result.to = _sb.getName();
+            const len = _sb.getVaruint32();
+            for (let i = 0; i < len; i++) {
+                result.assetids.push(_sb.getUint64AsNumber());
+            }
+            result.memo = _sb.getString();
+            return result;
+        });
 
-		this.actionReinterpretMap.set('*::saeclaim', (act) => {
-			const _sb = this.createSerialBuffer(act.data);
-			const result = {who: "", assetids: {}};
-			result.who = _sb.getName();
-			const len = _sb.getVaruint32();
-			for (let i = 0; i < len; i++) {
-				result.assetids[_sb.getUint64AsNumber()] = _sb.getName();
-			}
-			return result;
-		});
+        this.actionReinterpretMap.set('*::saeclaim', (act) => {
+            const _sb = this.createSerialBuffer(act.data);
+            const result = {who: "", assetids: {}};
+            result.who = _sb.getName();
+            const len = _sb.getVaruint32();
+            for (let i = 0; i < len; i++) {
+                result.assetids[_sb.getUint64AsNumber()] = _sb.getName();
+            }
+            return result;
+        });
 
-		this.actionReinterpretMap.set('*::saeburn', (act) => {
-			const _sb = this.createSerialBuffer(act.data);
-			const result = {who: "", assetids: [], memo: ""};
-			result.who = _sb.getName();
-			const len = _sb.getVaruint32();
-			for (let i = 0; i < len; i++) {
-				result.assetids.push(_sb.getUint64AsNumber());
-			}
-			result.memo = _sb.getString();
-			return result;
-		});
-	}
+        this.actionReinterpretMap.set('*::saeburn', (act) => {
+            const _sb = this.createSerialBuffer(act.data);
+            const result = {who: "", assetids: [], memo: ""};
+            result.who = _sb.getName();
+            const len = _sb.getVaruint32();
+            for (let i = 0; i < len; i++) {
+                result.assetids.push(_sb.getUint64AsNumber());
+            }
+            result.memo = _sb.getString();
+            return result;
+        });
+    }
 
-	async reinterpretActionData(act: HyperionActionAct) {
-		if (this.actionReinterpretMap.has(`${act.account}::${act.name}`)) {
-			// code and action
-			return await this.actionReinterpretMap.get(`${act.account}::${act.name}`)(act);
+    async reinterpretActionData(act: HyperionActionAct) {
+        if (this.actionReinterpretMap.has(`${act.account}::${act.name}`)) {
+            // code and action
+            return await this.actionReinterpretMap.get(`${act.account}::${act.name}`)(act);
 
-		} else if (this.actionReinterpretMap.has(`*::${act.name}`)) {
-			// wildcard action
-			return await this.actionReinterpretMap.get(`*::${act.name}`)(act);
+        } else if (this.actionReinterpretMap.has(`*::${act.name}`)) {
+            // wildcard action
+            return await this.actionReinterpretMap.get(`*::${act.name}`)(act);
 
-		} else {
-			return;
-		}
-	}
+        } else {
+            return;
+        }
+    }
 
-	async deserializeActionData(worker: DSPoolWorker, action: ActionTrace, trx_data) {
-		let act = action.act;
-		const original_act = Object.assign({}, act);
-		let ds_act, error_message;
-		try {
-			ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num);
-		} catch (e) {
-			console.log(e);
-			error_message = e.message;
-		}
+    async deserializeActionData(worker: DSPoolWorker, action: ActionTrace, trx_data) {
+        let act = action.act;
+        const original_act = Object.assign({}, act);
+        let ds_act, error_message;
+        try {
+            ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num);
+        } catch (e) {
+            console.log(e);
+            error_message = e.message;
+        }
 
-		// retry failed deserialization for custom ABI maps
-		if (!ds_act) {
-			try {
-				ds_act = await this.reinterpretActionData(act);
-			} catch (e) {
-				hLog(`Failed to reinterpret action: ${act.account}::${act.name}`);
-				hLog(act.data);
-			}
-		}
+        // retry failed deserialization for custom ABI maps
+        if (!ds_act) {
+            try {
+                ds_act = await this.reinterpretActionData(act);
+            } catch (e) {
+                hLog(`Failed to reinterpret action: ${act.account}::${act.name}`);
+                hLog(act.data);
+            }
+        }
 
-		// retry with last abi before the given block_num
-		if (!ds_act) {
-			debugLog('DS Failed ->>', original_act);
-			ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num - 1);
-			debugLog('Retry with previous ABI ->>', ds_act);
-		}
+        // retry with last abi before the given block_num
+        if (!ds_act) {
+            debugLog('DS Failed ->>', original_act);
+            ds_act = await worker.common.deserializeActionAtBlockNative(worker, act, trx_data.block_num - 1);
+            debugLog('Retry with previous ABI ->>', ds_act);
+        }
 
-		if (ds_act) {
-			// save serialized data
-			action.act.data = ds_act;
-			try {
-				worker.common.attachActionExtras(worker, action);
-			} catch (e) {
-				hLog('Failed to call attachActionExtras:', e.message);
-				hLog(action?.act?.data);
-			}
-		} else {
-			action['act'] = original_act;
-			if (typeof action.act.data !== 'string') {
-				action.act.data = Buffer.from(action.act.data).toString('hex');
-			}
-			action['ds_error'] = true;
-			process.send({
-				event: 'ds_error',
-				data: {
-					type: 'action_ds_error',
-					block: trx_data.block_num,
-					account: act.account,
-					action: act.name,
-					gs: parseInt(action.receipt[1].global_sequence, 10),
-					message: error_message
-				}
-			});
-		}
-	}
+        if (ds_act) {
+            // save serialized data
+            action.act.data = ds_act;
+            try {
+                worker.common.attachActionExtras(worker, action);
+            } catch (e) {
+                hLog('Failed to call attachActionExtras:', e.message);
+                hLog(action?.act?.account, action?.act?.name, action?.act?.data);
+            }
+        } else {
+            action['act'] = original_act;
+            if (typeof action.act.data !== 'string') {
+                action.act.data = Buffer.from(action.act.data).toString('hex');
+            }
+            action['ds_error'] = true;
+            process.send({
+                event: 'ds_error',
+                data: {
+                    type: 'action_ds_error',
+                    block: trx_data.block_num,
+                    account: act.account,
+                    action: act.name,
+                    gs: parseInt(action.receipt[1].global_sequence, 10),
+                    message: error_message
+                }
+            });
+        }
+    }
 
-	abstract parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded: { status: boolean }): Promise<boolean>
+    abstract parseAction(worker: DSPoolWorker, ts, action: ActionTrace, trx_data: TrxMetadata, _actDataArray, _processedTraces: ActionTrace[], full_trace, usageIncluded: { status: boolean }): Promise<boolean>
 
-	abstract parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void>
+    abstract parseMessage(worker: MainDSWorker, messages: Message[]): Promise<void>
 
-	abstract flattenInlineActions(action_traces: any[], level?: number, trace_counter?: any, parent_index?: number): Promise<any[]>
+    abstract flattenInlineActions(action_traces: any[], level?: number, trace_counter?: any, parent_index?: number): Promise<any[]>
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperion-history",
-  "version": "3.3.4-rc7",
+  "version": "3.3.5",
   "description": "Scalable Full History API Solution for EOSIO based blockchains",
   "main": "launcher.js",
   "scripts": {
@@ -10,8 +10,7 @@
     "tsc": "tsc",
     "build": "tsc",
     "postinstall": "npm run build && npm run fix-permissions",
-    "fix-permissions": "node scripts/fix-permissions.js",
-    "plugin-manager": "node plugins/hyperion-plugin-manager.js"
+    "fix-permissions": "node scripts/fix-permissions.js"
   },
   "engines": {
     "node": "^16"
@@ -26,52 +25,52 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@elastic/elasticsearch": "7.14.1",
+    "@elastic/elasticsearch": "^7.15.0",
     "@eosrio/node-abieos": "^2.1.1",
     "@pm2/io": "^5.0.0",
     "amqplib": "^0.8.0",
-    "async": "^3.2.1",
-    "base-x": "^3.0.8",
+    "async": "^3.2.2",
+    "base-x": "^3.0.9",
     "cross-fetch": "^3.1.4",
     "eosjs": "^22.1.0",
     "fast-json-stringify": "^2.7.9",
-    "fastify": "^3.20.2",
-    "fastify-autoload": "^3.8.1",
-    "fastify-compress": "^3.6.0",
+    "fastify": "^3.24.0",
+    "fastify-autoload": "^3.9.0",
     "fastify-cors": "^6.0.2",
     "fastify-elasticsearch": "^2.0.0",
-    "fastify-formbody": "^5.1.0",
+    "fastify-formbody": "^5.2.0",
     "fastify-plugin": "^3.0.0",
     "fastify-rate-limit": "^5.6.2",
-    "fastify-redis": "^4.3.1",
-    "fastify-static": "^4.2.3",
-    "fastify-swagger": "^4.9.1",
+    "fastify-redis": "^4.3.3",
+    "fastify-swagger": "^4.12.6",
     "flatstr": "^1.0.12",
-    "got": "11.8.2",
-    "ioredis": "^4.27.9",
+    "global-agent": "^3.0.0",
+    "got": "11.8.3",
+    "ioredis": "^4.28.1",
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
     "nodemailer": "^6.6.3",
     "portfinder": "^1.0.28",
-    "socket.io": "^4.2.0",
-    "socket.io-client": "^4.2.0",
+    "socket.io": "^4.4.0",
+    "socket.io-client": "^4.4.0",
     "socket.io-redis": "^6.1.1",
-    "telegraf": "^4.4.1",
-    "typescript": "^4.3.5",
-    "ws": "^8.2.1",
-    "yargs": "^17.1.1"
+    "telegraf": "^4.4.2",
+    "typescript": "^4.5.2",
+    "ws": "^8.3.0",
+    "commander": "^8.3.0"
   },
   "devDependencies": {
     "@types/amqplib": "^0.8.2",
-    "@types/async": "^3.2.7",
-    "@types/ioredis": "^4.27.2",
-    "@types/lodash": "^4.14.172",
-    "@types/node": "^16.7.10",
+    "@types/async": "^3.2.10",
+    "@types/ioredis": "^4.28.1",
+    "@types/lodash": "^4.14.177",
+    "@types/node": "^16.11.10",
     "@types/nodemailer": "^6.4.4",
-    "@types/ws": "^7.4.7"
+    "@types/global-agent": "^2.1.1",
+    "@types/ws": "^8.2.0"
   },
   "optionalDependencies": {
-    "bufferutil": "^4.0.3",
-    "utf-8-validate": "^5.0.5"
+    "bufferutil": "^4.0.5",
+    "utf-8-validate": "^5.0.7"
   }
 }

plugins/hyperion-plugin-manager.js

@@ -72,7 +72,6 @@ async function verifyInstallation(data) {
       process.exit(0);
     }
   }
-
 }
 
 function startInstallation(data) {

plugins/hyperion-plugin.js

@@ -3,8 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.HyperionPlugin = void 0;
 class HyperionPlugin {
     constructor(config) {
+        this.internalPluginName = '';
         this.actionHandlers = [];
         this.deltaHandlers = [];
+        this.streamHandlers = [];
         this.dynamicContracts = [];
         this.hasApiRoutes = false;
         this.chainName = '';
@@ -12,6 +14,13 @@ class HyperionPlugin {
             this.baseConfig = config;
         }
     }
+    // abstract processActionData(input: any): Promise<any>;
+    initOnce() {
+        // called only once
+    }
+    initHandlerMap() {
+        return {};
+    }
 }
 exports.HyperionPlugin = HyperionPlugin;
 //# sourceMappingURL=hyperion-plugin.js.map

plugins/hyperion-plugin.js.map

@@ -1 +1 @@
-{"version":3,"file":"hyperion-plugin.js","sourceRoot":"","sources":["hyperion-plugin.ts"],"names":[],"mappings":";;;AAmBA,MAAsB,cAAc;IAQnC,YAAsB,MAAY;QAPlC,mBAAc,GAA4B,EAAE,CAAC;QAC7C,kBAAa,GAA2B,EAAE,CAAC;QAC3C,qBAAgB,GAAa,EAAE,CAAC;QAChC,iBAAY,GAAY,KAAK,CAAC;QAE9B,cAAS,GAAW,EAAE,CAAC;QAGtB,IAAI,MAAM,EAAE;YACX,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;SACzB;IACF,CAAC;CAKD;AAjBD,wCAiBC"}
+{"version":3,"file":"hyperion-plugin.js","sourceRoot":"","sources":["hyperion-plugin.ts"],"names":[],"mappings":";;;AAgCA,MAAsB,cAAc;IAYhC,YAAsB,MAAY;QAXlC,uBAAkB,GAAW,EAAE,CAAC;QAGhC,mBAAc,GAA4B,EAAE,CAAC;QAC7C,kBAAa,GAA2B,EAAE,CAAC;QAC3C,mBAAc,GAA4B,EAAE,CAAC;QAC7C,qBAAgB,GAAa,EAAE,CAAC;QAChC,iBAAY,GAAY,KAAK,CAAC;QAE9B,cAAS,GAAW,EAAE,CAAC;QAGnB,IAAI,MAAM,EAAE;YACR,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;SAC5B;IACL,CAAC;IAID,wDAAwD;IACxD,QAAQ;QACJ,mBAAmB;IACvB,CAAC;IAED,cAAc;QACV,OAAO,EAAE,CAAC;IACd,CAAC;CACJ;AA5BD,wCA4BC"}

plugins/hyperion-plugin.ts

@@ -4,34 +4,58 @@ import {HyperionDelta} from "../interfaces/hyperion-delta";
 
 
 interface HyperionActionHandler {
-	action: string;
-	contract: string;
-	mappings?: any;
-	handler: (action: HyperionAction) => Promise<void>;
+    action: string;
+    contract: string;
+    mappings?: any;
+    handler: (action: HyperionAction) => Promise<void>;
 }
 
 interface HyperionDeltaHandler {
-	table: string;
-	contract: string;
-	mappings?: any;
-	handler: (delta: HyperionDelta) => Promise<void>;
+    table: string;
+    contract: string;
+    mappings?: any;
+    handler: (delta: HyperionDelta) => Promise<void>;
+}
+
+interface HyperionStreamEvent {
+
+}
+
+export interface HyperionStreamHandler {
+    event: string;
+    code?: string;
+    account?: string;
+    name?: string;
+    table?: string;
+    handler: (streamEvent: any) => Promise<void>;
 }
 
 export abstract class HyperionPlugin {
-	actionHandlers: HyperionActionHandler[] = [];
-	deltaHandlers: HyperionDeltaHandler[] = [];
-	dynamicContracts: string[] = [];
-	hasApiRoutes: boolean = false;
-	baseConfig: any;
-	chainName: string = '';
+    internalPluginName: string = '';
+    indexerPlugin: boolean;
+    apiPlugin: boolean;
+    actionHandlers: HyperionActionHandler[] = [];
+    deltaHandlers: HyperionDeltaHandler[] = [];
+    streamHandlers: HyperionStreamHandler[] = [];
+    dynamicContracts: string[] = [];
+    hasApiRoutes: boolean = false;
+    baseConfig: any;
+    chainName: string = '';
 
-	protected constructor(config?: any) {
-		if (config) {
-			this.baseConfig = config;
-		}
-	}
+    protected constructor(config?: any) {
+        if (config) {
+            this.baseConfig = config;
+        }
+    }
 
-	abstract addRoutes(server: FastifyInstance): void;
+    abstract addRoutes(server: FastifyInstance): void;
 
-	// abstract processActionData(input: any): Promise<any>;
+    // abstract processActionData(input: any): Promise<any>;
+    initOnce() {
+        // called only once
+    }
+
+    initHandlerMap(): any {
+        return {};
+    }
 }

scripts/fix-permissions.js

@@ -5,5 +5,5 @@ if (os.type() === 'Windows_NT') {
   console.log('Skipping permission update on windows');
 } else {
   console.log('Updating permissions...');
-  execSync('chmod u+x run.sh stop.sh');
+  execSync('chmod u+x run.sh stop.sh hpm hyp-config');
 }

scripts/fix_missing_blocks/README.MD

@@ -0,0 +1,38 @@
+# Hyperion History API - Script
+
+## Fix Missing Blocks
+
+- Python script that connects to elasticsearch and identifies any missing blocks within indexes, and performs a re-indexing of those ranges to include any missing blocks.
+- Once the script completes, it will automatically re-start the indexer on live mode with re-write disabled.
+
+### Install
+
+```
+pip3 install -r requirements.txt
+```
+
+### Update the following 2 config lines in the python code.
+
+Please set the Chain to whatever your chain in chains/"chain".config.json is named
+This script supports Hyperion versions 3.3.1, 3.3.3, or 3.3.5 - which you must enter as: '3.1', '3.3', or '3.5' because that's how Charles likes it.
+
+```
+chain = "libre"  
+hyperion_version = '3.5'  
+```
+
+### RUN
+
+You should run this from your Hyperion directory by using the following command and indicating the user, pass from your elastic_pass.txt and url to your elasticsearch (default is 127.0.0.1):
+
+```
+python3 ./scripts/fix-missing-blocks.py http://elasticusername:elasticpasswordD@127.0.0.1
+```
+
+
+### TODO:
+- Add progress 
+
+
+
+

scripts/fix_missing_blocks/fix-missing-blocks.py

@@ -0,0 +1,330 @@
+from elasticsearch import Elasticsearch
+import re
+import subprocess
+from subprocess import Popen, PIPE, STDOUT
+from sh import tail
+import time
+import sys
+from pathlib import Path
+
+# set paths
+home_dir = Path.home()
+hyperionpath = Path().resolve()
+hyperionfolder = str(hyperionpath)
+
+# Variables for you to change for your system / hyperion
+chain = "libre"  # set the Chain to whatever your chains/chain.config.json is named
+hyperion_version = '3.5'  # '3.1' or '3.3' or '3.5'
+
+# Configs
+indexer_log_file = f"{home_dir}/.pm2/logs/{chain}-indexer-out.log"
+jsonpath = hyperionfolder+f"/chains/{chain}.config.json"
+indexer_stop = ["pm2", "trigger", f"{chain}-indexer", "stop"]
+indexer_start = [hyperionfolder+"/run.sh", f"{chain}-indexer"]
+tail_indexer_logs = ["tail", "-f", indexer_log_file]
+es_index = f'{chain}-block-*'
+
+# Check if chains/chain.config.json exists
+if Path(jsonpath).is_file():
+    print(jsonpath + " found")
+else:
+    raise ValueError(
+        jsonpath + " not found - please check fix-missing-blocks.py lines 14-18")
+
+# Check if chains/chain.config.json exists
+if Path(indexer_log_file).is_file():
+    print(indexer_log_file + " found.")
+else:
+    raise ValueError(
+        indexer_log_file + " not found - please check fix-missing-blocks.py lines 14-18")
+
+# Import that you stop the indexer before you start running this script
+
+if len(sys.argv) != 2:
+    raise ValueError(
+        'Please provide Elastic username, password and Elastic IP address. http://elastic:password@127.0.0.1')
+elasticnode = sys.argv[1]+":9200/"
+
+es = Elasticsearch(
+    [
+        elasticnode
+    ],
+    verify_certs=True
+)
+
+
+class bcolors:
+    HEADER = '\033[95m'
+    OKYELLOW = '\033[93m'
+    OKGREEN = '\033[92m'
+    OKBLUE = '\033[94m'
+    WARNING = '\033[91m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
+
+# For testing purposes
+testing = [{'gt': 114688000.0, 'lt': 114688999.0, 'doc_count': 989}, {'gt': 115153000.0, 'lt': 115153999.0, 'doc_count': 994}, {'gt': 116634000.0, 'lt': 116634999.0, 'doc_count': 995}, {'gt': 116712000.0, 'lt': 116712999.0, 'doc_count': 994}, {'gt': 116851000.0, 'lt': 116851999.0, 'doc_count': 995}, {'gt': 117153000.0, 'lt': 117153999.0, 'doc_count': 993}, {'gt': 117419000.0, 'lt': 117419999.0, 'doc_count': 995}, {'gt': 117443000.0, 'lt': 117443999.0, 'doc_count': 993}, {'gt': 117444000.0, 'lt': 117444999.0, 'doc_count': 992}, {'gt': 117481000.0, 'lt': 117481999.0, 'doc_count': 989}, {'gt': 117515000.0, 'lt': 117515999.0, 'doc_count': 996}, {'gt': 117547000.0, 'lt': 117547999.0, 'doc_count': 993}, {'gt': 117578000.0, 'lt': 117578999.0, 'doc_count': 993}, {'gt': 117614000.0, 'lt': 117614999.0, 'doc_count': 996}, {'gt': 117650000.0, 'lt': 117650999.0, 'doc_count': 995}, {'gt': 117683000.0, 'lt': 117683999.0, 'doc_count': 994}, {'gt': 117717000.0, 'lt': 117717999.0, 'doc_count': 995}, {'gt': 117750000.0, 'lt': 117750999.0, 'doc_count': 998}, {'gt': 117787000.0, 'lt': 117787999.0, 'doc_count': 991}, {'gt': 117821000.0, 'lt': 117821999.0, 'doc_count': 992}, {'gt': 117851000.0, 'lt': 117851999.0, 'doc_count': 991}, {'gt': 117888000.0, 'lt': 117888999.0, 'doc_count': 991}, {'gt': 117918000.0, 'lt': 117918999.0, 'doc_count': 993}, {'gt': 117949000.0, 'lt': 117949999.0, 'doc_count': 997}, {'gt': 117980000.0, 'lt': 117980999.0, 'doc_count': 994}, {'gt': 118008000.0, 'lt': 118008999.0, 'doc_count': 998}, {'gt': 118069000.0, 'lt': 118069999.0, 'doc_count': 995}, {'gt': 118107000.0, 'lt': 118107999.0, 'doc_count': 999}, {'gt': 118141000.0, 'lt': 118141999.0, 'doc_count': 991}, {'gt': 118172000.0, 'lt': 118172999.0, 'doc_count': 998}, {'gt': 118206000.0, 'lt': 118206999.0, 'doc_count': 994}, {'gt': 118212000.0, 'lt': 118212999.0, 'doc_count': 994}, {'gt': 118247000.0, 'lt': 118247999.0, 'doc_count': 996}, {'gt': 118277000.0, 'lt': 118277999.0, 'doc_count': 992}, {'gt': 118309000.0, 'lt': 118309999.0, 'doc_count': 997}, {'gt': 118341000.0, 'lt': 118341999.0, 'doc_count': 997}, {'gt': 118375000.0, 'lt': 118375999.0, 'doc_count': 986}, {'gt': 118408000.0, 'lt': 118408999.0, 'doc_count': 999}, {'gt': 118487000.0, 'lt': 118487999.0, 'doc_count': 998}, {'gt': 118518000.0, 'lt': 118518999.0, 'doc_count': 995}, {'gt': 118555000.0, 'lt': 118555999.0, 'doc_count': 995}, {'gt': 118589000.0, 'lt': 118589999.0, 'doc_count': 993}, {'gt': 118621000.0, 'lt': 118621999.0, 'doc_count': 995}, {'gt': 118625000.0, 'lt': 118625999.0, 'doc_count': 998}, {'gt': 118655000.0, 'lt': 118655999.0, 'doc_count': 995}, {'gt': 118685000.0, 'lt': 118685999.0, 'doc_count': 995}, {'gt': 118719000.0, 'lt': 118719999.0, 'doc_count': 993}, {'gt': 118753000.0, 'lt': 118753999.0, 'doc_count': 999}, {'gt': 118788000.0, 'lt': 118788999.0, 'doc_count': 996}, {'gt': 118823000.0, 'lt': 118823999.0, 'doc_count': 992}, {'gt': 118857000.0, 'lt': 118857999.0, 'doc_count': 995}, {'gt': 118864000.0, 'lt': 118864999.0, 'doc_count': 994}, {'gt': 118953000.0, 'lt': 118953999.0, 'doc_count': 998}, {'gt': 119026000.0, 'lt': 119026999.0, 'doc_count': 994}, {'gt': 119068000.0, 'lt': 119068999.0, 'doc_count': 997}, {'gt': 119107000.0, 'lt': 119107999.0, 'doc_count': 994}, {'gt': 119148000.0, 'lt': 119148999.0, 'doc_count': 997}, {'gt': 119190000.0, 'lt': 119190999.0, 'doc_count': 995}, {'gt': 119231000.0, 'lt': 119231999.0, 'doc_count': 996}, {'gt': 119273000.0, 'lt': 119273999.0, 'doc_count': 997}, {'gt': 119314000.0, 'lt': 119314999.0, 'doc_count': 999}, {'gt': 119351000.0, 'lt': 119351999.0, 'doc_count': 996}, {'gt': 119387000.0, 'lt': 119387999.0, 'doc_count': 993}, {'gt': 119427000.0, 'lt': 119427999.0, 'doc_count': 995}, {'gt': 119468000.0, 'lt': 119468999.0, 'doc_count': 996}, {'gt': 119507000.0, 'lt': 119507999.0, 'doc_count': 998}, {'gt': 119548000.0, 'lt': 119548999.0, 'doc_count': 997}, {'gt': 119567000.0, 'lt': 119567999.0, 'doc_count': 996}, {
+    'gt': 119632000.0, 'lt': 119632999.0, 'doc_count': 997}, {'gt': 119664000.0, 'lt': 119664999.0, 'doc_count': 998}, {'gt': 119679000.0, 'lt': 119679999.0, 'doc_count': 999}, {'gt': 119685000.0, 'lt': 119685999.0, 'doc_count': 998}, {'gt': 119688000.0, 'lt': 119688999.0, 'doc_count': 992}, {'gt': 119694000.0, 'lt': 119694999.0, 'doc_count': 996}, {'gt': 119695000.0, 'lt': 119695999.0, 'doc_count': 997}, {'gt': 119703000.0, 'lt': 119703999.0, 'doc_count': 998}, {'gt': 119731000.0, 'lt': 119731999.0, 'doc_count': 995}, {'gt': 119733000.0, 'lt': 119733999.0, 'doc_count': 999}, {'gt': 119737000.0, 'lt': 119737999.0, 'doc_count': 990}, {'gt': 119754000.0, 'lt': 119754999.0, 'doc_count': 999}, {'gt': 119777000.0, 'lt': 119777999.0, 'doc_count': 985}, {'gt': 119807000.0, 'lt': 119807999.0, 'doc_count': 998}, {'gt': 119810000.0, 'lt': 119810999.0, 'doc_count': 996}, {'gt': 119828000.0, 'lt': 119828999.0, 'doc_count': 997}, {'gt': 119866000.0, 'lt': 119866999.0, 'doc_count': 993}, {'gt': 120000000.0, 'lt': 120000999.0, 'doc_count': 1}, {'gt': 120088000.0, 'lt': 120088999.0, 'doc_count': 998}, {'gt': 120090000.0, 'lt': 120090999.0, 'doc_count': 992}, {'gt': 120102000.0, 'lt': 120102999.0, 'doc_count': 998}, {'gt': 120103000.0, 'lt': 120103999.0, 'doc_count': 983}, {'gt': 120109000.0, 'lt': 120109999.0, 'doc_count': 995}, {'gt': 120110000.0, 'lt': 120110999.0, 'doc_count': 997}, {'gt': 120228000.0, 'lt': 120228999.0, 'doc_count': 994}, {'gt': 120238000.0, 'lt': 120238999.0, 'doc_count': 998}, {'gt': 120239000.0, 'lt': 120239999.0, 'doc_count': 995}, {'gt': 120243000.0, 'lt': 120243999.0, 'doc_count': 997}, {'gt': 120247000.0, 'lt': 120247999.0, 'doc_count': 776}, {'gt': 120248000.0, 'lt': 120248999.0, 'doc_count': 607}, {'gt': 120249000.0, 'lt': 120249999.0, 'doc_count': 500}, {'gt': 120250000.0, 'lt': 120250999.0, 'doc_count': 979}, {'gt': 120253000.0, 'lt': 120253999.0, 'doc_count': 733}, {'gt': 120254000.0, 'lt': 120254999.0, 'doc_count': 251}, {'gt': 120255000.0, 'lt': 120255999.0, 'doc_count': 249}, {'gt': 120256000.0, 'lt': 120256999.0, 'doc_count': 251}, {'gt': 120257000.0, 'lt': 120257999.0, 'doc_count': 249}, {'gt': 120258000.0, 'lt': 120258999.0, 'doc_count': 250}, {'gt': 120259000.0, 'lt': 120259999.0, 'doc_count': 250}, {'gt': 120260000.0, 'lt': 120260999.0, 'doc_count': 250}, {'gt': 120261000.0, 'lt': 120261999.0, 'doc_count': 251}, {'gt': 120262000.0, 'lt': 120262999.0, 'doc_count': 249}, {'gt': 120263000.0, 'lt': 120263999.0, 'doc_count': 250}, {'gt': 120264000.0, 'lt': 120264999.0, 'doc_count': 250}, {'gt': 120265000.0, 'lt': 120265999.0, 'doc_count': 251}, {'gt': 120266000.0, 'lt': 120266999.0, 'doc_count': 250}, {'gt': 120267000.0, 'lt': 120267999.0, 'doc_count': 250}, {'gt': 120268000.0, 'lt': 120268999.0, 'doc_count': 250}, {'gt': 120269000.0, 'lt': 120269999.0, 'doc_count': 555}, {'gt': 125010000.0, 'lt': 125010999.0, 'doc_count': 997}, {'gt': 137832000.0, 'lt': 137832999.0, 'doc_count': 948}, {'gt': 137833000.0, 'lt': 137833999.0, 'doc_count': 997}, {'gt': 137834000.0, 'lt': 137834999.0, 'doc_count': 870}, {'gt': 137835000.0, 'lt': 137835999.0, 'doc_count': 815}, {'gt': 137837000.0, 'lt': 137837999.0, 'doc_count': 908}, {'gt': 137838000.0, 'lt': 137838999.0, 'doc_count': 900}, {'gt': 137914000.0, 'lt': 137914999.0, 'doc_count': 265}, {'gt': 137922000.0, 'lt': 137922999.0, 'doc_count': 706}, {'gt': 137941000.0, 'lt': 137941999.0, 'doc_count': 559}, {'gt': 137832000.0, 'lt': 137832999.0, 'doc_count': 948}, {'gt': 137833000.0, 'lt': 137833999.0, 'doc_count': 997}, {'gt': 137834000.0, 'lt': 137834999.0, 'doc_count': 870}, {'gt': 137835000.0, 'lt': 137835999.0, 'doc_count': 815}, {'gt': 137837000.0, 'lt': 137837999.0, 'doc_count': 908}, {'gt': 137838000.0, 'lt': 137838999.0, 'doc_count': 900}, {'gt': 137914000.0, 'lt': 137914999.0, 'doc_count': 265}, {'gt': 137922000.0, 'lt': 137922999.0, 'doc_count': 706}, {'gt': 137941000.0, 'lt': 137941999.0, 'doc_count': 559}]
+# Regex
+start_on = '.start_on.*'
+stop_on = '.stop_on.*'
+live_reader = '.live_reader.*'
+rewrite = '.rewrite.*'
+
+
+shutting_down = r'.*Shutting down master.*'
+offline = r'.*(offline or unexistent)'
+# group the last block indexed for reference.
+lastblock_indexed = r'.*Last Indexed Block:.* (\d*)'
+
+
+# No blocks being processed differs between 3.3 and 3.1
+if (hyperion_version == '3.5' or hyperion_version == '3.3'):
+    no_blocks_processed = r'.*No blocks are being processed.*'
+else:
+    no_blocks_processed = r'.*No blocks processed.*'
+
+
+def query_body(interval):
+    query_body = {
+        "aggs": {
+            "block_histogram": {
+                "histogram": {
+                    "field": "block_num",
+                    "interval": interval,
+                    "min_doc_count": 1
+                },
+                "aggs": {
+                    "max_block": {
+                        "max": {
+                            "field": "block_num"
+                        }
+                    }
+                }
+            }
+        },
+        "size": 0,
+        "query": {
+            "match_all": {}
+        }
+    }
+    return query_body
+
+
+def query_body2(gte, lte, interval):
+    query = {
+        "aggs": {
+            "block_histogram": {
+                "histogram": {
+                    "field": "block_num",
+                    "interval": interval,
+                    "min_doc_count": 1
+                },
+                "aggs": {
+                    "max_block": {
+                        "max": {
+                            "field": "block_num"
+                        }
+                    }
+                }
+            }
+        },
+        "size": 0,
+        "query": {
+            "bool": {
+                "must": [
+                    {
+                        "range": {
+                            "block_num": {
+                                "gte": gte,
+                                "lte": lte
+                            }
+                        }
+                    }
+                ]
+            }
+        }
+    }
+    return query
+
+
+# Search and return buckets
+def get_buckets(interval, query):
+    query = query(interval)
+    result = es.search(index=es_index, body=query)
+    buckets = result['aggregations']['block_histogram']['buckets']
+    return buckets
+
+# Find buckets with missing blocks
+
+
+def buckets_missing(bucketlist, count1, count2):
+    buckets_final = []
+    bucketdict = {}
+    for num, bucket in enumerate(bucketlist):
+        # Create copy of dict and call it new
+        new = bucketdict.copy()
+        key = bucket['key']
+        doc_count = bucket['doc_count']
+        # Check if count is less than
+        if num == 0 and doc_count < count1:
+            new.update({'key': key, 'doc_count': doc_count})
+            buckets_final.append(new)
+        # If not first bucket but has less than count2
+        elif doc_count < count2 and num != 0:
+            new.update({'key': key, 'doc_count': doc_count})
+            buckets_final.append(new)
+    return buckets_final
+
+
+# Find and replace text
+def replacement(Path, text, subs, flags=0):
+    with open(jsonpath, "r+") as f1:
+        contents = f1.read()
+        pattern = re.compile(text, flags)
+        contents = pattern.sub(subs, contents)
+        f1.seek(0)
+        f1.truncate()
+        f1.write(contents)
+
+
+# Build text patterns
+def buildReplacementText(key, value):
+    subs = '"'+str(key)+'": '+str(value)+','
+    return subs
+
+
+def MagicFuzz(missing):
+    for num, missed in enumerate(missing):
+        gt = str(int(missed['gt']))
+        lt = str(int(missed['lt']))
+        print(bcolors.OKYELLOW,
+              f"{'='*100}\n1.Building replacement text for config file ", bcolors.ENDC)
+        subs_start = buildReplacementText('start_on', gt)
+        subs_stop = buildReplacementText('stop_on', lt)
+        live_reader_k = buildReplacementText('live_reader', 'false')
+        rewrite_k = buildReplacementText('rewrite', 'true')
+        print(bcolors.OKYELLOW,
+              f"{'='*100}\n2.Updating the config file ", bcolors.ENDC)
+        replacement(jsonpath, start_on, subs_start)
+        replacement(jsonpath, stop_on, subs_stop)
+        replacement(jsonpath, live_reader, live_reader_k)
+        replacement(jsonpath, rewrite, rewrite_k)
+        print(bcolors.OKYELLOW,
+              f"{'='*100}\n3.Config file is now updated for: ", gt, "-", lt, bcolors.ENDC)
+        print(bcolors.OKYELLOW,
+              f"{'='*100}\n4.Running Re-indexing for: ", gt, "-", lt, bcolors.ENDC)
+        startRewrite(gt, lt)
+        time.sleep(0.1)
+    print(bcolors.OKYELLOW,
+          f"{'='*100}\nRe-indexing Complete ", gt, "-", lt, bcolors.ENDC)
+    exit
+
+
+def spinning_cursor(run):
+    while run == True:
+        for cursor in '\\|/-':
+            time.sleep(0.1)
+            # Use '\r' to move cursor back to line beginning
+            # Or use '\b' to erase the last character
+            sys.stdout.write('\r{}'.format(cursor))
+            # Force Python to write data into terminal.
+            sys.stdout.flush()
+
+
+def startRewrite(gt, lt):
+    run = Popen(indexer_start, stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL)
+    time.sleep(10)
+    logtail = Popen(tail_indexer_logs, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
+    completedcount = 0
+    for linenum, line in enumerate(logtail.stdout):
+        # spinning_cursor(True)
+        # decode line from bytes-object
+        line = line.decode('utf-8')
+        print(line)
+        completed = re.match(no_blocks_processed, line)
+        shutdown = re.match(shutting_down, line)
+        lastblock = re.match(lastblock_indexed, line)
+        if not completedcount > 0 and completed:
+            print(bcolors.OKYELLOW,
+                  f"{'='*100}\n5.Shutting down indexer: ", bcolors.ENDC)
+            subprocess.run(indexer_stop, stdout=subprocess.DEVNULL,
+                           stderr=subprocess.DEVNULL)
+            # Increment completed count as regex shows up multipletimes. To not try and send shutdown command multiple times.
+            completedcount += 1
+            # Try without sleep and break
+            # Sleep process to wait for indexer to shutdown
+            # time.sleep(15)
+            # break
+        elif lastblock:
+            # Indicates indexer has already shutdown, break out of loop and restart process.
+            print(bcolors.OKYELLOW,
+                  f"{'='*100}\n6.Indexer shutdown: ", bcolors.ENDC)
+            break
+        elif shutdown:
+            # If indexer already shutdown, break out of loop and restart process.
+            print(bcolors.OKYELLOW,
+                  f"{'='*100}\n6.Indexer shutdown: ", bcolors.ENDC)
+            # spinning_cursor(False)
+            break
+        else:
+            continue
+
+
+# Create greater, less than lists.
+def CreateGtLT(missing):
+    lte = 10000000
+    buckets_final = []
+    bucketdict = {}
+    for num, bucket in enumerate(missing):
+        gte = bucket['key']
+        lte = gte+lte
+        result = es.search(index=es_index, body=query_body2(gte, lte, 1000))
+        buckets = result['aggregations']['block_histogram']['buckets']
+        # Search for missing buckets with less than 1000
+        missing = buckets_missing(buckets, 1000, 1000)
+        for bucket in missing:
+            new = bucketdict.copy()
+            doc_count = bucket['doc_count']
+            gt = bucket['key']
+            lt = gt+1000
+            new.update({'gt': gt, 'lt': lt, 'doc_count': doc_count})
+            buckets_final.append(new)
+    return buckets_final
+
+
+def start_indexer_live():
+    print('Live indexer startig again')
+    subs_start = buildReplacementText('start_on', 0)
+    subs_stop = buildReplacementText('stop_on', 0)
+    live_reader_k = buildReplacementText('live_reader', 'true')
+    rewrite_k = buildReplacementText('rewrite', 'false')
+    replacement(jsonpath, start_on, subs_start)
+    replacement(jsonpath, stop_on, subs_stop)
+    replacement(jsonpath, live_reader, live_reader_k)
+    replacement(jsonpath, rewrite, rewrite_k)
+    run = Popen(indexer_start, stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL)
+
+
+if __name__ == '__main__':
+    # Shutdown indexer if running
+    subprocess.run(indexer_stop, stdout=subprocess.DEVNULL,
+                   stderr=subprocess.DEVNULL)
+    print(bcolors.OKYELLOW,
+          f"{'='*100}\nSearching for missing Blocks ", bcolors.ENDC)
+    # Search for buckets using histogram interval
+    buckets = get_buckets(10000000, query_body)
+    # Get buckets with missing blocks, first count is 9999999 to account for bucket 0-9999999.
+    missing = buckets_missing(buckets, 9999999, 10000000)
+    # Redefine search and set interval to 1000
+    gt_lt_list = CreateGtLT(missing)
+    print(bcolors.OKYELLOW, f"{'='*100}\nCompleted Search ", bcolors.ENDC)
+    # For testing purposes use  MagicFuzz(testing)
+    MagicFuzz(gt_lt_list)
+    # Once rewrite is completed go back to live reading
+    start_indexer_live()

scripts/fix_missing_blocks/requirements.txt

@@ -0,0 +1,2 @@
+elasticsearch==7.15.2
+sh==1.14.3

scripts/hpm.js

@@ -0,0 +1,405 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const path_1 = __importDefault(require("path"));
+const promises_1 = require("fs/promises");
+const fs_1 = __importStar(require("fs"));
+const child_process_1 = require("child_process");
+const crypto_1 = __importDefault(require("crypto"));
+const package_json = JSON.parse((0, fs_1.readFileSync)('./package.json').toString());
+const debug = process.env.HPM_DEBUG;
+const program = new commander_1.Command();
+const pluginDirAbsolutePath = path_1.default.join(path_1.default.resolve(), 'plugins', 'repos');
+const pluginStatePath = path_1.default.join(path_1.default.resolve(), 'plugins', '.state.json');
+let stateJson;
+const ignoredDirsForHash = ['.git', 'node_modules'];
+program.version(package_json.version);
+function checkUrl(value) {
+    if (value.startsWith('https://') || value.startsWith('git@')) {
+        return value;
+    }
+    else {
+        throw new commander_1.InvalidArgumentError('Invalid url.');
+    }
+}
+function throwAndQuit(message) {
+    console.error('⚠ >>', message);
+    process.exit(1);
+}
+function getCurrentBranch(dir) {
+    const gitCmd = `(cd ${dir} && git rev-parse --abbrev-ref HEAD)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return (0, child_process_1.execSync)(gitCmd).toString().trim();
+}
+function checkoutBranch(dir, branchName) {
+    const gitCmd = `(cd ${dir} && git checkout ${branchName})`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return (0, child_process_1.execSync)(gitCmd).toString();
+}
+function gitPull(dir) {
+    const gitCmd = `(cd ${dir} && git pull)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return (0, child_process_1.execSync)(gitCmd).toString();
+}
+function gitReset(dir) {
+    const gitCmd = `(cd ${dir} && git reset --hard)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return (0, child_process_1.execSync)(gitCmd).toString();
+}
+async function buildPlugin(name, flags) {
+    const pluginDir = path_1.default.join(pluginDirAbsolutePath, name);
+    if (!(0, fs_1.existsSync)(pluginDir)) {
+        throwAndQuit(`plugin ${name} not found!`);
+    }
+    const currentBranch = getCurrentBranch(pluginDir);
+    console.log(`-------- ⚒ ${name} @ ${currentBranch} -----------`);
+    if (!flags.skipInstall) {
+        // run npm install
+        await new Promise(resolve => {
+            const npmInstall = (0, child_process_1.spawn)('npm', ['install'], {
+                cwd: pluginDir,
+                stdio: 'inherit',
+            });
+            npmInstall.on('close', (code) => {
+                if (code !== 0) {
+                    console.log(`process exited with code ${code}`);
+                    process.exit(code);
+                }
+                resolve(true);
+            });
+        });
+    }
+    // typescript build
+    await new Promise(resolve => {
+        const npmInstall = (0, child_process_1.spawn)('npm', ['run', 'build'], {
+            cwd: pluginDir,
+            stdio: 'inherit',
+        });
+        npmInstall.on('close', (code) => {
+            if (code !== 0) {
+                console.log(`process exited with code ${code}`);
+                process.exit(code);
+            }
+            resolve(true);
+        });
+    });
+    console.log('-------------------------------');
+    // save build date
+    if (stateJson) {
+        if (!stateJson.plugins[name]) {
+            stateJson.plugins[name] = {};
+        }
+        stateJson.plugins[name].last_build_date = new Date().toISOString();
+        const hash = await hashItem(pluginDir);
+        console.log(`SHA1 Hash: ${hash}`);
+        stateJson.plugins[name].hash = hash;
+        stateJson.plugins[name].branch = currentBranch;
+        saveState();
+    }
+}
+async function verifyInstalledPlugin(pluginName, options) {
+    const dir = path_1.default.join(pluginDirAbsolutePath, pluginName);
+    const dirExists = (0, fs_1.existsSync)(dir);
+    if (dirExists) {
+        if (options.branch && options.branch !== '') {
+            const currentBranch = getCurrentBranch(dir);
+            if (currentBranch === options.branch) {
+                console.log('✅ Plugin already installed, branch check OK');
+                process.exit(0);
+            }
+            else {
+                // checkout new branch
+                const checkoutResults = checkoutBranch(dir, options.branch);
+                console.log(checkoutResults);
+                console.log('done!');
+                process.exit(0);
+            }
+        }
+        throwAndQuit('plugin already installed');
+    }
+}
+async function hashItem(item) {
+    if (fs_1.default.lstatSync(item).isDirectory()) {
+        const dir = fs_1.default.readdirSync(item);
+        const sha = crypto_1.default.createHash('sha1');
+        for (const file of dir) {
+            if (ignoredDirsForHash.includes(file) && fs_1.default.lstatSync(path_1.default.join(item, file)).isDirectory()) {
+                sha.update('');
+            }
+            else {
+                const hash = await hashItem(path_1.default.join(item, file));
+                sha.update(hash.toString());
+            }
+        }
+        return sha.digest('hex');
+    }
+    else {
+        const sha = crypto_1.default.createHash('sha1');
+        const reader = fs_1.default.createReadStream(item);
+        return new Promise(resolve => {
+            reader.on('data', chunk => {
+                sha.update(chunk);
+            });
+            reader.on('close', () => {
+                resolve(sha.digest('hex'));
+            });
+        });
+    }
+}
+async function verifyInstallation(installPath, pluginName, opts) {
+    console.log('Verifying integrity...');
+    const hash = await hashItem(installPath);
+    console.log(`SHA1 Hash: ${hash}`);
+    if (stateJson && stateJson.plugins) {
+        stateJson.plugins[pluginName].hash = hash;
+        saveState();
+    }
+    if (!opts.skipValidation) {
+        // if (hash !== data['integrityHash']) {
+        //     console.log('WARNING! Integrity hash dos not match!');
+        //     console.log(`Expected:\t${data['integrityHash']}`);
+        //     console.log(`Current:\t${hash}`);
+        //     process.exit(0);
+        // }
+        const installedFiles = fs_1.default.readdirSync(installPath);
+        if (!installedFiles.includes('package.json')) {
+            console.log('package.json not found!');
+            process.exit(0);
+        }
+    }
+}
+async function clonePluginRepo(pluginName, options) {
+    const dir = path_1.default.join(pluginDirAbsolutePath, pluginName);
+    await new Promise((resolve, reject) => {
+        const gitClone = (0, child_process_1.spawn)('git', ['clone', options.repository, dir]);
+        gitClone.stdout.on('data', chunk => {
+            console.log(`->> ${chunk}`);
+        });
+        gitClone.stderr.on('data', chunk => {
+            console.log(`->> ${chunk}`);
+        });
+        gitClone.on('close', (code) => {
+            if (code !== 0) {
+                console.log(`process exited with code ${code}`);
+            }
+            verifyInstallation(dir, pluginName, options)
+                .then(() => {
+                resolve();
+            })
+                .catch((err) => {
+                throwAndQuit(err.message);
+                reject();
+            });
+        });
+    });
+}
+function init() {
+    const check = (0, fs_1.existsSync)(pluginStatePath);
+    if (check) {
+        try {
+            stateJson = JSON.parse((0, fs_1.readFileSync)(pluginStatePath).toString());
+        }
+        catch (e) {
+            console.log(e);
+        }
+    }
+    else {
+        console.log('Creating new state file...');
+        stateJson = {
+            update_ts: Date.now(),
+            hyperion_version: package_json.version,
+            plugins: {}
+        };
+        (0, fs_1.writeFileSync)(pluginStatePath, JSON.stringify(stateJson, null, 2));
+    }
+}
+async function installPlugin(plugin, options) {
+    console.log(`Plugin Name: ${plugin}`);
+    console.log(`Repository Url: ${options.repository}`);
+    if (options.branch) {
+        console.log(`Branch Name: ${options.branch}`);
+    }
+    // check installation
+    await verifyInstalledPlugin(plugin, options);
+    if (stateJson && stateJson.plugins) {
+        stateJson.plugins[plugin] = {};
+    }
+    else {
+        throwAndQuit('fatal error on state');
+    }
+    // clone repository
+    await clonePluginRepo(plugin, options);
+    // checkout branch
+    if (options.branch) {
+        const dir = path_1.default.join(pluginDirAbsolutePath, plugin);
+        const results = checkoutBranch(dir, options.branch);
+        stateJson.plugins[plugin].branch = options.branch;
+        console.log(results);
+    }
+    // build plugin
+    await buildPlugin(plugin, {
+        skipInstall: false
+    });
+}
+async function uninstall(plugin) {
+    const pluginDir = path_1.default.join(pluginDirAbsolutePath, plugin);
+    try {
+        if (!(0, fs_1.existsSync)(pluginDir)) {
+            throwAndQuit(`plugin ${plugin} not found!`);
+        }
+        console.log(`Removing ${pluginDir}...`);
+        await (0, promises_1.rm)(pluginDir, {
+            recursive: true
+        });
+        if (stateJson && stateJson.plugins) {
+            delete stateJson.plugins[plugin];
+            saveState();
+        }
+        console.log(`${plugin} removed!`);
+    }
+    catch (e) {
+        throwAndQuit(e.message);
+    }
+}
+async function listPlugins() {
+    const dirs = await (0, promises_1.readdir)(pluginDirAbsolutePath);
+    const results = [];
+    for (const dir of dirs) {
+        try {
+            const packageJsonPath = path_1.default.join(pluginDirAbsolutePath, dir, 'package.json');
+            const package_json = JSON.parse((0, fs_1.readFileSync)(packageJsonPath).toString());
+            const { name, version, description } = package_json;
+            // console.table([dir, name, version, description].join('\t'));
+            let pluginState = false;
+            let branch = '';
+            if (stateJson.plugins && stateJson.plugins[dir]) {
+                if (stateJson.plugins[dir].enabled) {
+                    pluginState = true;
+                }
+                branch = stateJson.plugins[dir].branch;
+            }
+            results.push({
+                alias: dir,
+                enabled: pluginState,
+                plugin_name: name,
+                version,
+                branch,
+                description
+            });
+        }
+        catch (e) {
+            console.log(e.message);
+        }
+    }
+    results.sort((a, b) => b.ative - a.ative);
+    console.table(results);
+}
+function saveState() {
+    stateJson.update_ts = Date.now();
+    (0, fs_1.writeFileSync)(pluginStatePath, JSON.stringify(stateJson, null, 2));
+}
+async function setPluginState(pluginName, newState) {
+    if (stateJson) {
+        if (stateJson.plugins[pluginName]) {
+            stateJson.plugins[pluginName].enabled = newState;
+        }
+        else {
+            stateJson.plugins[pluginName] = {
+                enabled: newState
+            };
+        }
+    }
+    saveState();
+    console.log(`[${pluginName}] plugin ${newState ? 'enabled' : 'disabled'} successfully!`);
+}
+function enablePlugin(pluginName) {
+    return setPluginState(pluginName, true);
+}
+function disablePlugin(pluginName) {
+    return setPluginState(pluginName, false);
+}
+function printState() {
+    console.log(JSON.stringify(stateJson, null, 2));
+}
+async function buildAllPlugins(flags) {
+    const names = (0, fs_1.readdirSync)(pluginDirAbsolutePath);
+    for (const name of names) {
+        await buildPlugin(name, flags);
+    }
+}
+async function updatePlugin(name) {
+    const pluginDir = path_1.default.join(pluginDirAbsolutePath, name);
+    const resetStatus = gitReset(pluginDir);
+    console.log(resetStatus);
+    const pullStatus = gitPull(pluginDir);
+    console.log(pullStatus);
+    await buildPlugin(name, {});
+}
+(() => {
+    init();
+    program.description('Command-line utility to manage Hyperion plugins\nMade with ♥  by EOS Rio');
+    program.command('install <plugin>').alias('i')
+        .description('install new plugin')
+        .option('-r, --repository <url>', 'custom repository url', checkUrl)
+        .option('-b, --branch <branch>', 'checkout specific branch')
+        .option('-s, --skip-validation', 'skip plugin integrity validation')
+        .action(installPlugin);
+    program.command('enable <plugin>')
+        .description('enable plugin')
+        .action(enablePlugin);
+    program.command('disable <plugin>')
+        .description('disable plugin')
+        .action(disablePlugin);
+    program.command('state')
+        .description('print current state file')
+        .action(printState);
+    program.command('list').alias('ls')
+        .description('list installed plugins')
+        .action(listPlugins);
+    program.command('build <plugin>').alias('b')
+        .description('build a single plugin')
+        .option('-s, --skip-install', 'skip "npm install" step')
+        .action(buildPlugin);
+    program.command('update <plugin>')
+        .description('update a single plugin')
+        .action(updatePlugin);
+    program.command('build-all')
+        .description('build all plugins')
+        .option('-s, --skip-install', 'skip "npm install" step')
+        .action(buildAllPlugins);
+    program.command('uninstall <plugin>').alias('rm')
+        .description('uninstall plugin')
+        .action(uninstall);
+    program.parse();
+})();
+//# sourceMappingURL=hpm.js.map

scripts/hpm.ts

@@ -0,0 +1,432 @@
+import {Command, InvalidArgumentError} from 'commander';
+import path from 'path'
+import {readdir, rm} from "fs/promises";
+import fs, {existsSync, readdirSync, readFileSync, writeFileSync} from "fs";
+import {execSync, spawn} from "child_process";
+import crypto from "crypto";
+
+const package_json = JSON.parse(readFileSync('./package.json').toString());
+const debug = process.env.HPM_DEBUG;
+const program = new Command();
+const pluginDirAbsolutePath = path.join(path.resolve(), 'plugins', 'repos');
+const pluginStatePath = path.join(path.resolve(), 'plugins', '.state.json');
+
+let stateJson;
+
+const ignoredDirsForHash = ['.git', 'node_modules'];
+
+program.version(package_json.version);
+
+function checkUrl(value) {
+    if (value.startsWith('https://') || value.startsWith('git@')) {
+        return value;
+    } else {
+        throw new InvalidArgumentError('Invalid url.');
+    }
+}
+
+function throwAndQuit(message: string) {
+    console.error('⚠ >>', message);
+    process.exit(1);
+}
+
+function getCurrentBranch(dir): string {
+    const gitCmd = `(cd ${dir} && git rev-parse --abbrev-ref HEAD)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return execSync(gitCmd).toString().trim();
+}
+
+function checkoutBranch(dir, branchName): string {
+    const gitCmd = `(cd ${dir} && git checkout ${branchName})`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return execSync(gitCmd).toString();
+}
+
+function gitPull(dir): string {
+    const gitCmd = `(cd ${dir} && git pull)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return execSync(gitCmd).toString();
+}
+
+function gitReset(dir): string {
+    const gitCmd = `(cd ${dir} && git reset --hard)`;
+    if (debug) {
+        console.log(` >> ${gitCmd}`);
+    }
+    return execSync(gitCmd).toString();
+}
+
+async function buildPlugin(name, flags) {
+
+    const pluginDir = path.join(pluginDirAbsolutePath, name);
+
+    if (!existsSync(pluginDir)) {
+        throwAndQuit(`plugin ${name} not found!`);
+    }
+
+    const currentBranch = getCurrentBranch(pluginDir);
+
+    console.log(`-------- ⚒ ${name} @ ${currentBranch} -----------`);
+
+    if (!flags.skipInstall) {
+        // run npm install
+        await new Promise(resolve => {
+            const npmInstall = spawn('npm', ['install'], {
+                cwd: pluginDir,
+                stdio: 'inherit',
+            });
+            npmInstall.on('close', (code) => {
+                if (code !== 0) {
+                    console.log(`process exited with code ${code}`);
+                    process.exit(code);
+                }
+                resolve(true);
+            });
+        });
+    }
+
+    // typescript build
+    await new Promise(resolve => {
+        const npmInstall = spawn('npm', ['run', 'build'], {
+            cwd: pluginDir,
+            stdio: 'inherit',
+        });
+        npmInstall.on('close', (code) => {
+            if (code !== 0) {
+                console.log(`process exited with code ${code}`);
+                process.exit(code);
+            }
+            resolve(true);
+        });
+    });
+
+    console.log('-------------------------------');
+
+    // save build date
+    if (stateJson) {
+
+        if (!stateJson.plugins[name]) {
+            stateJson.plugins[name] = {};
+        }
+
+        stateJson.plugins[name].last_build_date = new Date().toISOString();
+        const hash = await hashItem(pluginDir);
+        console.log(`SHA1 Hash: ${hash}`);
+        stateJson.plugins[name].hash = hash;
+        stateJson.plugins[name].branch = currentBranch;
+        saveState();
+    }
+}
+
+async function verifyInstalledPlugin(pluginName: string, options: any): Promise<void> {
+    const dir = path.join(pluginDirAbsolutePath, pluginName);
+    const dirExists = existsSync(dir);
+    if (dirExists) {
+        if (options.branch && options.branch !== '') {
+            const currentBranch = getCurrentBranch(dir);
+            if (currentBranch === options.branch) {
+                console.log('✅ Plugin already installed, branch check OK');
+                process.exit(0);
+            } else {
+                // checkout new branch
+                const checkoutResults = checkoutBranch(dir, options.branch);
+                console.log(checkoutResults);
+                console.log('done!');
+                process.exit(0);
+            }
+        }
+        throwAndQuit('plugin already installed');
+    }
+}
+
+async function hashItem(item) {
+    if (fs.lstatSync(item).isDirectory()) {
+        const dir = fs.readdirSync(item);
+        const sha = crypto.createHash('sha1');
+        for (const file of dir) {
+            if (ignoredDirsForHash.includes(file) && fs.lstatSync(path.join(item, file)).isDirectory()) {
+                sha.update('');
+            } else {
+                const hash = await hashItem(path.join(item, file));
+                sha.update(hash.toString());
+            }
+        }
+        return sha.digest('hex');
+    } else {
+        const sha = crypto.createHash('sha1');
+        const reader = fs.createReadStream(item);
+        return new Promise(resolve => {
+            reader.on('data', chunk => {
+                sha.update(chunk);
+            });
+            reader.on('close', () => {
+                resolve(sha.digest('hex'));
+            });
+        });
+    }
+}
+
+async function verifyInstallation(installPath, pluginName, opts) {
+    console.log('Verifying integrity...');
+    const hash = await hashItem(installPath);
+    console.log(`SHA1 Hash: ${hash}`);
+
+    if (stateJson && stateJson.plugins) {
+        stateJson.plugins[pluginName].hash = hash;
+        saveState();
+    }
+
+    if (!opts.skipValidation) {
+
+        // if (hash !== data['integrityHash']) {
+        //     console.log('WARNING! Integrity hash dos not match!');
+        //     console.log(`Expected:\t${data['integrityHash']}`);
+        //     console.log(`Current:\t${hash}`);
+        //     process.exit(0);
+        // }
+
+        const installedFiles = fs.readdirSync(installPath);
+        if (!installedFiles.includes('package.json')) {
+            console.log('package.json not found!');
+            process.exit(0);
+        }
+    }
+}
+
+async function clonePluginRepo(pluginName: string, options: any): Promise<void> {
+    const dir = path.join(pluginDirAbsolutePath, pluginName);
+    await new Promise<void>((resolve, reject) => {
+        const gitClone = spawn('git', ['clone', options.repository, dir]);
+        gitClone.stdout.on('data', chunk => {
+            console.log(`->> ${chunk}`);
+        });
+        gitClone.stderr.on('data', chunk => {
+            console.log(`->> ${chunk}`);
+        });
+        gitClone.on('close', (code) => {
+            if (code !== 0) {
+                console.log(`process exited with code ${code}`);
+            }
+            verifyInstallation(dir, pluginName, options)
+                .then(() => {
+                    resolve();
+                })
+                .catch((err) => {
+                    throwAndQuit(err.message);
+                    reject();
+                });
+        });
+    });
+}
+
+function init() {
+    const check = existsSync(pluginStatePath);
+    if (check) {
+        try {
+            stateJson = JSON.parse(readFileSync(pluginStatePath).toString());
+        } catch (e) {
+            console.log(e);
+        }
+    } else {
+        console.log('Creating new state file...');
+        stateJson = {
+            update_ts: Date.now(),
+            hyperion_version: package_json.version,
+            plugins: {}
+        };
+        writeFileSync(pluginStatePath, JSON.stringify(stateJson, null, 2));
+    }
+}
+
+async function installPlugin(plugin: string, options: any) {
+    console.log(`Plugin Name: ${plugin}`);
+    console.log(`Repository Url: ${options.repository}`);
+
+    if (options.branch) {
+        console.log(`Branch Name: ${options.branch}`);
+    }
+
+    // check installation
+    await verifyInstalledPlugin(plugin, options);
+
+    if (stateJson && stateJson.plugins) {
+        stateJson.plugins[plugin] = {};
+    } else {
+        throwAndQuit('fatal error on state');
+    }
+
+    // clone repository
+    await clonePluginRepo(plugin, options);
+
+    // checkout branch
+    if (options.branch) {
+        const dir = path.join(pluginDirAbsolutePath, plugin);
+        const results = checkoutBranch(dir, options.branch);
+        stateJson.plugins[plugin].branch = options.branch;
+        console.log(results);
+    }
+
+    // build plugin
+    await buildPlugin(plugin, {
+        skipInstall: false
+    });
+}
+
+async function uninstall(plugin) {
+    const pluginDir = path.join(pluginDirAbsolutePath, plugin);
+    try {
+        if (!existsSync(pluginDir)) {
+            throwAndQuit(`plugin ${plugin} not found!`);
+        }
+        console.log(`Removing ${pluginDir}...`);
+        await rm(pluginDir, {
+            recursive: true
+        });
+        if (stateJson && stateJson.plugins) {
+            delete stateJson.plugins[plugin];
+            saveState();
+        }
+        console.log(`${plugin} removed!`);
+    } catch (e) {
+        throwAndQuit(e.message);
+    }
+}
+
+async function listPlugins() {
+    const dirs = await readdir(pluginDirAbsolutePath);
+    const results: any[] = [];
+    for (const dir of dirs) {
+        try {
+            const packageJsonPath = path.join(pluginDirAbsolutePath, dir, 'package.json');
+            const package_json = JSON.parse(readFileSync(packageJsonPath).toString());
+            const {name, version, description} = package_json;
+            // console.table([dir, name, version, description].join('\t'));
+
+            let pluginState = false;
+            let branch = '';
+
+            if (stateJson.plugins && stateJson.plugins[dir]) {
+                if (stateJson.plugins[dir].enabled) {
+                    pluginState = true;
+                }
+                branch = stateJson.plugins[dir].branch;
+            }
+
+            results.push({
+                alias: dir,
+                enabled: pluginState,
+                plugin_name: name,
+                version,
+                branch,
+                description
+            });
+        } catch (e) {
+            console.log(e.message);
+        }
+    }
+    results.sort((a, b) => b.ative - a.ative);
+    console.table(results);
+}
+
+function saveState() {
+    stateJson.update_ts = Date.now();
+    writeFileSync(pluginStatePath, JSON.stringify(stateJson, null, 2));
+}
+
+async function setPluginState(pluginName: string, newState: boolean) {
+    if (stateJson) {
+        if (stateJson.plugins[pluginName]) {
+            stateJson.plugins[pluginName].enabled = newState;
+        } else {
+            stateJson.plugins[pluginName] = {
+                enabled: newState
+            }
+        }
+    }
+    saveState();
+    console.log(`[${pluginName}] plugin ${newState ? 'enabled' : 'disabled'} successfully!`);
+}
+
+function enablePlugin(pluginName: string) {
+    return setPluginState(pluginName, true);
+}
+
+function disablePlugin(pluginName: string) {
+    return setPluginState(pluginName, false);
+}
+
+function printState() {
+    console.log(JSON.stringify(stateJson, null, 2));
+}
+
+async function buildAllPlugins(flags) {
+    const names = readdirSync(pluginDirAbsolutePath);
+    for (const name of names) {
+        await buildPlugin(name, flags)
+    }
+}
+
+async function updatePlugin(name: string) {
+    const pluginDir = path.join(pluginDirAbsolutePath, name);
+    const resetStatus = gitReset(pluginDir);
+    console.log(resetStatus);
+    const pullStatus = gitPull(pluginDir);
+    console.log(pullStatus);
+    await buildPlugin(name, {});
+}
+
+(() => {
+
+    init();
+
+    program.description('Command-line utility to manage Hyperion plugins\nMade with ♥  by EOS Rio');
+
+    program.command('install <plugin>').alias('i')
+        .description('install new plugin')
+        .option('-r, --repository <url>', 'custom repository url', checkUrl)
+        .option('-b, --branch <branch>', 'checkout specific branch')
+        .option('-s, --skip-validation', 'skip plugin integrity validation')
+        .action(installPlugin);
+
+    program.command('enable <plugin>')
+        .description('enable plugin')
+        .action(enablePlugin);
+
+    program.command('disable <plugin>')
+        .description('disable plugin')
+        .action(disablePlugin);
+
+    program.command('state')
+        .description('print current state file')
+        .action(printState);
+
+    program.command('list').alias('ls')
+        .description('list installed plugins')
+        .action(listPlugins);
+
+    program.command('build <plugin>').alias('b')
+        .description('build a single plugin')
+        .option('-s, --skip-install', 'skip "npm install" step')
+        .action(buildPlugin);
+
+    program.command('update <plugin>')
+        .description('update a single plugin')
+        .action(updatePlugin);
+
+    program.command('build-all')
+        .description('build all plugins')
+        .option('-s, --skip-install', 'skip "npm install" step')
+        .action(buildAllPlugins);
+
+    program.command('uninstall <plugin>').alias('rm')
+        .description('uninstall plugin')
+        .action(uninstall);
+
+    program.parse();
+})();

scripts/hyp-config.js

@@ -0,0 +1,259 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const path_1 = __importDefault(require("path"));
+const promises_1 = require("fs/promises");
+const fs_1 = require("fs");
+const eosjs_1 = require("eosjs");
+const cross_fetch_1 = __importDefault(require("cross-fetch"));
+const ws_1 = __importDefault(require("ws"));
+const program = new commander_1.Command();
+const chainsDir = path_1.default.join(path_1.default.resolve(), 'chains');
+async function getConnections() {
+    const connectionsJsonFile = await (0, promises_1.readFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'));
+    return JSON.parse(connectionsJsonFile.toString());
+}
+async function getExampleConfig() {
+    const exampleChainData = await (0, promises_1.readFile)(path_1.default.join(chainsDir, 'example.config.json'));
+    return JSON.parse(exampleChainData.toString());
+}
+async function listChains(flags) {
+    const dirdata = await (0, promises_1.readdir)(chainsDir);
+    const connections = await getConnections();
+    const exampleChain = await getExampleConfig();
+    const configuredTable = [];
+    const pendingTable = [];
+    for (const file of dirdata.filter(f => f.endsWith('.json'))) {
+        if (file === 'example.config.json') {
+            continue;
+        }
+        try {
+            const fileData = await (0, promises_1.readFile)(path_1.default.join(chainsDir, file));
+            const jsonData = JSON.parse(fileData.toString());
+            const chainName = jsonData.settings.chain;
+            const chainConn = connections.chains[chainName];
+            const fullChainName = jsonData.api.chain_name;
+            // validate configs with example
+            let missingSections = 0;
+            let missingFields = 0;
+            for (const key in exampleChain) {
+                if (!jsonData[key]) {
+                    if (flags.fixMissingFields) {
+                        jsonData[key] = exampleChain[key];
+                    }
+                    else {
+                        console.log(`⚠ Section ${key} missing on ${file}`);
+                        missingSections++;
+                    }
+                }
+                else {
+                    for (const subKey in exampleChain[key]) {
+                        if (!jsonData[key].hasOwnProperty(subKey)) {
+                            if (flags.fixMissingFields) {
+                                jsonData[key][subKey] = exampleChain[key][subKey];
+                            }
+                            else {
+                                console.log(`⚠ Field ${subKey} missing from ${key} on ${file} (default = ${exampleChain[key][subKey]})`);
+                                missingFields++;
+                            }
+                        }
+                    }
+                }
+            }
+            if (flags.fixMissingFields) {
+                await (0, promises_1.writeFile)(path_1.default.join(chainsDir, file), JSON.stringify(jsonData, null, 2));
+            }
+            const common = {
+                full_name: fullChainName,
+                name: chainName,
+                parser: jsonData.settings.parser,
+                abi_scan: jsonData.indexer.abi_scan_mode,
+                live_reader: jsonData.indexer.live_reader,
+                start_on: jsonData.indexer.start_on,
+                stop_on: jsonData.indexer.stop_on
+            };
+            if (chainConn) {
+                configuredTable.push({
+                    ...common,
+                    nodeos_http: chainConn?.http,
+                    nodeos_ship: chainConn?.ship,
+                    chain_id: chainConn?.chain_id.substr(0, 16) + '...'
+                });
+            }
+            else {
+                pendingTable.push(common);
+            }
+        }
+        catch (e) {
+            console.log(`Failed to parse ${file} - ${e.message}`);
+        }
+    }
+    console.log(' >>>> Fully Configured Chains');
+    console.table(configuredTable);
+    if (!flags.valid) {
+        console.log(' >>>> Pending Configuration Chains');
+        console.table(pendingTable);
+    }
+}
+async function newChain(shortName, options) {
+    console.log(`Creating new config for ${shortName}...`);
+    const targetPath = path_1.default.join(chainsDir, `${shortName}.config.json`);
+    if ((0, fs_1.existsSync)(targetPath)) {
+        console.error(`Chain config for ${shortName} already defined!`);
+        process.exit(0);
+    }
+    // read example
+    const exampleChain = await getExampleConfig();
+    // read connections.json
+    const connections = await getConnections();
+    if (connections.chains[shortName]) {
+        console.log('Connections already defined!');
+        console.log(connections.chains[shortName]);
+        process.exit(0);
+    }
+    else {
+        connections.chains[shortName] = {
+            name: '',
+            ship: '',
+            http: '',
+            chain_id: '',
+            WS_ROUTER_HOST: '127.0.0.1',
+            WS_ROUTER_PORT: 7001
+        };
+    }
+    const jsonData = exampleChain;
+    jsonData.settings.chain = shortName;
+    if (options.http) {
+        if (options.http.startsWith('http://') || options.http.startsWith('https://')) {
+            console.log(`Testing connection on ${options.http}`);
+        }
+        else {
+            console.error(`Invalid HTTP Endpoint [${options.http}] - Url must start with either http:// or https://`);
+            process.exit(1);
+        }
+        // test nodeos availability
+        const jsonRpc = new eosjs_1.JsonRpc(options.http, { fetch: cross_fetch_1.default });
+        const info = await jsonRpc.get_info();
+        jsonData.api.chain_api = options.http;
+        connections.chains[shortName].chain_id = info.chain_id;
+        connections.chains[shortName].http = options.http;
+        if (info.server_version_string.includes('2.1')) {
+            jsonData.settings.parser = '2.1';
+        }
+        else {
+            jsonData.settings.parser = '1.8';
+        }
+        console.log(`Parser version set to ${jsonData.settings.parser}`);
+        console.log(`Chain ID: ${info.chain_id}`);
+    }
+    if (options.ship) {
+        if (options.ship.startsWith('ws://') || options.ship.startsWith('wss://')) {
+            console.log(`Testing connection on ${options.ship}`);
+        }
+        else {
+            console.error(`Invalid WS Endpoint [${options.ship}] - Url must start with either ws:// or wss://`);
+            process.exit(1);
+        }
+        // test ship availability
+        const status = await new Promise(resolve => {
+            const ws = new ws_1.default(options.ship);
+            ws.on("message", (data) => {
+                try {
+                    const abi = JSON.parse(data.toString());
+                    if (abi.version) {
+                        console.log(`Received SHIP Abi Version: ${abi.version}`);
+                        resolve(true);
+                    }
+                    else {
+                        resolve(false);
+                    }
+                }
+                catch (e) {
+                    console.log(e.message);
+                    resolve(false);
+                }
+                ws.close();
+            });
+            ws.on("error", err => {
+                console.log(err);
+                ws.close();
+                resolve(false);
+            });
+        });
+        if (status) {
+            connections.chains[shortName].ship = options.ship;
+        }
+        else {
+            console.log(`Invalid SHIP Endpoint [${options.ship}]`);
+        }
+    }
+    const fullNameArr = [];
+    shortName.split('-').forEach((word) => {
+        fullNameArr.push(word[0].toUpperCase() + word.substr(1));
+    });
+    const fullChainName = fullNameArr.join(' ');
+    jsonData.api.chain_name = fullChainName;
+    connections.chains[shortName].name = fullChainName;
+    console.log(connections.chains[shortName]);
+    console.log('Saving connections.json...');
+    await (0, promises_1.writeFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    console.log(`Saving chains/${shortName}.config.json...`);
+    await (0, promises_1.writeFile)(targetPath, JSON.stringify(jsonData, null, 2));
+}
+async function rmChain(shortName) {
+    console.log(`Removing config for ${shortName}...`);
+    const targetPath = path_1.default.join(chainsDir, `${shortName}.config.json`);
+    if (!(0, fs_1.existsSync)(targetPath)) {
+        console.error(`Chain config for ${shortName} not found!`);
+        process.exit(0);
+    }
+    // create backups
+    const backupDir = path_1.default.join(path_1.default.resolve(), 'configuration_backups');
+    if (!(0, fs_1.existsSync)(backupDir)) {
+        await (0, promises_1.mkdir)(backupDir);
+    }
+    const connections = await getConnections();
+    try {
+        const chainConn = connections.chains[shortName];
+        const now = Date.now();
+        if (chainConn) {
+            await (0, promises_1.writeFile)(path_1.default.join(backupDir, `${shortName}_${now}_connections.json`), JSON.stringify(chainConn, null, 2));
+        }
+        await (0, promises_1.cp)(targetPath, path_1.default.join(backupDir, `${shortName}_${now}_config.json`));
+    }
+    catch (e) {
+        console.log(e.message);
+        console.log('Failed to create backups! Aborting!');
+        process.exit(1);
+    }
+    console.log(`Deleting ${targetPath}`);
+    await (0, promises_1.rm)(targetPath);
+    delete connections.chains[shortName];
+    console.log('Saving connections.json...');
+    await (0, promises_1.writeFile)(path_1.default.join(path_1.default.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    console.log(`✅ ${shortName} removal completed!`);
+}
+// main program
+(() => {
+    const list = program.command('list').alias('ls');
+    list.command('chains')
+        .option('--valid', 'only show valid chains')
+        .option('--fix-missing-fields', 'set defaults on missing fields')
+        .description('list configured chains')
+        .action(listChains);
+    const newCmd = program.command('new').alias('n');
+    newCmd.command('chain <shortName>')
+        .description('initialize new chain config based on example')
+        .option('--http <http_endpoint>', 'define chain api http endpoint')
+        .option('--ship <ship_endpoint>', 'define state history ws endpoint')
+        .action(newChain);
+    const remove = program.command('remove').alias('rm');
+    remove.command('chain <shortName>')
+        .description('backup and delete chain configuration')
+        .action(rmChain);
+    program.parse(process.argv);
+})();
+//# sourceMappingURL=hyp-config.js.map

scripts/hyp-config.ts

@@ -0,0 +1,283 @@
+import {Command} from "commander";
+import path from "path";
+import {cp, mkdir, readdir, readFile, rm, writeFile} from "fs/promises";
+import {HyperionConfig} from "../interfaces/hyperionConfig";
+import {HyperionConnections} from "../interfaces/hyperionConnections";
+import {existsSync} from "fs";
+import {JsonRpc} from "eosjs";
+import fetch from "cross-fetch";
+import WebSocket from 'ws';
+
+const program = new Command();
+const chainsDir = path.join(path.resolve(), 'chains');
+
+async function getConnections(): Promise<HyperionConnections> {
+    const connectionsJsonFile = await readFile(path.join(path.resolve(), 'connections.json'));
+    return JSON.parse(connectionsJsonFile.toString());
+}
+
+async function getExampleConfig(): Promise<HyperionConfig> {
+    const exampleChainData = await readFile(path.join(chainsDir, 'example.config.json'));
+    return JSON.parse(exampleChainData.toString());
+}
+
+
+async function listChains(flags) {
+
+    const dirdata = await readdir(chainsDir);
+    const connections = await getConnections();
+    const exampleChain = await getExampleConfig();
+
+    const configuredTable = [];
+    const pendingTable = [];
+    for (const file of dirdata.filter(f => f.endsWith('.json'))) {
+        if (file === 'example.config.json') {
+            continue;
+        }
+        try {
+            const fileData = await readFile(path.join(chainsDir, file));
+            const jsonData: HyperionConfig = JSON.parse(fileData.toString());
+            const chainName = jsonData.settings.chain;
+            const chainConn = connections.chains[chainName];
+            const fullChainName = jsonData.api.chain_name;
+
+            // validate configs with example
+            let missingSections = 0;
+            let missingFields = 0;
+            for (const key in exampleChain) {
+                if (!jsonData[key]) {
+                    if (flags.fixMissingFields) {
+                        jsonData[key] = exampleChain[key];
+                    } else {
+                        console.log(`⚠ Section ${key} missing on ${file}`);
+                        missingSections++;
+                    }
+                } else {
+                    for (const subKey in exampleChain[key]) {
+                        if (!jsonData[key].hasOwnProperty(subKey)) {
+                            if (flags.fixMissingFields) {
+                                jsonData[key][subKey] = exampleChain[key][subKey];
+                            } else {
+                                console.log(`⚠ Field ${subKey} missing from ${key} on ${file} (default = ${exampleChain[key][subKey]})`);
+                                missingFields++;
+                            }
+                        }
+                    }
+                }
+            }
+
+            if (flags.fixMissingFields) {
+                await writeFile(path.join(chainsDir, file), JSON.stringify(jsonData, null, 2));
+            }
+
+            const common = {
+                full_name: fullChainName,
+                name: chainName,
+                parser: jsonData.settings.parser,
+                abi_scan: jsonData.indexer.abi_scan_mode,
+                live_reader: jsonData.indexer.live_reader,
+                start_on: jsonData.indexer.start_on,
+                stop_on: jsonData.indexer.stop_on
+            };
+
+            if (chainConn) {
+                configuredTable.push({
+                    ...common,
+                    nodeos_http: chainConn?.http,
+                    nodeos_ship: chainConn?.ship,
+                    chain_id: chainConn?.chain_id.substr(0,16) + '...'
+                });
+            } else {
+                pendingTable.push(common);
+            }
+        } catch (e) {
+            console.log(`Failed to parse ${file} - ${e.message}`);
+        }
+    }
+
+    console.log(' >>>> Fully Configured Chains');
+    console.table(configuredTable);
+
+    if (!flags.valid) {
+        console.log(' >>>> Pending Configuration Chains');
+        console.table(pendingTable);
+    }
+}
+
+async function newChain(shortName, options) {
+    console.log(`Creating new config for ${shortName}...`);
+    const targetPath = path.join(chainsDir, `${shortName}.config.json`);
+
+    if (existsSync(targetPath)) {
+        console.error(`Chain config for ${shortName} already defined!`);
+        process.exit(0);
+    }
+
+    // read example
+    const exampleChain = await getExampleConfig();
+
+    // read connections.json
+    const connections = await getConnections();
+
+    if (connections.chains[shortName]) {
+        console.log('Connections already defined!');
+        console.log(connections.chains[shortName]);
+        process.exit(0);
+    } else {
+        connections.chains[shortName] = {
+            name: '',
+            ship: '',
+            http: '',
+            chain_id: '',
+            WS_ROUTER_HOST: '127.0.0.1',
+            WS_ROUTER_PORT: 7001
+        };
+    }
+
+    const jsonData = exampleChain;
+    jsonData.settings.chain = shortName;
+
+    if (options.http) {
+
+        if (options.http.startsWith('http://') || options.http.startsWith('https://')) {
+            console.log(`Testing connection on ${options.http}`);
+        } else {
+            console.error(`Invalid HTTP Endpoint [${options.http}] - Url must start with either http:// or https://`);
+            process.exit(1);
+        }
+
+        // test nodeos availability
+        const jsonRpc = new JsonRpc(options.http, {fetch});
+        const info = await jsonRpc.get_info();
+        jsonData.api.chain_api = options.http;
+        connections.chains[shortName].chain_id = info.chain_id;
+        connections.chains[shortName].http = options.http;
+        if (info.server_version_string.includes('2.1')) {
+            jsonData.settings.parser = '2.1';
+        } else {
+            jsonData.settings.parser = '1.8';
+        }
+        console.log(`Parser version set to ${jsonData.settings.parser}`);
+        console.log(`Chain ID: ${info.chain_id}`)
+    }
+
+    if (options.ship) {
+
+        if (options.ship.startsWith('ws://') || options.ship.startsWith('wss://')) {
+            console.log(`Testing connection on ${options.ship}`);
+        } else {
+            console.error(`Invalid WS Endpoint [${options.ship}] - Url must start with either ws:// or wss://`);
+            process.exit(1);
+        }
+
+        // test ship availability
+        const status = await new Promise<boolean>(resolve => {
+            const ws = new WebSocket(options.ship);
+            ws.on("message", (data: Buffer) => {
+                try {
+                    const abi = JSON.parse(data.toString());
+                    if (abi.version) {
+                        console.log(`Received SHIP Abi Version: ${abi.version}`);
+                        resolve(true);
+                    } else {
+                        resolve(false);
+                    }
+                } catch (e) {
+                    console.log(e.message);
+                    resolve(false);
+                }
+                ws.close();
+            });
+            ws.on("error", err => {
+                console.log(err);
+                ws.close();
+                resolve(false);
+            });
+        });
+        if (status) {
+            connections.chains[shortName].ship = options.ship;
+        } else {
+            console.log(`Invalid SHIP Endpoint [${options.ship}]`);
+        }
+    }
+
+    const fullNameArr = [];
+    shortName.split('-').forEach((word: string) => {
+        fullNameArr.push(word[0].toUpperCase() + word.substr(1));
+        });
+    const fullChainName = fullNameArr.join(' ');
+
+    jsonData.api.chain_name = fullChainName;
+    connections.chains[shortName].name = fullChainName;
+
+    console.log(connections.chains[shortName]);
+
+    console.log('Saving connections.json...');
+    await writeFile(path.join(path.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    console.log(`Saving chains/${shortName}.config.json...`);
+    await writeFile(targetPath, JSON.stringify(jsonData, null, 2));
+}
+
+async function rmChain(shortName) {
+    console.log(`Removing config for ${shortName}...`);
+    const targetPath = path.join(chainsDir, `${shortName}.config.json`);
+
+    if (!existsSync(targetPath)) {
+        console.error(`Chain config for ${shortName} not found!`);
+        process.exit(0);
+    }
+
+    // create backups
+    const backupDir = path.join(path.resolve(), 'configuration_backups');
+    if (!existsSync(backupDir)) {
+        await mkdir(backupDir);
+    }
+
+    const connections = await getConnections();
+    try {
+        const chainConn = connections.chains[shortName];
+        const now = Date.now();
+        if (chainConn) {
+            await writeFile(
+                path.join(backupDir, `${shortName}_${now}_connections.json`),
+                JSON.stringify(chainConn, null, 2)
+            );
+        }
+        await cp(targetPath, path.join(backupDir, `${shortName}_${now}_config.json`));
+    } catch (e) {
+        console.log(e.message);
+        console.log('Failed to create backups! Aborting!');
+        process.exit(1);
+    }
+
+    console.log(`Deleting ${targetPath}`);
+    await rm(targetPath);
+    delete connections.chains[shortName];
+    console.log('Saving connections.json...');
+    await writeFile(path.join(path.resolve(), 'connections.json'), JSON.stringify(connections, null, 2));
+    console.log(`✅ ${shortName} removal completed!`);
+}
+
+// main program
+(() => {
+    const list = program.command('list').alias('ls');
+    list.command('chains')
+        .option('--valid', 'only show valid chains')
+        .option('--fix-missing-fields', 'set defaults on missing fields')
+        .description('list configured chains')
+        .action(listChains);
+
+    const newCmd = program.command('new').alias('n');
+    newCmd.command('chain <shortName>')
+        .description('initialize new chain config based on example')
+        .option('--http <http_endpoint>', 'define chain api http endpoint')
+        .option('--ship <ship_endpoint>', 'define state history ws endpoint')
+        .action(newChain)
+
+    const remove = program.command('remove').alias('rm');
+    remove.command('chain <shortName>')
+        .description('backup and delete chain configuration')
+        .action(rmChain);
+
+    program.parse(process.argv);
+})();

scripts/sync-by-range.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# syncbyrange.sh - Bash script for synchronizing Hyperion EOS in spans 
+# Save this in your Hyperion folder and add to cron 
+# to run this every 10 minutes... just add this to your cron
+# */10 * * * * /opt/eosio/src/Hyperion-History-API/scripts/sync-by=range.sh
+#
+# You can modify the CHAINNAME variable to match your chains/$CHAINNAME.config.json
+#
+# variables that need to be set based on your configuration
+
+PM2="/home/eosio/.npm-global/bin/pm2"
+HYPERIONPATH="/opt/eosio/src/Hyperion-History-API"
+BLOCKSPAN="1000000"
+CHAINNAME="proton"
+CHAINCONF="$HYPERIONPATH/chains/$CHAINNAME.config.json"
+
+# variables that probably don't need to change
+INDEXER="$CHAINNAME-indexer"
+LOGFILE=$HYPERIONPATH/logs/$CHAINNAME-sync.log
+echo `date` >> $LOGFILE
+
+# Get logs from pm2 and lastblock from config file
+LOG=`$PM2 log $INDEXER --nostream --lines 4`
+LASTBLOCK=`cat $CHAINCONF |jq '.indexer.stop_on'|tr -d '"'`
+
+# if the log contains "No blocks are being processed" then update the configs for the next block span
+if echo $LOG | grep -q "No blocks are being processed"; then 
+    echo "Updating configs to next span" >> $LOGFILE
+    timeout 10 $PM2 trigger "$INDEXER" stop
+    LASTBLOCK=`cat $CHAINCONF |jq '.indexer.stop_on'|tr -d '"'`
+    STARTBLOCK=`echo $LASTBLOCK`
+    STOPBLOCK=`expr $LASTBLOCK + $BLOCKSPAN`
+    echo "Updating $CHAINCONF to $STARTBLOCK:$STOPBLOCK" >> $LOGFILE
+    jq --argjson STARTBLOCK "$STARTBLOCK" '.indexer.start_on = $STARTBLOCK' $CHAINCONF > $CHAINCONF.tmp
+    jq --argjson STOPBLOCK "$STOPBLOCK" '.indexer.stop_on = $STOPBLOCK' $CHAINCONF.tmp > $CHAINCONF.tmp2
+    cp $CHAINCONF.tmp2 $CHAINCONF
+    echo "Updated config to sync $CHAIN from $STARTBLOCK to $STOPBLOCK" >> $LOGFILE
+    echo -e "\n-->> Starting $1..."
+    cd $HYPERIONPATH
+    $PM2 start --only "$INDEXER" --update-env --silent
+    $PM2 save
+else
+    LASTBLOCK=`cat $CHAINCONF |jq '.indexer.stop_on'|tr -d '"'`
+    STARTBLOCK=`cat $CHAINCONF |jq '.indexer.start_on'|tr -d '"'`
+   echo "Still synchronizing $CHAIN from $STARTBLOCK to $LASTBLOCK" >> $LOGFILE
+fi 

scripts/watch-missing-blocks.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# fixmissing.sh - Bash script for finding missing blocks in the pm2 logs 
+# Update paths and then you can run this as a test if you have missing blocks in the log 
+# to run this every 2 minutes... just add this to your cron
+# */2 * * * * /opt/eosio/src/Hyperion-History-API/scripts/watch-missing-blocks.sh
+#
+# You can modify the CHAINNAME variable to match your chains/$CHAINNAME.config.json
+#
+# variables that need to be set based on your configuration
+PM2="/home/eosio/.npm-global/bin/pm2"
+HYPERIONPATH="/opt/eosio/src/Hyperion-History-API"
+BLOCKSPAN="1000000"
+CHAINNAME="proton"
+CHAINCONF="$HYPERIONPATH/chains/$CHAINNAME.config.json"
+PM2USER="eosio"
+PM2HOME="`eval echo ~$PM2USER`/.pm2/logs"
+
+# variables that probably don't need to change
+INDEXER="$CHAINNAME-indexer"
+PM2LOG=$PM2HOME/$INDEXER-out.log
+LOGFILE=$HYPERIONPATH/logs/$CHAINNAME-fixmissing.log
+echo `date` >> $LOGFILE
+
+# Check pm2 logs to see if there is a "Mising block:" and then set that as the new STARTBLOCK
+if tail -n 100 $PM2LOG | grep -q "Missing block:"; then 
+    timeout 10 $PM2 trigger "$INDEXER" stop
+    echo "Missed block found" >> $LOGFILE
+    STARTBLOCK=`tail -n 100 $PM2LOG|grep 'Missing block:'|awk '{print $7}'|head -n 1`
+    echo "Updating $CHAINCONF to reindex $STARTBLOCK" >> $LOGFILE
+    jq --argjson STARTBLOCK "$STARTBLOCK" '.indexer.start_on = $STARTBLOCK' $CHAINCONF > $CHAINCONF.tmp3
+    cp $CHAINCONF.tmp3 $CHAINCONF
+    echo "Updated config to sync $CHAIN to reindex $STARTBLOCK" >> $LOGFILE
+    echo -e "\n-->> Starting $INDEXER..."
+    cd $HYPERIONPATH
+    $PM2 start --only "$INDEXER" --update-env --silent
+    $PM2 save
+else
+    LASTBLOCK=`cat $CHAINCONF |jq '.indexer.stop_on'|tr -d '"'`
+    STARTBLOCK=`cat $CHAINCONF |jq '.indexer.start_on'|tr -d '"'`
+   echo "No missed blocks on $CHAIN from $STARTBLOCK to $LASTBLOCK" >> $LOGFILE
+fi 

types/fastify/index.d.ts

@@ -1,21 +1,23 @@
 import {Client} from "@elastic/elasticsearch";
-import {Redis} from "ioredis";
 import {ConnectionManager} from "../../connections/manager.class";
 import {Api, JsonRpc} from "eosjs";
+import {CacheManager} from "../../api/helpers/cacheManager";
+import {FastifyRedis} from "fastify-redis";
 
 declare module 'fastify' {
-	export interface FastifyInstance {
-		manager: ConnectionManager;
-		redis: Redis;
-		elastic: Client;
-		eosjs: {
-			rpc: JsonRpc;
-			api: Api;
-		};
-		chain_api: string;
-		push_api: string;
-		tokenCache: Map<string, any>;
-		allowedActionQueryParamSet: Set<string>;
-		routeSet: Set<string>;
-	}
+    export interface FastifyInstance {
+        manager: ConnectionManager;
+        cacheManager: CacheManager;
+        redis: FastifyRedis;
+        elastic: Client;
+        eosjs: {
+            rpc: JsonRpc;
+            api: Api;
+        };
+        chain_api: string;
+        push_api: string;
+        tokenCache: Map<string, any>;
+        allowedActionQueryParamSet: Set<string>;
+        routeSet: Set<string>;
+    }
 }

workers/deserializer.ts

@@ -1145,21 +1145,8 @@ export default class MainDSWorker extends HyperionWorker {
                     }
                 }
 
-                let jsonRow;
-                // forced deserialization
-                try {
-                    if (payload.code === 'm.federation' && payload.table === 'leaders') {
-                        payload['data'] = {};
-                        jsonRow = payload;
-                    }
-                } catch (e) {
-                    console.log(e);
-                }
-
                 // decode contract data
-                if (!jsonRow) {
-                    jsonRow = await this.processContractRowNative(payload, block_num);
-                }
+                let jsonRow = await this.processContractRowNative(payload, block_num);
 
                 if (jsonRow?.value && !jsonRow['_blacklisted']) {
                     debugLog('Delta DS failed ->>', jsonRow);
@@ -1471,7 +1458,6 @@ export default class MainDSWorker extends HyperionWorker {
                                 await this.deltaStructHandlers[key](data[1], block_num, block_ts, row, block_id);
                             } catch (e) {
                                 hLog(`Delta struct [${key}] processing error: ${e.message}`);
-                                // hLog(data);
                             }
                         }
                     }

workers/ws-router.ts

@@ -8,429 +8,433 @@ const greylist = ['eosio.token'];
 
 export default class WSRouter extends HyperionWorker {
 
-	q: string;
-	totalRoutedMessages = 0;
-	firstData = false;
-	relays = {};
-	clientIndex = new Map();
-	codeActionMap = new Map();
-	notifiedMap = new Map();
-	codeDeltaMap = new Map();
-	payerMap = new Map();
-	activeRequests = new Map();
-	private io: Server;
-	private totalClients = 0;
+    q: string;
+    totalRoutedMessages = 0;
+    firstData = false;
+    relays = {};
+    clientIndex = new Map();
+    codeActionMap = new Map();
+    notifiedMap = new Map();
+    codeDeltaMap = new Map();
+    payerMap = new Map();
+    activeRequests = new Map();
+    private io: Server;
+    private totalClients = 0;
 
-	constructor() {
-		super();
-		this.q = this.chain + ':stream';
-		this.activeRequests.set('*', {
-			sockets: []
-		});
-	}
+    constructor() {
+        super();
+        this.q = this.chain + ':stream';
+        this.activeRequests.set('*', {
+            sockets: []
+        });
+    }
 
-	assertQueues(): void {
-		this.ch.assertQueue(this.q);
-		this.ch.consume(this.q, this.onConsume.bind(this));
-	}
+    assertQueues(): void {
+        this.ch.assertQueue(this.q);
+        this.ch.consume(this.q, this.onConsume.bind(this));
+    }
 
 
-	onIpcMessage(msg: any): void {
-		switch (msg.event) {
-			case 'lib_update': {
-				this.io.emit('lib_update', {
-					chain_id: this.manager.conn.chains[this.chain]?.chain_id,
-					...msg.data
-				});
-				break;
-			}
-			case 'fork_event': {
-				this.io.emit('fork_event', {
-					chain_id: this.manager.conn.chains[this.chain]?.chain_id,
-					...msg.data
-				});
-				break;
-			}
-		}
-	}
+    onIpcMessage(msg: any): void {
+        switch (msg.event) {
+            case 'lib_update': {
+                this.io.emit('lib_update', {
+                    chain_id: this.manager.conn.chains[this.chain]?.chain_id,
+                    ...msg.data
+                });
+                break;
+            }
+            case 'fork_event': {
+                this.io.emit('fork_event', {
+                    chain_id: this.manager.conn.chains[this.chain]?.chain_id,
+                    ...msg.data
+                });
+                break;
+            }
+        }
+    }
 
-	async run(): Promise<void> {
-		this.initRoutingServer();
-		this.startRoutingRateMonitor();
-		return undefined;
-	}
+    async run(): Promise<void> {
+        this.initRoutingServer();
+        this.startRoutingRateMonitor();
+        return undefined;
+    }
 
-	onConsume(msg) {
-		if (!this.firstData) {
-			this.firstData = true;
-		}
-		switch (msg.properties.headers.event) {
-			case 'trace': {
-				const actHeader = msg.properties.headers;
-				const code = actHeader.account;
-				const name = actHeader.name;
-				const notified = actHeader.notified.split(',');
-				let decodedMsg;
+    onConsume(msg) {
+        if (!this.firstData) {
+            this.firstData = true;
+        }
 
-				// send to contract subscribers
-				if (this.codeActionMap.has(code)) {
-					const codeReq = this.codeActionMap.get(code);
-					decodedMsg = Buffer.from(msg.content).toString();
+        // push to plugin handlers
+        this.mLoader.processStreamEvent(msg);
 
-					// send to action subscribers
-					if (codeReq.has(name)) {
-						for (const link of codeReq.get(name).links) {
-							this.forwardActionMessage(decodedMsg, link, notified);
-						}
-					}
-					// send to wildcard subscribers
-					if (codeReq.has("*")) {
-						for (const link of codeReq.get("*").links) {
-							this.forwardActionMessage(decodedMsg, link, notified);
-						}
-					}
-				}
+        switch (msg.properties.headers.event) {
+            case 'trace': {
+                const actHeader = msg.properties.headers;
+                const code = actHeader.account;
+                const name = actHeader.name;
+                const notified = actHeader.notified.split(',');
+                let decodedMsg;
 
-				// send to notification subscribers
-				notified.forEach((acct) => {
-					if (this.notifiedMap.has(acct)) {
-						if (!decodedMsg) {
-							decodedMsg = Buffer.from(msg.content).toString();
-						}
-						for (const link of this.notifiedMap.get(acct).links) {
-							this.forwardActionMessage(decodedMsg, link, notified);
-						}
-					}
-				});
-				break;
-			}
+                // send to contract subscribers
+                if (this.codeActionMap.has(code)) {
+                    const codeReq = this.codeActionMap.get(code);
+                    decodedMsg = Buffer.from(msg.content).toString();
 
-			case 'delta': {
-				const deltaHeader = msg.properties.headers;
-				const code = deltaHeader.code;
-				const table = deltaHeader.table;
-				// const scope = deltaHeader.scope;
-				const payer = deltaHeader.payer;
-				// console.log(code, table, scope, payer);
-				let decodedDeltaMsg;
-				// Forward to CODE/TABLE listeners
-				if (this.codeDeltaMap.has(code)) {
-					decodedDeltaMsg = Buffer.from(msg.content).toString();
+                    // send to action subscribers
+                    if (codeReq.has(name)) {
+                        for (const link of codeReq.get(name).links) {
+                            this.forwardActionMessage(decodedMsg, link, notified);
+                        }
+                    }
+                    // send to wildcard subscribers
+                    if (codeReq.has("*")) {
+                        for (const link of codeReq.get("*").links) {
+                            this.forwardActionMessage(decodedMsg, link, notified);
+                        }
+                    }
+                }
 
-					const tableDeltaMap = this.codeDeltaMap.get(code);
-					// Send specific table
-					if (tableDeltaMap.has(table)) {
-						for (const link of tableDeltaMap.get(table).links) {
-							this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
-						}
-					}
-					// Send any table
-					if (tableDeltaMap.has("*")) {
-						for (const link of tableDeltaMap.get("*").links) {
-							this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
-						}
-					}
-				}
-				// Forward to PAYER listeners
-				if (this.payerMap.has(payer)) {
-					decodedDeltaMsg = Buffer.from(msg.content).toString();
-					for (const link of this.payerMap.get(payer).links) {
-						this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
-					}
-				}
-				break;
-			}
+                // send to notification subscribers
+                notified.forEach((acct) => {
+                    if (this.notifiedMap.has(acct)) {
+                        if (!decodedMsg) {
+                            decodedMsg = Buffer.from(msg.content).toString();
+                        }
+                        for (const link of this.notifiedMap.get(acct).links) {
+                            this.forwardActionMessage(decodedMsg, link, notified);
+                        }
+                    }
+                });
+                break;
+            }
 
-			default: {
-				console.log('Unindentified message!');
-				console.log(msg);
-			}
-		}
-		this.ch.ack(msg);
-	}
+            case 'delta': {
+                const deltaHeader = msg.properties.headers;
+                const code = deltaHeader.code;
+                const table = deltaHeader.table;
+                // const scope = deltaHeader.scope;
+                const payer = deltaHeader.payer;
+                // console.log(code, table, scope, payer);
+                let decodedDeltaMsg;
+                // Forward to CODE/TABLE listeners
+                if (this.codeDeltaMap.has(code)) {
+                    decodedDeltaMsg = Buffer.from(msg.content).toString();
 
-	startRoutingRateMonitor() {
-		setInterval(() => {
-			if (this.totalRoutedMessages > 0) {
-				hLog('[Router] Routing rate: ' + (this.totalRoutedMessages / 20) + ' msg/s');
-				this.totalRoutedMessages = 0;
-			}
-		}, 20000);
-	}
+                    const tableDeltaMap = this.codeDeltaMap.get(code);
+                    // Send specific table
+                    if (tableDeltaMap.has(table)) {
+                        for (const link of tableDeltaMap.get(table).links) {
+                            this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
+                        }
+                    }
+                    // Send any table
+                    if (tableDeltaMap.has("*")) {
+                        for (const link of tableDeltaMap.get("*").links) {
+                            this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
+                        }
+                    }
+                }
+                // Forward to PAYER listeners
+                if (this.payerMap.has(payer)) {
+                    decodedDeltaMsg = Buffer.from(msg.content).toString();
+                    for (const link of this.payerMap.get(payer).links) {
+                        this.forwardDeltaMessage(decodedDeltaMsg, link, payer);
+                    }
+                }
+                break;
+            }
 
-	countClients() {
-		let total = 0;
-		for (let key in this.relays) {
-			if (this.relays.hasOwnProperty(key)) {
-				if (this.relays[key].connected) {
-					total += this.relays[key].clients;
-				}
-			}
-		}
-		this.totalClients = total;
-		hLog('Total WS clients:', this.totalClients);
-	}
+            default: {
+                console.log('Unindentified message!');
+                console.log(msg);
+            }
+        }
+        this.ch.ack(msg);
+    }
 
-	appendToL1Map(target, primary, link) {
-		if (target.has(primary)) {
-			target.get(primary).links.push(link);
-		} else {
-			target.set(primary, {links: [link]});
-		}
-	}
+    startRoutingRateMonitor() {
+        setInterval(() => {
+            if (this.totalRoutedMessages > 0) {
+                hLog('[Router] Routing rate: ' + (this.totalRoutedMessages / 20) + ' msg/s');
+                this.totalRoutedMessages = 0;
+            }
+        }, 20000);
+    }
 
-	appendToL2Map(target, primary, secondary, link) {
-		if (target.has(primary)) {
-			const pMap = target.get(primary);
-			if (pMap.has(secondary)) {
-				const pLinks = pMap.get(secondary);
-				pLinks.links.push(link);
-			} else {
-				pMap.set(secondary, {
-					links: [link]
-				});
-			}
-		} else {
-			const sMap = new Map();
-			sMap.set(secondary, {
-				links: [link]
-			});
-			target.set(primary, sMap);
-		}
-	}
+    countClients() {
+        let total = 0;
+        for (let key in this.relays) {
+            if (this.relays.hasOwnProperty(key)) {
+                if (this.relays[key].connected) {
+                    total += this.relays[key].clients;
+                }
+            }
+        }
+        this.totalClients = total;
+        hLog('Total WS clients:', this.totalClients);
+    }
 
-	addActionRequest(data, id) {
-		const req = data.request;
-		if (typeof req.account !== 'string') {
-			return {status: 'FAIL', reason: 'invalid request'};
-		}
-		if (greylist.indexOf(req.contract) !== -1) {
-			if (req.account === '' || req.account === req.contract) {
-				return {
-					status: 'FAIL',
-					reason: 'request too broad, please be more specific'
-				};
-			}
-		}
-		const link = {
-			type: 'action',
-			relay: id,
-			client: data.client_socket,
-			filters: req.filters,
-			account: req.account,
-			added_on: Date.now()
-		};
-		if (req.contract !== '' && req.contract !== '*') {
-			this.appendToL2Map(this.codeActionMap, req.contract, req.action, link);
-		} else {
-			if (req.account !== '') {
-				this.appendToL1Map(this.notifiedMap, req.account, link);
-			} else {
-				return {status: 'FAIL', reason: 'invalid request'};
-			}
-		}
-		this.addToClientIndex(data, id, [req.contract, req.action, req.account]);
-		return {
-			status: 'OK'
-		};
-	}
+    appendToL1Map(target, primary, link) {
+        if (target.has(primary)) {
+            target.get(primary).links.push(link);
+        } else {
+            target.set(primary, {links: [link]});
+        }
+    }
 
-	addToClientIndex(data, id, path) {
-		// register client on index
-		if (this.clientIndex.has(data.client_socket)) {
-			this.clientIndex.get(data.client_socket).set(id, path);
-			// console.log('new relay added to existing client');
-		} else {
-			const list = new Map();
-			list.set(id, path);
-			this.clientIndex.set(data.client_socket, list);
-			// console.log('new client added to index');
-		}
-	}
+    appendToL2Map(target, primary, secondary, link) {
+        if (target.has(primary)) {
+            const pMap = target.get(primary);
+            if (pMap.has(secondary)) {
+                const pLinks = pMap.get(secondary);
+                pLinks.links.push(link);
+            } else {
+                pMap.set(secondary, {
+                    links: [link]
+                });
+            }
+        } else {
+            const sMap = new Map();
+            sMap.set(secondary, {
+                links: [link]
+            });
+            target.set(primary, sMap);
+        }
+    }
 
-	addDeltaRequest(data, id) {
-		const req = data.request;
-		const link = {
-			type: 'delta',
-			relay: id,
-			client: data.client_socket,
-			filters: data.request.filters,
-			payer: data.request.payer,
-			added_on: Date.now()
-		};
-		if (req.code !== '' && req.code !== '*') {
-			this.appendToL2Map(this.codeDeltaMap, req.code, req.table, link);
-		} else {
-			if (req.payer !== '' && req.payer !== '*') {
-				this.appendToL1Map(this.payerMap, req.payer, link);
-			} else {
-				return {status: 'FAIL', reason: 'invalid request'};
-			}
-		}
-		this.addToClientIndex(data, id, [req.code, req.table, req.payer]);
-		return {
-			status: 'OK'
-		};
-	}
+    addActionRequest(data, id) {
+        const req = data.request;
+        if (typeof req.account !== 'string') {
+            return {status: 'FAIL', reason: 'invalid request'};
+        }
+        if (greylist.indexOf(req.contract) !== -1) {
+            if (req.account === '' || req.account === req.contract) {
+                return {
+                    status: 'FAIL',
+                    reason: 'request too broad, please be more specific'
+                };
+            }
+        }
+        const link = {
+            type: 'action',
+            relay: id,
+            client: data.client_socket,
+            filters: req.filters,
+            account: req.account,
+            added_on: Date.now()
+        };
+        if (req.contract !== '' && req.contract !== '*') {
+            this.appendToL2Map(this.codeActionMap, req.contract, req.action, link);
+        } else {
+            if (req.account !== '') {
+                this.appendToL1Map(this.notifiedMap, req.account, link);
+            } else {
+                return {status: 'FAIL', reason: 'invalid request'};
+            }
+        }
+        this.addToClientIndex(data, id, [req.contract, req.action, req.account]);
+        return {
+            status: 'OK'
+        };
+    }
 
-	removeDeepLinks(map, path, key, id) {
-		if (map.has(path[0])) {
-			if (map.get(path[0]).has(path[1])) {
-				const currentLinks = map.get(path[0]).get(path[1]).links;
-				currentLinks.forEach((item, index) => {
-					if (item.relay === key && item.client === id) {
-						currentLinks.splice(index, 1);
-					}
-				});
-			}
-		}
-	}
+    addToClientIndex(data, id, path) {
+        // register client on index
+        if (this.clientIndex.has(data.client_socket)) {
+            this.clientIndex.get(data.client_socket).set(id, path);
+            // console.log('new relay added to existing client');
+        } else {
+            const list = new Map();
+            list.set(id, path);
+            this.clientIndex.set(data.client_socket, list);
+            // console.log('new client added to index');
+        }
+    }
 
-	removeSingleLevelLinks(map, path, key, id) {
-		if (map.has(path[2])) {
-			const _links = map.get(path[2]).links;
-			_links.forEach((item, index) => {
-				if (item.relay === key && item.client === id) {
-					_links.splice(index, 1);
-				}
-			});
-		}
-	}
+    addDeltaRequest(data, id) {
+        const req = data.request;
+        const link = {
+            type: 'delta',
+            relay: id,
+            client: data.client_socket,
+            filters: data.request.filters,
+            payer: data.request.payer,
+            added_on: Date.now()
+        };
+        if (req.code !== '' && req.code !== '*') {
+            this.appendToL2Map(this.codeDeltaMap, req.code, req.table, link);
+        } else {
+            if (req.payer !== '' && req.payer !== '*') {
+                this.appendToL1Map(this.payerMap, req.payer, link);
+            } else {
+                return {status: 'FAIL', reason: 'invalid request'};
+            }
+        }
+        this.addToClientIndex(data, id, [req.code, req.table, req.payer]);
+        return {
+            status: 'OK'
+        };
+    }
 
-	removeLinks(id) {
-		// console.log(`Removing links for ${id}...`);
-		if (this.clientIndex.has(id)) {
-			const links = this.clientIndex.get(id);
-			links.forEach((path, key) => {
-				this.removeDeepLinks(this.codeActionMap, path, key, id);
-				this.removeDeepLinks(this.codeDeltaMap, path, key, id);
-				this.removeSingleLevelLinks(this.notifiedMap, path, key, id);
-				this.removeSingleLevelLinks(this.payerMap, path, key, id);
-			});
-		}
-	}
+    removeDeepLinks(map, path, key, id) {
+        if (map.has(path[0])) {
+            if (map.get(path[0]).has(path[1])) {
+                const currentLinks = map.get(path[0]).get(path[1]).links;
+                currentLinks.forEach((item, index) => {
+                    if (item.relay === key && item.client === id) {
+                        currentLinks.splice(index, 1);
+                    }
+                });
+            }
+        }
+    }
 
-	initRoutingServer() {
-		const server = createServer();
-		this.io = new Server(server, {
-			path: '/router',
-			serveClient: false,
-			cookie: false
-		});
+    removeSingleLevelLinks(map, path, key, id) {
+        if (map.has(path[2])) {
+            const _links = map.get(path[2]).links;
+            _links.forEach((item, index) => {
+                if (item.relay === key && item.client === id) {
+                    _links.splice(index, 1);
+                }
+            });
+        }
+    }
 
-		this.io.on('connection', (socket: Socket) => {
-			console.log(`[ROUTER] New relay connected with ID = ${socket.id}`);
-			this.relays[socket.id] = {clients: 0, connected: true};
-			socket.on('event', (data, callback) => {
-				switch (data.type) {
-					case 'client_count': {
-						this.relays[socket.id]['clients'] = data.counter;
-						this.countClients();
-						break;
-					}
-					case 'action_request': {
-						const result = this.addActionRequest(data, socket.id);
-						if (result.status === 'OK') {
-							callback(result);
-						} else {
-							callback(result.reason);
-						}
-						break;
-					}
-					case 'delta_request': {
-						const result = this.addDeltaRequest(data, socket.id);
-						if (result.status === 'OK') {
-							callback(result);
-						} else {
-							callback(result.reason);
-						}
-						break;
-					}
-					case 'client_disconnected': {
-						this.removeLinks(data.id);
-						break;
-					}
-					default: {
-						console.log(data);
-					}
-				}
-			});
-			socket.on('disconnect', () => {
-				this.relays[socket.id].connected = false;
-				this.countClients();
-			});
-		});
+    removeLinks(id) {
+        // console.log(`Removing links for ${id}...`);
+        if (this.clientIndex.has(id)) {
+            const links = this.clientIndex.get(id);
+            links.forEach((path, key) => {
+                this.removeDeepLinks(this.codeActionMap, path, key, id);
+                this.removeDeepLinks(this.codeDeltaMap, path, key, id);
+                this.removeSingleLevelLinks(this.notifiedMap, path, key, id);
+                this.removeSingleLevelLinks(this.payerMap, path, key, id);
+            });
+        }
+    }
 
-		const connOpts = this.manager.conn.chains[this.chain];
+    initRoutingServer() {
+        const server = createServer();
+        this.io = new Server(server, {
+            path: '/router',
+            serveClient: false,
+            cookie: false
+        });
 
-		let _port = 57200;
-		if (connOpts.WS_ROUTER_PORT) {
-			_port = connOpts.WS_ROUTER_PORT;
-		}
+        this.io.on('connection', (socket: Socket) => {
+            console.log(`[ROUTER] New relay connected with ID = ${socket.id}`);
+            this.relays[socket.id] = {clients: 0, connected: true};
+            socket.on('event', (data, callback) => {
+                switch (data.type) {
+                    case 'client_count': {
+                        this.relays[socket.id]['clients'] = data.counter;
+                        this.countClients();
+                        break;
+                    }
+                    case 'action_request': {
+                        const result = this.addActionRequest(data, socket.id);
+                        if (result.status === 'OK') {
+                            callback(result);
+                        } else {
+                            callback(result.reason);
+                        }
+                        break;
+                    }
+                    case 'delta_request': {
+                        const result = this.addDeltaRequest(data, socket.id);
+                        if (result.status === 'OK') {
+                            callback(result);
+                        } else {
+                            callback(result.reason);
+                        }
+                        break;
+                    }
+                    case 'client_disconnected': {
+                        this.removeLinks(data.id);
+                        break;
+                    }
+                    default: {
+                        console.log(data);
+                    }
+                }
+            });
+            socket.on('disconnect', () => {
+                this.relays[socket.id].connected = false;
+                this.countClients();
+            });
+        });
 
-		let _host = "127.0.0.1";
-		if (connOpts.WS_ROUTER_HOST) {
-			_host = connOpts.WS_ROUTER_HOST;
-		}
+        const connOpts = this.manager.conn.chains[this.chain];
 
-		server.listen(_port, _host, () => {
-			this.ready();
-			setTimeout(() => {
-				if (!this.firstData) {
-					this.ready();
-				}
-			}, 5000);
-		});
+        let _port = 57200;
+        if (connOpts.WS_ROUTER_PORT) {
+            _port = connOpts.WS_ROUTER_PORT;
+        }
 
-	}
+        let _host = "127.0.0.1";
+        if (connOpts.WS_ROUTER_HOST) {
+            _host = connOpts.WS_ROUTER_HOST;
+        }
 
-	ready() {
-		process.send({event: 'router_ready'});
-	}
+        server.listen(_port, _host, () => {
+            this.ready();
+            setTimeout(() => {
+                if (!this.firstData) {
+                    this.ready();
+                }
+            }, 5000);
+        });
 
-	private forwardActionMessage(msg: any, link: any, notified: string[]) {
-		let allow = false;
-		const relay = this.io.of('/').sockets.get(link.relay);
-		if (relay) {
-			if (link.account !== '') {
-				allow = notified.indexOf(link.account) !== -1;
-			} else {
-				allow = true;
-			}
-			if (link.filters?.length > 0) {
-				// check filters
-				const _parsedMsg = JSON.parse(msg);
-				allow = link.filters.every(filter => {
-					return checkFilter(filter, _parsedMsg);
-				});
-			}
-			if (allow) {
-				relay.emit('trace', {client: link.client, message: msg});
-				this.totalRoutedMessages++;
-			}
-		}
-	}
+    }
 
-	private forwardDeltaMessage(msg, link, payer) {
-		let allow = false;
-		const relay = this.io.of('/').sockets.get(link.relay);
-		if (relay) {
-			if (link.payer !== '') {
-				allow = link.payer === payer;
-			} else {
-				allow = true;
-			}
-			// if (link.filters?.length > 0) {
-			//     // check filters
-			//     const _parsedMsg = JSON.parse(msg);
-			//     allow = link.filters.every(filter => {
-			//         return checkDeltaFilter(filter, _parsedMsg);
-			//     });
-			// }
-			if (allow) {
-				relay.emit('delta', {client: link.client, message: msg});
-				this.totalRoutedMessages++;
-			}
-		}
-	}
+    ready() {
+        process.send({event: 'router_ready'});
+    }
+
+    private forwardActionMessage(msg: any, link: any, notified: string[]) {
+        let allow = false;
+        const relay = this.io.of('/').sockets.get(link.relay);
+        if (relay) {
+            if (link.account !== '') {
+                allow = notified.indexOf(link.account) !== -1;
+            } else {
+                allow = true;
+            }
+            if (link.filters?.length > 0) {
+                // check filters
+                const _parsedMsg = JSON.parse(msg);
+                allow = link.filters.every(filter => {
+                    return checkFilter(filter, _parsedMsg);
+                });
+            }
+            if (allow) {
+                relay.emit('trace', {client: link.client, message: msg});
+                this.totalRoutedMessages++;
+            }
+        }
+    }
+
+    private forwardDeltaMessage(msg, link, payer) {
+        let allow = false;
+        const relay = this.io.of('/').sockets.get(link.relay);
+        if (relay) {
+            if (link.payer !== '') {
+                allow = link.payer === payer;
+            } else {
+                allow = true;
+            }
+            // if (link.filters?.length > 0) {
+            //     // check filters
+            //     const _parsedMsg = JSON.parse(msg);
+            //     allow = link.filters.every(filter => {
+            //         return checkDeltaFilter(filter, _parsedMsg);
+            //     });
+            // }
+            if (allow) {
+                relay.emit('delta', {client: link.client, message: msg});
+                this.totalRoutedMessages++;
+            }
+        }
+    }
 }