Update package.json

TTL is configurable via `api.tx_cache_expiration_sec`

api/routes.ts

@@ -5,6 +5,7 @@ import {ServerResponse} from "http";
 import {createReadStream, existsSync, readFileSync, unlinkSync} from "fs";
 import * as AutoLoad from "fastify-autoload";
 import {addSharedSchemas, handleChainApiRedirect} from "./helpers/functions";
+import got from "got";
 
 function addRedirect(server: FastifyInstance, url: string, redirectTo: string) {
     server.route({
@@ -27,6 +28,24 @@ function addRoute(server: FastifyInstance, handlersPath: string, prefix: string)
 
 export function registerRoutes(server: FastifyInstance) {
 
+    // build internal map of routes
+    const routeSet = new Set<string>();
+    server.decorate('routeSet', routeSet);
+    const ignoreList = [
+        '/v2',
+        '/v2/history',
+        '/v2/state',
+        '/v1/chain/*',
+        '/v1/chain',
+    ];
+    server.addHook('onRoute', opts => {
+        if (!ignoreList.includes(opts.url)) {
+            if (opts.url.startsWith('/v') && !opts.url.startsWith('/v2/explore') && !opts.url.startsWith('/v2/docs')) {
+                routeSet.add(opts.url);
+            }
+        }
+    });
+
     // Register fastify api routes
     addRoute(server, 'v2', '/v2');
     addRoute(server, 'v2-history', '/v2/history');
@@ -53,6 +72,26 @@ export function registerRoutes(server: FastifyInstance) {
         }
     });
 
+    // /v1/node/get_supported_apis
+    server.route({
+        url: '/v1/node/get_supported_apis',
+        method: ["GET"],
+        schema: {
+            summary: "Get list of supported APIs",
+            tags: ["node"]
+        },
+        handler: async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+            const data = await got.get(`${server.chain_api}/v1/node/get_supported_apis`).json() as any;
+            if (data.apis && data.apis.length > 0) {
+                const apiSet = new Set(server.routeSet);
+                data.apis.forEach((a) => apiSet.add(a));
+                reply.send({apis: [...apiSet]});
+            } else {
+                reply.send({apis: [...server.routeSet], error: 'nodeos did not send any data'});
+            }
+        }
+    });
+
     server.addHook('onError', (request, reply, error, done) => {
         console.log(`[${request.req.headers['x-real-ip']}] ${request.req.method} ${request.req.url} failed with error: ${error.message}`);
         done();

api/routes/v2-history/get_transaction/get_transaction.ts

@@ -3,88 +3,111 @@ import {ServerResponse} from "http";
 import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
 
 async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
-    const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
 
-    let indexPattern = fastify.manager.chain + '-action-*';
-    if (request.query.hot_only) {
-        indexPattern = fastify.manager.chain + '-action';
-    }
+	const redis = fastify.redis;
+	const trxId = request.query.id.toLowerCase();
+	const cachedData = await redis.hgetall(trxId);
 
-    const pResults = await Promise.all([
-        fastify.eosjs.rpc.get_info(),
-        fastify.elastic.search({
-            index: indexPattern,
-            size: _size,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {term: {trx_id: request.query.id.toLowerCase()}}
-                        ]
-                    }
-                },
-                sort: {
-                    global_sequence: "asc"
-                }
-            }
-        }),
-        fastify.elastic.search({
-            index: fastify.manager.chain + '-gentrx-*',
-            size: _size,
-            body: {
-                query: {
-                    bool: {
-                        must: [
-                            {term: {trx_id: request.query.id.toLowerCase()}}
-                        ]
-                    }
-                }
-            }
-        })
-    ]);
+	let hits;
+	let hits2;
 
-    const results = pResults[1];
-    const genTrxRes = pResults[2];
+	const response = {
+		"query_time_ms": undefined,
+		"cached": undefined,
+		"cache_expires_in": undefined,
+		"executed": false,
+		"hot_only": false,
+		"trx_id": trxId,
+		"lib": undefined,
+		"actions": [],
+		"generated": undefined
+	};
 
-    const response = {
-        "executed": false,
-        "hot_only": false,
-        "trx_id": request.query.id,
-        "lib": pResults[0].last_irreversible_block_num,
-        "actions": [],
-        "generated": undefined
-    };
+	if (cachedData && Object.keys(cachedData).length > 0) {
+		const gsArr = [];
+		for (let cachedDataKey in cachedData) {
+			gsArr.push(cachedData[cachedDataKey]);
+		}
+		gsArr.sort((a, b) => {
+			return a.global_sequence - b.global_sequence;
+		});
+		response.cache_expires_in = await redis.ttl(trxId);
+		hits = gsArr.map(value => {
+			return {_source: JSON.parse(value)}
+		});
+	}
 
-    if (request.query.hot_only) {
-        response.hot_only = true;
-    }
+	if (!hits) {
+		const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
+		let indexPattern = fastify.manager.chain + '-action-*';
+		if (request.query.hot_only) {
+			indexPattern = fastify.manager.chain + '-action';
+		}
 
-    const hits = results['body']['hits']['hits'];
+		const pResults = await Promise.all([
+			fastify.eosjs.rpc.get_info(),
+			fastify.elastic.search({
+				index: indexPattern,
+				size: _size,
+				body: {
+					query: {bool: {must: [{term: {trx_id: request.query.id.toLowerCase()}}]}},
+					sort: {global_sequence: "asc"}
+				}
+			}),
+			fastify.elastic.search({
+				index: fastify.manager.chain + '-gentrx-*',
+				size: _size,
+				body: {
+					query: {bool: {must: [{term: {trx_id: request.query.id.toLowerCase()}}]}}
+				}
+			})
+		]);
 
-    if (hits.length > 0) {
-        for (let action of hits) {
-            action = action._source;
-            mergeActionMeta(action);
-            response.actions.push(action);
-        }
-        response.executed = true;
-    }
+		const results = pResults[1];
+		const genTrxRes = pResults[2];
 
-    const hits2 = genTrxRes['body']['hits']['hits'];
+		response.lib = pResults[0].last_irreversible_block_num;
 
-    if (hits2 && hits2.length > 0) {
-        if (hits2[0]._source['@timestamp']) {
-            hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
-            delete hits2[0]._source['@timestamp'];
-        }
-        response.generated = hits2[0]._source;
-    }
+		if (request.query.hot_only) {
+			response.hot_only = true;
+		}
 
-    return response;
+		hits = results['body']['hits']['hits'];
+		hits2 = genTrxRes['body']['hits']['hits'];
+	}
+
+	if (hits) {
+		if (hits.length > 0) {
+			let highestBlockNum = 0;
+			for (let action of hits) {
+				if (action._source.block_num > highestBlockNum) {
+					highestBlockNum = action._source.block_num;
+				}
+			}
+			for (let action of hits) {
+				if (action._source.block_num === highestBlockNum) {
+					action = action._source;
+					mergeActionMeta(action);
+					response.actions.push(action);
+				}
+			}
+			response.executed = true;
+		}
+	}
+
+	if (hits2 && hits2.length > 0) {
+		if (hits2[0]._source['@timestamp']) {
+			hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
+			delete hits2[0]._source['@timestamp'];
+		}
+		response.generated = hits2[0]._source;
+	}
+
+	return response;
 }
 
 export function getTransactionHandler(fastify: FastifyInstance, route: string) {
-    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
-        reply.send(await timedQuery(getTransaction, fastify, request, route));
-    }
+	return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+		reply.send(await timedQuery(getTransaction, fastify, request, route));
+	}
 }

api/routes/v2-history/get_transaction_legacy/get_transaction_legacy.ts

@@ -0,0 +1,128 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {ServerResponse} from "http";
+import {mergeActionMeta, timedQuery} from "../../../helpers/functions";
+
+async function getTransaction(fastify: FastifyInstance, request: FastifyRequest) {
+    const _size = fastify.manager.config.api.limits.get_trx_actions || 100;
+
+    let indexPattern = fastify.manager.chain + '-action-*';
+    if (request.query.hot_only) {
+        indexPattern = fastify.manager.chain + '-action';
+    }
+
+    const pResults = await Promise.all([
+        fastify.eosjs.rpc.get_info(),
+        fastify.elastic.search({
+            index: indexPattern,
+            size: _size,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {term: {trx_id: request.query.id.toLowerCase()}}
+                        ]
+                    }
+                },
+                sort: {
+                    global_sequence: "asc"
+                }
+            }
+        }),
+        fastify.elastic.search({
+            index: fastify.manager.chain + '-gentrx-*',
+            size: _size,
+            body: {
+                query: {
+                    bool: {
+                        must: [
+                            {term: {trx_id: request.query.id.toLowerCase()}}
+                        ]
+                    }
+                }
+            }
+        })
+    ]);
+
+    const results = pResults[1];
+    const genTrxRes = pResults[2];
+
+    const response = {
+        "executed": false,
+        "hot_only": false,
+        "trx_id": request.query.id,
+        "lib": pResults[0].last_irreversible_block_num,
+        "actions": [],
+        "generated": undefined
+    };
+
+    if (request.query.hot_only) {
+        response.hot_only = true;
+    }
+
+    const hits = results['body']['hits']['hits'];
+
+    if (hits.length > 0) {
+
+        // const producers = {};
+        // for (let hit of hits) {
+        // 	if (hit._source.producer) {
+        // 		if (producers[hit._source.producer]) {
+        // 			producers[hit._source.producer]++;
+        // 		} else {
+        // 			producers[hit._source.producer] = 1;
+        // 		}
+        // 	}
+        // }
+        // let useBlocknumber;
+        // if (Object.keys(producers).length > 1) {
+        // 	// multiple producers of the same tx id, forked actions are present, attempt to cleanup
+        // 	let trueProd = '';
+        // 	let highestActCount = 0;
+        // 	for (const prod in producers) {
+        // 		if (producers.hasOwnProperty(prod)) {
+        // 			if(producers[prod] === highestActCount) {
+        // 				useBlocknumber = true;
+        // 			} else if (producers[prod] > highestActCount) {
+        // 				highestActCount = producers[prod];
+        // 				trueProd = prod;
+        // 			}
+        // 		}
+        // 	}
+        // }
+
+        let highestBlockNum = 0;
+        for (let action of hits) {
+            if (action._source.block_num > highestBlockNum) {
+                highestBlockNum = action._source.block_num;
+            }
+        }
+
+        for (let action of hits) {
+            if (action._source.block_num === highestBlockNum) {
+                action = action._source;
+                mergeActionMeta(action);
+                response.actions.push(action);
+            }
+        }
+
+        response.executed = true;
+    }
+
+    const hits2 = genTrxRes['body']['hits']['hits'];
+
+    if (hits2 && hits2.length > 0) {
+        if (hits2[0]._source['@timestamp']) {
+            hits2[0]._source['timestamp'] = hits2[0]._source['@timestamp'];
+            delete hits2[0]._source['@timestamp'];
+        }
+        response.generated = hits2[0]._source;
+    }
+
+    return response;
+}
+
+export function getTransactionHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply<ServerResponse>) => {
+        reply.send(await timedQuery(getTransaction, fastify, request, route));
+    }
+}

api/routes/v2-history/get_transaction_legacy/index.ts

@@ -0,0 +1,29 @@
+import {FastifyInstance} from "fastify";
+import {getTransactionHandler} from "./get_transaction_legacy";
+import {addApiRoute, getRouteName} from "../../../helpers/functions";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get all actions belonging to the same transaction',
+        summary: 'get transaction by id',
+        tags: ['history'],
+        querystring: {
+            type: 'object',
+            properties: {
+                "id": {
+                    description: 'transaction id',
+                    type: 'string'
+                }
+            },
+            required: ["id"]
+        }
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getTransactionHandler,
+        schema
+    );
+    next();
+}

chains/example.config.json

@@ -22,7 +22,8 @@
     "enable_explorer": false,
     "chain_api_error_log": false,
     "custom_core_token": "",
-    "enable_export_action": false
+    "enable_export_action": false,
+    "tx_cache_expiration_sec": 3600
   },
   "settings": {
     "preview": false,

connections/amqp.ts

@@ -17,10 +17,14 @@ export async function createConnection(config): Promise<Connection> {
 }
 
 export function getAmpqUrl(config): string {
+	let frameMaxValue = '0x10000';
+	if (config.frameMax) {
+		frameMaxValue = config.frameMax;
+	}
 	const u = encodeURIComponent(config.user);
 	const p = encodeURIComponent(config.pass);
 	const v = encodeURIComponent(config.vhost)
-	return `amqp://${u}:${p}@${config.host}/${v}`;
+	return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
 }
 
 
@@ -76,7 +80,9 @@ export async function checkQueueSize(q_name, config) {
 	} catch (e) {
 		hLog('[WARNING] Checking queue size failed, HTTP API is not ready!');
 		if (e instanceof HTTPError) {
-			hLog(e.response);
+			if(e.response.body) {
+				hLog(e.response.body);
+			}
 		} else {
 			hLog(JSON.stringify(e, null, 2));
 		}

connections/manager.class.ts

@@ -6,7 +6,7 @@ import {HyperionConnections} from "../interfaces/hyperionConnections";
 import {HyperionConfig} from "../interfaces/hyperionConfig";
 import {amqpConnect, checkQueueSize, getAmpqUrl} from "./amqp";
 import {StateHistorySocket} from "./state-history";
-import fetch from 'node-fetch';
+import fetch from 'cross-fetch';
 import {exec} from "child_process";
 import {hLog} from "../helpers/common_functions";
 

interfaces/hyperionConfig.ts

@@ -72,6 +72,7 @@ interface ApiLimits {
 }
 
 interface ApiConfigs {
+	tx_cache_expiration_sec?: number;
     custom_core_token: string;
     chain_api_error_log?: boolean;
     chain_api?: string;

interfaces/hyperionConnections.ts

@@ -1,37 +1,38 @@
 interface AmqpConfig {
-    host: string;
-    api: string;
-    user: string;
-    pass: string;
-    vhost: string;
+	host: string;
+	api: string;
+	user: string;
+	pass: string;
+	vhost: string;
+	frameMax: string;
 }
 
 interface ESConfig {
-    protocol: string;
-    host: string;
-    ingest_nodes: string[];
-    user: string;
-    pass: string;
+	protocol: string;
+	host: string;
+	ingest_nodes: string[];
+	user: string;
+	pass: string;
 }
 
 interface HyperionChainData {
-    name: string;
-    chain_id: string;
-    http: string;
-    WS_ROUTER_PORT: number;
-    WS_ROUTER_HOST: string;
+	name: string;
+	chain_id: string;
+	http: string;
+	WS_ROUTER_PORT: number;
+	WS_ROUTER_HOST: string;
 }
 
 interface RedisConfig {
-    host: string;
-    port: number;
+	host: string;
+	port: number;
 }
 
 export interface HyperionConnections {
-    amqp: AmqpConfig;
-    elasticsearch: ESConfig;
-    redis: RedisConfig;
-    chains: {
-        [key: string]: HyperionChainData
-    }
+	amqp: AmqpConfig;
+	elasticsearch: ESConfig;
+	redis: RedisConfig;
+	chains: {
+		[key: string]: HyperionChainData
+	}
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperion-history",
-  "version": "3.1.2",
+  "version": "3.1.4",
   "description": "Scalable Full History API Solution for EOSIO based blockchains",
   "main": "launcher.js",
   "scripts": {
@@ -9,6 +9,7 @@
     "start:indexer": "pm2 start --only Indexer --update-env",
     "fix-permissions": "chmod u+x run.sh stop.sh install_env.sh",
     "tsc": "tsc",
+    "build": "tsc",
     "postinstall": "npm run tsc && npm run fix-permissions"
   },
   "author": {
@@ -21,19 +22,20 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@elastic/elasticsearch": "^7.9.0",
-    "@eosrio/node-abieos": "2.1.0-beta.0",
-    "@pm2/io": "4.3.5",
-    "amqplib": "0.6.0",
-    "async": "3.2.0",
-    "async-redis": "^1.1.7",
-    "eosjs": "21.0.3",
+    "@elastic/elasticsearch": "^7.13.0",
+    "@eosrio/node-abieos": "^2.1.1",
+    "@pm2/io": "^5.0.0",
+    "amqplib": "^0.8.0",
+    "async": "^3.2.0",
+    "async-redis": "^2.0.0",
+    "cross-fetch": "^3.1.4",
+    "eosjs": "^22.1.0",
     "eosjs-ecc": "4.0.7",
     "fast-json-parse": "1.0.3",
     "fast-json-stringify": "2.1.0",
     "fast-proxy": "^1.6.2",
     "fastbench": "^1.0.1",
-    "fastify": "2.14.1",
+    "fastify": "2.15.3",
     "fastify-autoload": "1.2.2",
     "fastify-compress": "2.0.1",
     "fastify-cors": "3.0.3",
@@ -47,27 +49,26 @@
     "fastify-websocket": "1.1.2",
     "flatstr": "1.0.12",
     "got": "^11.8.1",
-    "ioredis": "4.17.3",
-    "lodash": "4.17.20",
+    "ioredis": "^4.27.6",
+    "lodash": "^4.17.21",
     "moment": "2.27.0",
-    "node-fetch": "3.0.0-beta.8",
     "portfinder": "^1.0.26",
-    "redis": "3.0.2",
-    "socket.io": "2.3.0",
-    "socket.io-client": "2.3.0",
+    "redis": "^3.1.2",
+    "socket.io": "2.4.1",
+    "socket.io-client": "^2.4.0",
     "typescript": "3.9.5",
-    "ws": "7.3.1",
+    "ws": "^7.5.3",
     "yargs": "15.4.1"
   },
   "devDependencies": {
-    "@types/amqplib": "0.5.13",
-    "@types/async": "3.2.3",
-    "@types/got": "9.6.11",
-    "@types/ioredis": "4.16.6",
-    "@types/lodash": "4.14.155",
-    "@types/node": "14.0.13",
+    "@types/amqplib": "^0.8.1",
+    "@types/async": "3.2.0",
+    "@types/got": "^9.6.12",
+    "@types/ioredis": "^4.26.6",
+    "@types/lodash": "^4.14.171",
+    "@types/node": "^15.6.1",
     "@types/socket.io": "2.1.8",
     "@types/socket.io-client": "1.4.33",
-    "@types/ws": "7.2.5"
+    "@types/ws": "^7.4.7"
   }
 }

plugins/hyperion-plugin.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HyperionPlugin = void 0;
+class HyperionPlugin {
+    constructor() {
+        this.dynamicContracts = [];
+    }
+}
+exports.HyperionPlugin = HyperionPlugin;
+//# sourceMappingURL=hyperion-plugin.js.map

plugins/hyperion-plugin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hyperion-plugin.js","sourceRoot":"","sources":["hyperion-plugin.ts"],"names":[],"mappings":";;;AAEA,MAAsB,cAAc;IAApC;QACC,qBAAgB,GAAa,EAAE,CAAC;IAGjC,CAAC;CAAA;AAJD,wCAIC"}

tsconfig.json

@@ -15,6 +15,7 @@
   "exclude": [
     "node_modules",
     "hyperion-explorer",
-    "build"
+    "build",
+    "plugins"
   ]
 }

types/fastify/index.d.ts

@@ -19,6 +19,7 @@ declare module 'fastify' {
         },
         chain_api: string,
         push_api: string,
-        tokenCache: Map<string, any>
+        tokenCache: Map<string, any>,
+        routeSet: Set<string>
     }
 }

workers/ds-pool.ts

@@ -8,6 +8,7 @@ import {parseDSPEvent} from "../modules/custom/dsp-parser";
 import {join, resolve} from "path";
 import {existsSync, readdirSync, readFileSync} from "fs";
 import * as flatstr from 'flatstr';
+import IORedis from "ioredis";
 
 const abi_remapping = {
 	"_Bool": "bool",
@@ -43,10 +44,13 @@ export default class DSPoolWorker extends HyperionWorker {
 
 	customAbiMap: Map<string, CustomAbiDef[]> = new Map();
 	private noActionCounter = 0;
+	private ioRedisClient: IORedis.Redis;
 
 	constructor() {
 		super();
 
+		this.ioRedisClient = new IORedis(this.manager.conn.redis);
+
 		this.consumerQueue = cargo((payload: any[], cb) => {
 			this.processMessages(payload).catch((err) => {
 				hLog('NackAll:', err);
@@ -326,7 +330,8 @@ export default class DSPoolWorker extends HyperionWorker {
 		for (const {name, type} of abi.actions) {
 			try {
 				actions.set(name, Serialize.getType(types, type));
-			} catch {}
+			} catch {
+			}
 		}
 
 		const result = {types, actions, tables: abi.tables};
@@ -508,6 +513,9 @@ export default class DSPoolWorker extends HyperionWorker {
 
 			// Submit Actions after deduplication
 			// hLog(_finalTraces.length);
+
+			const redisPayload = new Map<string, IORedis.ValueType>();
+
 			for (const uniqueAction of _finalTraces) {
 
 				// let tref = process.hrtime.bigint();
@@ -520,12 +528,25 @@ export default class DSPoolWorker extends HyperionWorker {
 				// console.log(`flatstr improvement (uniqueAction): ${(((t1 / t2) * 100) - 100).toFixed(2)}% | size: ${payload.length}`);
 
 				const payload = Buffer.from(flatstr(JSON.stringify(uniqueAction)));
+				redisPayload.set(uniqueAction.global_sequence, payload);
+
 				this.actionDsCounter++;
 				this.pushToActionsQueue(payload, block_num);
 				if (live === 'true') {
 					this.pushToActionStreamingQueue(payload, uniqueAction);
 				}
 			}
+
+			// save payload to redis
+			if (this.ioRedisClient) {
+				await this.ioRedisClient.hset(trx_data.trx_id, redisPayload);
+				let expires_in = 3600; // 1 hour by default
+				if (this.conf.api.tx_cache_expiration_sec) {
+					expires_in = this.conf.api.tx_cache_expiration_sec
+				}
+				await this.ioRedisClient.expire(trx_data.trx_id, expires_in);
+			}
+
 		}
 	}