C9S/explorer-api
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/eosrio/hyperion-history-api into main

Browse Source
This commit is contained in:
Bruno Leal
2023-09-25 15:07:17 -03:00
parent b7e17f9dc7 b94f99d552
commit fb2fc20d1d
8 changed files with 79 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
connections/amqp.ts
+8 -9
View File
@@ -17,17 +17,16 @@ export async function createConnection(config): Promise<Connection> {
}
export function getAmpqUrl(config): string {
let frameMaxValue = '0x10000';
if (config.frameMax) {
frameMaxValue = config.frameMax;
}
const u = encodeURIComponent(config.user);
const p = encodeURIComponent(config.pass);
const v = encodeURIComponent(config.vhost)
return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
let frameMaxValue = '0x10000';
if (config.frameMax) {
frameMaxValue = config.frameMax;
}
const u = encodeURIComponent(config.user);
const p = encodeURIComponent(config.pass);
const v = encodeURIComponent(config.vhost);
return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
}
async function createChannels(connection) {
try {
const channel = await connection.createChannel();
connections/manager.class.ts
+2 -3
View File
@@ -9,7 +9,6 @@ import {StateHistorySocket} from "./state-history";
import fetch from 'cross-fetch';
import {exec} from "child_process";
import {hLog} from "../helpers/common_functions";
import {readFileSync} from "fs";
export class ConnectionManager {
@@ -20,8 +19,8 @@ export class ConnectionManager {
last_commit_hash: string;
current_version: string;
private readonly esIngestClients: Client[];
private esIngestClient: Client;
esIngestClients: Client[];
esIngestClient: Client;
constructor(private cm: ConfigurationModule) {
this.config = cm.config;
ecosystem.config.js
+7
View File
@@ -20,4 +20,11 @@ readdirSync(chainsRoot)
}
});
apps.push({
name: 'hyperion-governor',
namespace: 'hyperion',
script: 'governor/server/index.js',
watch: false,
});
module.exports = {apps};
modules/parsers/base-parser.ts
-1
View File
@@ -192,7 +192,6 @@ export abstract class BaseParser {
if (ds_act) {
if (ds_act.account && ds_act.name && ds_act.authorization) {
console.log(ds_act);
action.act.data = ds_act.data;
}
package-lock.json
Generated
+16 -24
View File
@@ -44,11 +44,11 @@
"socket.io-redis": "^6.1.1",
"telegraf": "^4.12.3-canary.1",
"typescript": "^4.5.2",
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.24.0",
"ws": "^8.12.1"
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.31.0",
"ws": "^8.14.2"
},
"devDependencies": {
"@types/amqplib": "^0.10.1",
"@types/amqplib": "^0.10.2",
"@types/async": "^3.2.18",
"@types/global-agent": "^2.1.1",
"@types/ioredis": "^4.28.10",
@@ -242,9 +242,9 @@
}
},
"node_modules/@pm2/io": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/@pm2/io/-/io-5.0.0.tgz",
"integrity": "sha512-3rToDVJaRoob5Lq8+7Q2TZFruoEkdORxwzFpZaqF4bmH6Bkd7kAbdPrI/z8X6k1Meq5rTtScM7MmDgppH6aLlw==",
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/@pm2/io/-/io-5.0.2.tgz",
"integrity": "sha512-XAvrNoQPKOyO/jJyCu8jPhLzlyp35MEf7w/carHXmWKddPzeNOFSEpSEqMzPDawsvpxbE+i918cNN+MwgVsStA==",
"dependencies": {
"@opencensus/core": "0.0.9",
"@opencensus/propagation-b3": "0.0.8",
@@ -252,7 +252,7 @@
"debug": "~4.3.1",
"eventemitter2": "^6.3.1",
"require-in-the-middle": "^5.0.0",
"semver": "6.3.0",
"semver": "~7.5.4",
"shimmer": "^1.2.0",
"signal-exit": "^3.0.3",
"tslib": "1.9.3"
@@ -269,14 +269,6 @@
"lodash": "^4.17.14"
}
},
"node_modules/@pm2/io/node_modules/semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
"bin": {
"semver": "bin/semver.js"
}
},
"node_modules/@sindresorhus/is": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz",
@@ -305,9 +297,9 @@
}
},
"node_modules/@types/amqplib": {
"version": "0.10.1",
"resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.1.tgz",
"integrity": "sha512-j6ANKT79ncUDnAs/+9r9eDujxbeJoTjoVu33gHHcaPfmLQaMhvfbH2GqSe8KUM444epAp1Vl3peVOQfZk3UIqA==",
"version": "0.10.2",
"resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.2.tgz",
"integrity": "sha512-K0qC2YR0ZcQWWMOifg4yvCAu5wi/V6wY6MnMS4LSvqx6qwXBAhxno6OBN8D76FIaajLNfgHKOXobZBL/uAwXAQ==",
"dev": true,
"dependencies": {
"@types/node": "*"
@@ -2518,9 +2510,9 @@
"integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
},
"node_modules/semver": {
"version": "7.5.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz",
"integrity": "sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==",
"version": "7.5.4",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
"integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
"dependencies": {
"lru-cache": "^6.0.0"
},
@@ -2987,9 +2979,9 @@
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
},
"node_modules/ws": {
"version": "8.13.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
"integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
"version": "8.14.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
"integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
"engines": {
"node": ">=10.0.0"
},
package.json
+4 -4
View File
@@ -1,6 +1,6 @@
{
"name": "hyperion-history",
"version": "3.3.9-7",
"version": "3.3.9-8",
"description": "Scalable Full History API Solution for EOSIO based blockchains",
"main": "launcher.js",
"scripts": {
@@ -60,11 +60,11 @@
"socket.io-redis": "^6.1.1",
"telegraf": "^4.12.3-canary.1",
"typescript": "^4.5.2",
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.24.0",
"ws": "^8.12.1"
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.31.0",
"ws": "^8.14.2"
},
"devDependencies": {
"@types/amqplib": "^0.10.1",
"@types/amqplib": "^0.10.2",
"@types/async": "^3.2.18",
"@types/global-agent": "^2.1.1",
"@types/ioredis": "^4.28.10",
workers/deserializer.ts
+40 -8
View File
@@ -8,6 +8,7 @@ import {Type} from "../addons/eosjs-native/eosjs-serialize";
import {debugLog, hLog} from "../helpers/common_functions";
import {createHash} from "crypto";
import flatstr from 'flatstr';
import {Options} from "amqplib";
const FJS = require('fast-json-stringify');
@@ -124,6 +125,9 @@ export default class MainDSWorker extends HyperionWorker {
allowedDynamicContracts: Set<string> = new Set<string>();
backpressureQueue: any[] = [];
waitToSend = false;
constructor() {
super();
@@ -290,6 +294,16 @@ export default class MainDSWorker extends HyperionWorker {
this.ch.consume(process.env['worker_queue'], (data) => {
this.consumerQueue.push(data).catch(console.log);
});
this.ch.on('drain', args => {
this.waitToSend = false;
while (this.backpressureQueue.length > 0) {
const msg = this.backpressureQueue.shift();
const status = this.controlledSendToQueue(msg.queue, msg.payload, msg.options);
if (!status) {
break;
}
}
});
}
}
@@ -478,6 +492,7 @@ export default class MainDSWorker extends HyperionWorker {
hLog(`${block_num} was filtered with ${inline_count} actions!`);
}
try {
trace[1].signatures = signatures;
this.routeToPool(trace[1], {
block_num,
block_id,
@@ -485,8 +500,7 @@ export default class MainDSWorker extends HyperionWorker {
ts,
inline_count,
filtered,
live: process.env['live_mode'],
signatures
live: process.env['live_mode']
});
} catch (e) {
hLog(e);
@@ -622,15 +636,33 @@ export default class MainDSWorker extends HyperionWorker {
}
const pool_queue = `${this.chain}:ds_pool:${selected_q}`;
if (this.ch_ready) {
// console.log('selected_q', pool_queue);
this.ch.sendToQueue(pool_queue, bufferFromJson(trace, true), {headers});
return true;
const payload = bufferFromJson(trace, true);
if (!this.waitToSend) {
if (this.ch_ready) {
this.controlledSendToQueue(pool_queue, payload, {headers});
return true;
} else {
return false;
}
} else {
this.backpressureQueue.push({
queue: pool_queue,
payload: payload,
options: {headers}
});
return false;
}
}
controlledSendToQueue(pool_queue: string, payload: Buffer, options: Options.Publish): boolean {
const enqueueResult = this.ch.sendToQueue(pool_queue, payload, options);
if (!enqueueResult) {
this.waitToSend = true;
}
return enqueueResult;
}
createSerialBuffer(inputArray) {
return new Serialize.SerialBuffer({textEncoder: this.txEnc, textDecoder: this.txDec, array: inputArray});
}
@@ -1160,7 +1192,7 @@ export default class MainDSWorker extends HyperionWorker {
let jsonRow = await this.processContractRowNative(payload, block_num);
if (jsonRow?.value && !jsonRow['_blacklisted']) {
console.log(jsonRow);
debugLog(jsonRow);
debugLog('Delta DS failed ->>', jsonRow);
jsonRow = await this.processContractRowNative(payload, block_num - 1);
debugLog('Retry with previous ABI ->>', jsonRow);
@@ -1495,7 +1527,7 @@ export default class MainDSWorker extends HyperionWorker {
} catch (e) {
hLog(`Delta struct [${key}] processing error: ${e.message}`);
hLog(e);
console.log(data[1]);
hLog(data[1]);
}
}
}
workers/ds-pool.ts
+2 -2
View File
@@ -454,8 +454,8 @@ export default class DSPoolWorker extends HyperionWorker {
}
async processTraces(transaction_trace, extra) {
const {cpu_usage_us, net_usage_words} = transaction_trace;
const {block_num, block_id, producer, ts, inline_count, filtered, live, signatures} = extra;
const {cpu_usage_us, net_usage_words, signatures} = transaction_trace;
const {block_num, block_id, producer, ts, inline_count, filtered, live} = extra;
if (transaction_trace.status === 0) {
let action_count = 0;