C9S/explorer-api
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "include amqp-connection-manager"

Browse Source 
This reverts commit 0579da66
This commit is contained in:
Igor Lins e Silva
2023-09-25 01:05:33 -03:00
parent e804aff53c
commit e5d0126d08
13 changed files with 339 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
connections/amqp.ts
+68 -76
View File
@@ -1,21 +1,19 @@
import {debugLog, hLog} from "../helpers/common_functions"; import {debugLog, hLog} from "../helpers/common_functions";
import got, {HTTPError} from "got"; import got, {HTTPError} from "got";
import amqp, {ChannelWrapper} from "amqp-connection-manager"; import {connect, Connection} from 'amqplib';
import {IAmqpConnectionManager} from "amqp-connection-manager/dist/types/AmqpConnectionManager";
export async function createConnection(config): Promise<IAmqpConnectionManager> { export async function createConnection(config): Promise<Connection> {
try { try {
const amqp_url = getAmpqUrl(config); const amqp_url = getAmpqUrl(config);
// const conn: Connection = await connect(amqp_url); const conn: Connection = await connect(amqp_url);
const conn: IAmqpConnectionManager = amqp.connect(amqp_url); debugLog("[AMQP] connection established");
debugLog("[AMQP] connection established"); return conn;
return conn; } catch (e) {
} catch (e) { hLog("[AMQP] failed to connect!");
hLog("[AMQP] failed to connect!"); hLog(e.message);
hLog(e.message); await new Promise(resolve => setTimeout(resolve, 5000));
await new Promise(resolve => setTimeout(resolve, 5000)); return await createConnection(config);
return await createConnection(config); }
}
} }
export function getAmpqUrl(config): string { export function getAmpqUrl(config): string {
@@ -29,70 +27,64 @@ export function getAmpqUrl(config): string {
return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`; return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
} }
async function createChannels(connection: IAmqpConnectionManager) { async function createChannels(connection) {
try { try {
// const channel = await connection.createChannel(); const channel = await connection.createChannel();
const channel = connection.createChannel({ const confirmChannel = await connection.createConfirmChannel();
confirm: false return [channel, confirmChannel];
}); } catch (e) {
// const confirmChannel = await connection.createConfirmChannel(); hLog("[AMQP] failed to create channels");
const confirmChannel = connection.createChannel({ hLog(e);
confirm: true return null;
}) }
return [channel, confirmChannel];
} catch (e) {
hLog("[AMQP] failed to create channels");
hLog(e);
return null;
}
} }
export async function amqpConnect(onReconnect, config, onClose): Promise<ChannelWrapper[] | null> { export async function amqpConnect(onReconnect, config, onClose) {
let connection = await createConnection(config); let connection = await createConnection(config);
if (connection) { if (connection) {
const channels = await createChannels(connection); const channels = await createChannels(connection);
if (channels) { if (channels) {
connection.on('error', (err) => { connection.on('error', (err) => {
hLog(err.message); hLog(err.message);
}); });
connection.on('close', () => { connection.on('close', () => {
hLog('Connection closed!'); hLog('Connection closed!');
onClose(); onClose();
setTimeout(async () => { setTimeout(async () => {
hLog('Retrying in 5 seconds...'); hLog('Retrying in 5 seconds...');
const _channels = await amqpConnect(onReconnect, config, onClose); const _channels = await amqpConnect(onReconnect, config, onClose);
onReconnect(_channels); onReconnect(_channels);
return _channels; return _channels;
}, 5000); }, 5000);
}); });
return channels; return channels;
} else { } else {
return null; return null;
} }
} else { } else {
return null; return null;
} }
} }
export async function checkQueueSize(q_name, config) { export async function checkQueueSize(q_name, config) {
try { try {
const v = encodeURIComponent(config.vhost); const v = encodeURIComponent(config.vhost);
const apiUrl = `${config.protocol}://${config.api}/api/queues/${v}/${encodeURIComponent(q_name)}`; const apiUrl = `${config.protocol}://${config.api}/api/queues/${v}/${encodeURIComponent(q_name)}`;
const opts = { const opts = {
username: config.user, username: config.user,
password: config.pass password: config.pass
}; };
const data = await got.get(apiUrl, opts).json() as any; const data = await got.get(apiUrl, opts).json() as any;
return data.messages; return data.messages;
} catch (e) { } catch (e) {
hLog(`[WARNING] Checking queue size failed! - ${e.message}`); hLog(`[WARNING] Checking queue size failed! - ${e.message}`);
if (e.response && e.response.body) { if (e.response && e.response.body) {
if (e instanceof HTTPError) { if (e instanceof HTTPError) {
hLog(e.response.body); hLog(e.response.body);
} else { } else {
hLog(JSON.stringify(e.response.body, null, 2)); hLog(JSON.stringify(e.response.body, null, 2));
} }
} }
return 0; return 0;
} }
} }
connections/manager.class.ts
+1 -2
View File
@@ -9,7 +9,6 @@ import {StateHistorySocket} from "./state-history";
import fetch from 'cross-fetch'; import fetch from 'cross-fetch';
import {exec} from "child_process"; import {exec} from "child_process";
import {hLog} from "../helpers/common_functions"; import {hLog} from "../helpers/common_functions";
import {ChannelWrapper} from "amqp-connection-manager";
export class ConnectionManager { export class ConnectionManager {
@@ -132,7 +131,7 @@ export class ConnectionManager {
} }
} }
async createAMQPChannels(onReconnect, onClose): Promise<ChannelWrapper[]> { async createAMQPChannels(onReconnect, onClose) {
return await amqpConnect(onReconnect, this.conn.amqp, onClose); return await amqpConnect(onReconnect, this.conn.amqp, onClose);
} }
ecosystem.config.js
+7
View File
@@ -20,4 +20,11 @@ readdirSync(chainsRoot)
} }
}); });
apps.push({
name: 'hyperion-governor',
namespace: 'hyperion',
script: 'governor/server/index.js',
watch: false,
});
module.exports = {apps}; module.exports = {apps};
package-lock.json
Generated
-21
View File
@@ -19,7 +19,6 @@
"@fastify/redis": "^5.0.0", "@fastify/redis": "^5.0.0",
"@fastify/swagger": "6.1.0", "@fastify/swagger": "6.1.0",
"@pm2/io": "^5.0.0", "@pm2/io": "^5.0.0",
"amqp-connection-manager": "^4.1.14",
"amqplib": "^0.10.3", "amqplib": "^0.10.3",
"async": "^3.2.4", "async": "^3.2.4",
"base-x": "^4.0.0", "base-x": "^4.0.0",
@@ -444,21 +443,6 @@
"url": "https://github.com/sponsors/epoberezkin" "url": "https://github.com/sponsors/epoberezkin"
} }
}, },
"node_modules/amqp-connection-manager": {
"version": "4.1.14",
"resolved": "https://registry.npmjs.org/amqp-connection-manager/-/amqp-connection-manager-4.1.14.tgz",
"integrity": "sha512-1km47dIvEr0HhMUazqovSvNwIlSvDX2APdUpULaINtHpiki1O+cLRaTeXb/jav4OLtH+k6GBXx5gsKOT9kcGKQ==",
"dependencies": {
"promise-breaker": "^6.0.0"
},
"engines": {
"node": ">=10.0.0",
"npm": ">5.0.0"
},
"peerDependencies": {
"amqplib": "*"
}
},
"node_modules/amqplib": { "node_modules/amqplib": {
"version": "0.10.3", "version": "0.10.3",
"resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.3.tgz", "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.3.tgz",
@@ -2263,11 +2247,6 @@
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",
"integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==" "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q=="
}, },
"node_modules/promise-breaker": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/promise-breaker/-/promise-breaker-6.0.0.tgz",
"integrity": "sha512-BthzO9yTPswGf7etOBiHCVuugs2N01/Q/94dIPls48z2zCmrnDptUUZzfIb+41xq0MnYZ/BzmOd6ikDR4ibNZA=="
},
"node_modules/proxy-addr": { "node_modules/proxy-addr": {
"version": "2.0.7", "version": "2.0.7",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
package.json
-1
View File
@@ -35,7 +35,6 @@
"@fastify/redis": "^5.0.0", "@fastify/redis": "^5.0.0",
"@fastify/swagger": "6.1.0", "@fastify/swagger": "6.1.0",
"@pm2/io": "^5.0.0", "@pm2/io": "^5.0.0",
"amqp-connection-manager": "^4.1.14",
"amqplib": "^0.10.3", "amqplib": "^0.10.3",
"async": "^3.2.4", "async": "^3.2.4",
"base-x": "^4.0.0", "base-x": "^4.0.0",
tests/dsp-consumer.js
+6 -7
View File
@@ -16,21 +16,20 @@ class DspEventConsumer {
console.log('Starting DSP Consumer...'); console.log('Starting DSP Consumer...');
[this.ch] = await this.manager.createAMQPChannels((channels) => { [this.ch] = await this.manager.createAMQPChannels((channels) => {
[this.ch] = channels; [this.ch] = channels;
this.onConnect().catch(console.log); this.onConnect();
}, () => { }, () => {
this.ch_ready = false; this.ch_ready = false;
}); });
this.onConnect().catch(console.log); this.onConnect();
} }
async onConnect() { onConnect() {
if (this.conf.settings.dsp_parser) { if (this.conf.settings.dsp_parser) {
const q = `${this.manager.chain}:dsp`; const q = `${this.manager.chain}:dsp`;
console.log(q); console.log(q);
await this.ch.assertQueue(q, { durable: true }); this.ch.prefetch(100);
await this.ch.consume(q, (data) => { this.ch.assertQueue(q, { durable: true });
this.ch.consume(q, (data) => {
this.onMessage(data); this.onMessage(data);
}, {
prefetch: 100
}); });
} }
} }
tests/dsp-consumer.js.map
+1 -1
View File
@@ -1 +1 @@
{"version":3,"file":"dsp-consumer.js","sourceRoot":"","sources":["dsp-consumer.ts"],"names":[],"mappings":";;AAAA,8CAAsD;AACtD,gEAA+D;AAK/D,MAAM,gBAAgB;IAUlB;QALQ,aAAQ,GAAY,KAAK,CAAC;QAC1B,cAAS,GAAG,CAAC,CAAC;QACd,WAAM,GAAG,CAAC,CAAC;QACX,iBAAY,GAAuB,IAAI,GAAG,EAAE,CAAC;QAGjD,MAAM,EAAE,GAAG,IAAI,4BAAmB,EAAE,CAAC;QACrC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,IAAI,iCAAiB,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,GAAG;QACL,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;QACxC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,EAAE;YAC3D,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC;YACrB,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC,EAAE,GAAG,EAAE;YACJ,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAEO,KAAK,CAAC,SAAS;QACnB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE;YAC/B,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,MAAM,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,EAAC,OAAO,EAAE,IAAI,EAAC,CAAC,CAAC;YAC9C,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC9B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,EAAE;gBACC,QAAQ,EAAE,GAAG;aAChB,CAAC,CAAC;SACN;IACL,CAAC;IAED,aAAa,CAAC,SAAiB;QAC3B,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;gBAClC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;gBAC7G,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE;oBAChC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;iBAC/E;gBACD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;aACvC;QACL,CAAC,EAAE,GAAG,CAAC,CAAC;IACZ,CAAC;IAED,SAAS,CAAC,GAAY;QAClB,IAAI;YACA,MAAM,OAAO,GAAQ,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YACxD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACrB,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE;gBACpC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;gBACnC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;aACvD;iBAAM,IAAI,OAAO,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,EAAE;gBAC7C,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE;oBACvC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBACvD;aACJ;iBAAM;gBACH,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;aAC1D;YACD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SACpB;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACrB;IACL,CAAC;CAEJ;AAED,IAAI,gBAAgB,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC"} {"version":3,"file":"dsp-consumer.js","sourceRoot":"","sources":["dsp-consumer.ts"],"names":[],"mappings":";;AAAA,8CAAsD;AACtD,gEAA+D;AAI/D,MAAM,gBAAgB;IAUlB;QALQ,aAAQ,GAAY,KAAK,CAAC;QAC1B,cAAS,GAAG,CAAC,CAAC;QACd,WAAM,GAAG,CAAC,CAAC;QACX,iBAAY,GAAuB,IAAI,GAAG,EAAE,CAAC;QAGjD,MAAM,EAAE,GAAG,IAAI,4BAAmB,EAAE,CAAC;QACrC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,IAAI,iCAAiB,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,GAAG;QACL,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;QACxC,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE,EAAE;YAC3D,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC;YACrB,IAAI,CAAC,SAAS,EAAE,CAAC;QACrB,CAAC,EAAE,GAAG,EAAE;YACJ,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAC1B,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAEO,SAAS;QACb,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE;YAC/B,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,MAAM,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,EAAC,OAAO,EAAE,IAAI,EAAC,CAAC,CAAC;YACxC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;SACN;IACL,CAAC;IAED,aAAa,CAAC,SAAiB;QAC3B,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;gBAClC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;gBAC7G,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE;oBAChC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;iBAC/E;gBACD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;aACvC;QACL,CAAC,EAAE,GAAG,CAAC,CAAC;IACZ,CAAC;IAED,SAAS,CAAC,GAAY;QAClB,IAAI;YACA,MAAM,OAAO,GAAQ,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YACxD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACrB,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE;gBACpC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;gBACnC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;aACvD;iBAAM,IAAI,OAAO,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,EAAE;gBAC7C,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE;oBACvC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBACvD;aACJ;iBAAM;gBACH,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;aAC1D;YACD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SACpB;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACrB;IACL,CAAC;CAEJ;AAED,IAAI,gBAAgB,EAAE,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC"}
tests/dsp-consumer.ts
+8 -10
View File
@@ -1,14 +1,13 @@
import {ConfigurationModule} from "../modules/config"; import {ConfigurationModule} from "../modules/config";
import {ConnectionManager} from "../connections/manager.class"; import {ConnectionManager} from "../connections/manager.class";
import {HyperionConfig} from "../interfaces/hyperionConfig"; import {HyperionConfig} from "../interfaces/hyperionConfig";
import {Message} from "amqplib/callback_api"; import {Channel, Message} from "amqplib/callback_api";
import {ChannelWrapper} from "amqp-connection-manager";
class DspEventConsumer { class DspEventConsumer {
private conf: HyperionConfig; private conf: HyperionConfig;
private manager: ConnectionManager; private manager: ConnectionManager;
private ch: ChannelWrapper; private ch: Channel;
private ch_ready: boolean = false; private ch_ready: boolean = false;
private lastBlock = 0; private lastBlock = 0;
private lastGS = 0; private lastGS = 0;
@@ -24,22 +23,21 @@ class DspEventConsumer {
console.log('Starting DSP Consumer...'); console.log('Starting DSP Consumer...');
[this.ch] = await this.manager.createAMQPChannels((channels) => { [this.ch] = await this.manager.createAMQPChannels((channels) => {
[this.ch] = channels; [this.ch] = channels;
this.onConnect().catch(console.log); this.onConnect();
}, () => { }, () => {
this.ch_ready = false; this.ch_ready = false;
}); });
this.onConnect().catch(console.log); this.onConnect();
} }
private async onConnect(): Promise<void> { private onConnect() {
if (this.conf.settings.dsp_parser) { if (this.conf.settings.dsp_parser) {
const q = `${this.manager.chain}:dsp`; const q = `${this.manager.chain}:dsp`;
console.log(q); console.log(q);
await this.ch.assertQueue(q, {durable: true}); this.ch.prefetch(100);
await this.ch.consume(q, (data) => { this.ch.assertQueue(q, {durable: true});
this.ch.consume(q, (data) => {
this.onMessage(data); this.onMessage(data);
}, {
prefetch: 100
}); });
} }
} }
workers/delta-updater.ts
+3 -5
View File
@@ -29,11 +29,9 @@ export default class MainDSWorker extends HyperionWorker {
if (this.ch) { if (this.ch) {
this.queueName = this.chain + ":delta_rm"; this.queueName = this.chain + ":delta_rm";
hLog(`Launched delta updater, consuming from ${this.queueName}`); hLog(`Launched delta updater, consuming from ${this.queueName}`);
this.ch.assertQueue(this.queueName, {durable: true}).then(() => { this.ch.assertQueue(this.queueName, {durable: true});
this.ch.consume(this.queueName, this.onConsume.bind(this),{ this.ch.prefetch(1);
prefetch: 1 this.ch.consume(this.queueName, this.onConsume.bind(this));
}).catch(console.log);
});
} }
} }
workers/deserializer.ts
+9 -5
View File
@@ -286,11 +286,10 @@ export default class MainDSWorker extends HyperionWorker {
private initConsumer() { private initConsumer() {
if (this.ch_ready) { if (this.ch_ready) {
this.ch.prefetch(this.conf.prefetch.block);
this.ch.consume(process.env['worker_queue'], (data) => { this.ch.consume(process.env['worker_queue'], (data) => {
this.consumerQueue.push(data).catch(console.log); this.consumerQueue.push(data).catch(console.log);
}, { });
prefetch: this.conf.prefetch.block
}).catch(console.log);
} }
} }
@@ -624,7 +623,12 @@ export default class MainDSWorker extends HyperionWorker {
const pool_queue = `${this.chain}:ds_pool:${selected_q}`; const pool_queue = `${this.chain}:ds_pool:${selected_q}`;
if (this.ch_ready) { if (this.ch_ready) {
this.ch.sendToQueue(pool_queue, bufferFromJson(trace, true), {headers}).catch(console.log); const enqueueResult = this.ch.sendToQueue(pool_queue, bufferFromJson(trace, true), {headers});
if (!enqueueResult) {
hLog("Backpressure");
console.log("Header size: " + JSON.stringify(headers).length);
console.log(headers);
}
return true; return true;
} else { } else {
return false; return false;
@@ -1160,7 +1164,7 @@ export default class MainDSWorker extends HyperionWorker {
let jsonRow = await this.processContractRowNative(payload, block_num); let jsonRow = await this.processContractRowNative(payload, block_num);
if (jsonRow?.value && !jsonRow['_blacklisted']) { if (jsonRow?.value && !jsonRow['_blacklisted']) {
debugLog(jsonRow); console.log(jsonRow);
debugLog('Delta DS failed ->>', jsonRow); debugLog('Delta DS failed ->>', jsonRow);
jsonRow = await this.processContractRowNative(payload, block_num - 1); jsonRow = await this.processContractRowNative(payload, block_num - 1);
debugLog('Retry with previous ABI ->>', jsonRow); debugLog('Retry with previous ABI ->>', jsonRow);
workers/ds-pool.ts
+7 -7
View File
@@ -625,11 +625,12 @@ export default class DSPoolWorker extends HyperionWorker {
initConsumer() { initConsumer() {
if (this.ch_ready) { if (this.ch_ready) {
this.ch.prefetch(this.conf.prefetch.block);
this.ch.consume(this.local_queue, (data) => { this.ch.consume(this.local_queue, (data) => {
this.consumerQueue.push(data); this.consumerQueue.push(data);
}, { }, {}, (err, ok) => {
prefetch: this.conf.prefetch.block hLog(err, ok);
}).catch(console.log); });
debugLog(`started consuming from ${this.local_queue}`); debugLog(`started consuming from ${this.local_queue}`);
} }
} }
@@ -693,14 +694,13 @@ export default class DSPoolWorker extends HyperionWorker {
this.ch_ready = true; this.ch_ready = true;
this.ch.assertQueue(this.local_queue, { this.ch.assertQueue(this.local_queue, {
durable: true durable: true
}).then(() => { });
this.initConsumer(); this.initConsumer();
}).catch(console.log);
} }
if (this.conf.settings.dsp_parser) { if (this.conf.settings.dsp_parser) {
this.ch.assertQueue(`${queue_prefix}:dsp`, { this.ch.assertQueue(`${queue_prefix}:dsp`, {
durable: true durable: true
}).catch(console.log); });
} }
} }
workers/hyperionWorker.ts
+226 -229
View File
@@ -11,280 +11,277 @@ import {HeapInfo} from "v8";
import {debugLog, hLog} from "../helpers/common_functions"; import {debugLog, hLog} from "../helpers/common_functions";
import {StateHistorySocket} from "../connections/state-history"; import {StateHistorySocket} from "../connections/state-history";
import * as AbiEOS from "@eosrio/node-abieos"; import * as AbiEOS from "@eosrio/node-abieos";
import {ChannelWrapper} from "amqp-connection-manager";
export abstract class HyperionWorker { export abstract class HyperionWorker {
conf: HyperionConfig; conf: HyperionConfig;
manager: ConnectionManager; manager: ConnectionManager;
mLoader: HyperionModuleLoader; mLoader: HyperionModuleLoader;
chain: string; chain: string;
chainId: string; chainId: string;
// AMQP Channels // AMQP Channels
ch: ChannelWrapper; ch: Channel;
cch: ChannelWrapper; cch: ConfirmChannel;
rpc: JsonRpc; rpc: JsonRpc;
client: Client; client: Client;
ship: StateHistorySocket; ship: StateHistorySocket;
txEnc = new TextEncoder(); txEnc = new TextEncoder();
txDec = new TextDecoder(); txDec = new TextDecoder();
cch_ready = false; cch_ready = false;
ch_ready = false; ch_ready = false;
events: EventEmitter; events: EventEmitter;
filters: Filters; filters: Filters;
failedAbiMap: Map<string, Set<number>> = new Map(); failedAbiMap: Map<string, Set<number>> = new Map();
protected constructor() { protected constructor() {
this.checkDebugger(); this.checkDebugger();
const cm = new ConfigurationModule(); const cm = new ConfigurationModule();
this.conf = cm.config; this.conf = cm.config;
this.filters = cm.filters; this.filters = cm.filters;
this.manager = new ConnectionManager(cm); this.manager = new ConnectionManager(cm);
this.mLoader = new HyperionModuleLoader(cm); this.mLoader = new HyperionModuleLoader(cm);
this.chain = this.conf.settings.chain; this.chain = this.conf.settings.chain;
this.chainId = this.manager.conn.chains[this.chain].chain_id; this.chainId = this.manager.conn.chains[this.chain].chain_id;
this.rpc = this.manager.nodeosJsonRPC; this.rpc = this.manager.nodeosJsonRPC;
this.client = this.manager.elasticsearchClient; this.client = this.manager.elasticsearchClient;
this.ship = this.manager.shipClient; this.ship = this.manager.shipClient;
this.events = new EventEmitter(); this.events = new EventEmitter();
this.mLoader.init().then(() => { this.mLoader.init().then(() => {
// Connect to RabbitMQ (amqplib) // Connect to RabbitMQ (amqplib)
this.events.emit('loader_ready'); this.events.emit('loader_ready');
this.connectAMQP().then(() => { this.connectAMQP().then(() => {
this.onConnect(); this.onConnect();
}).catch(console.log); }).catch(console.log);
}); });
const bytesToString = (bytes: number) => { const bytesToString = (bytes: number) => {
const e = Math.log(bytes) / Math.log(1024) | 0; const e = Math.log(bytes) / Math.log(1024) | 0;
const n = (bytes / Math.pow(1024, e)).toFixed(2); const n = (bytes / Math.pow(1024, e)).toFixed(2);
return n + ' ' + (e == 0 ? 'bytes' : (['KB', 'MB', 'GB', 'TB'])[e - 1]); return n + ' ' + (e == 0 ? 'bytes' : (['KB', 'MB', 'GB', 'TB'])[e - 1]);
}; };
// handle ipc messages // handle ipc messages
process.on('message', (msg: any) => { process.on('message', (msg: any) => {
switch (msg.event) { switch (msg.event) {
case 'request_v8_heap_stats': { case 'request_v8_heap_stats': {
const report: HeapInfo = v8.getHeapStatistics(); const report: HeapInfo = v8.getHeapStatistics();
const used_pct = report.used_heap_size / report.heap_size_limit; const used_pct = report.used_heap_size / report.heap_size_limit;
process.send({ process.send({
event: 'v8_heap_report', event: 'v8_heap_report',
id: process.env.worker_role + ':' + process.env.worker_id, id: process.env.worker_role + ':' + process.env.worker_id,
data: { data: {
heap_usage: (used_pct * 100).toFixed(2) + "%", heap_usage: (used_pct * 100).toFixed(2) + "%",
...report ...report
} }
}); });
break; break;
} }
case 'request_memory_usage': { case 'request_memory_usage': {
const report = process.memoryUsage(); const report = process.memoryUsage();
process.send({ process.send({
event: 'memory_report', event: 'memory_report',
id: process.env.worker_role + ':' + process.env.worker_id, id: process.env.worker_role + ':' + process.env.worker_id,
data: { data: {
resident: bytesToString(report.rss), resident: bytesToString(report.rss),
} }
}); });
break; break;
} }
default: { default: {
this.onIpcMessage(msg); this.onIpcMessage(msg);
} }
} }
}); });
} }
async connectAMQP() { async connectAMQP() {
[this.ch, this.cch] = await this.manager.createAMQPChannels((channels: ChannelWrapper[]) => { [this.ch, this.cch] = await this.manager.createAMQPChannels((channels) => {
if (channels) { [this.ch, this.cch] = channels;
[this.ch, this.cch] = channels; hLog('AMQP Reconnecting...');
hLog('AMQP Reconnecting...'); this.onConnect();
this.onConnect(); }, () => {
} this.ch_ready = false;
}, () => { this.cch_ready = false;
this.ch_ready = false; });
this.cch_ready = false; }
});
}
onConnect() { onConnect() {
this.ch_ready = true; this.ch_ready = true;
this.cch_ready = true; this.cch_ready = true;
this.assertQueues(); this.assertQueues();
this.ch.on('close', () => { this.ch.on('close', () => {
this.ch_ready = false; this.ch_ready = false;
}); });
this.cch.on('close', () => { this.cch.on('close', () => {
this.cch_ready = false; this.cch_ready = false;
}); });
this.events.emit('ready'); this.events.emit('ready');
} }
checkDebugger() { checkDebugger() {
if (/--inspect/.test(process.execArgv.join(' '))) { if (/--inspect/.test(process.execArgv.join(' '))) {
const inspector = require('inspector'); const inspector = require('inspector');
hLog('DEBUGGER ATTACHED', inspector.url()); hLog('DEBUGGER ATTACHED', inspector.url());
} }
} }
private anyFromCode(act: any) { private anyFromCode(act: any) {
return this.chain + '::' + act.account + '::*' return this.chain + '::' + act.account + '::*'
} }
private anyFromName(act: any) { private anyFromName(act: any) {
return this.chain + '::*::' + act.name; return this.chain + '::*::' + act.name;
} }
private codeActionPair(act: any) { private codeActionPair(act: any) {
return this.chain + '::' + act.account + '::' + act.name; return this.chain + '::' + act.account + '::' + act.name;
} }
private anyFromDeltaCode(delta: any) { private anyFromDeltaCode(delta: any) {
return this.chain + '::' + delta.code + '::*' return this.chain + '::' + delta.code + '::*'
} }
private anyFromDeltaTable(delta: any) { private anyFromDeltaTable(delta: any) {
return this.chain + '::*::' + delta.table; return this.chain + '::*::' + delta.table;
} }
private codeDeltaPair(delta: any) { private codeDeltaPair(delta: any) {
return this.chain + '::' + delta.code + '::' + delta.table; return this.chain + '::' + delta.code + '::' + delta.table;
} }
protected checkBlacklist(act) { protected checkBlacklist(act) {
// test for chain::code::* // test for chain::code::*
if (this.filters.action_blacklist.has(this.anyFromCode(act))) { if (this.filters.action_blacklist.has(this.anyFromCode(act))) {
return true; return true;
} }
// test for chain::*::name // test for chain::*::name
if (this.filters.action_blacklist.has(this.anyFromName(act))) { if (this.filters.action_blacklist.has(this.anyFromName(act))) {
return true; return true;
} }
// test for chain::code::name // test for chain::code::name
return this.filters.action_blacklist.has(this.codeActionPair(act)); return this.filters.action_blacklist.has(this.codeActionPair(act));
} }
protected checkWhitelist(act) { protected checkWhitelist(act) {
// test for chain::code::* // test for chain::code::*
if (this.filters.action_whitelist.has(this.anyFromCode(act))) { if (this.filters.action_whitelist.has(this.anyFromCode(act))) {
return true; return true;
} }
// test for chain::*::name // test for chain::*::name
if (this.filters.action_whitelist.has(this.anyFromName(act))) { if (this.filters.action_whitelist.has(this.anyFromName(act))) {
return true; return true;
} }
// test for chain::code::name // test for chain::code::name
return this.filters.action_whitelist.has(this.codeActionPair(act)); return this.filters.action_whitelist.has(this.codeActionPair(act));
} }
protected checkDeltaBlacklist(delta) { protected checkDeltaBlacklist(delta) {
// test blacklist for chain::code::* // test blacklist for chain::code::*
if (this.filters.delta_blacklist.has(this.anyFromDeltaCode(delta))) { if (this.filters.delta_blacklist.has(this.anyFromDeltaCode(delta))) {
return true; return true;
} }
// test blacklist for chain::*::table // test blacklist for chain::*::table
if (this.filters.delta_blacklist.has(this.anyFromDeltaTable(delta))) { if (this.filters.delta_blacklist.has(this.anyFromDeltaTable(delta))) {
return true; return true;
} }
// test blacklist for chain::code::table // test blacklist for chain::code::table
return this.filters.delta_blacklist.has(this.codeDeltaPair(delta)); return this.filters.delta_blacklist.has(this.codeDeltaPair(delta));
} }
protected checkDeltaWhitelist(delta) { protected checkDeltaWhitelist(delta) {
// test whitelist for chain::code::* // test whitelist for chain::code::*
if (this.filters.delta_whitelist.has(this.anyFromDeltaCode(delta))) { if (this.filters.delta_whitelist.has(this.anyFromDeltaCode(delta))) {
return true; return true;
} }
// test whitelist for chain::*::table // test whitelist for chain::*::table
if (this.filters.delta_whitelist.has(this.anyFromDeltaTable(delta))) { if (this.filters.delta_whitelist.has(this.anyFromDeltaTable(delta))) {
return true; return true;
} }
// test whitelist for chain::code::table // test whitelist for chain::code::table
return this.filters.delta_whitelist.has(this.codeDeltaPair(delta)); return this.filters.delta_whitelist.has(this.codeDeltaPair(delta));
} }
loadAbiHex(contract, block_num, abi_hex) { loadAbiHex(contract, block_num, abi_hex) {
// check local blacklist for corrupted abis that failed to load before // check local blacklist for corrupted abis that failed to load before
let _status; let _status;
if (this.failedAbiMap.has(contract) && this.failedAbiMap.get(contract).has(block_num)) { if (this.failedAbiMap.has(contract) && this.failedAbiMap.get(contract).has(block_num)) {
_status = false; _status = false;
debugLog('ignore saved abi for', contract, block_num); debugLog('ignore saved abi for', contract, block_num);
} else { } else {
_status = AbiEOS.load_abi_hex(contract, abi_hex); _status = AbiEOS.load_abi_hex(contract, abi_hex);
if (!_status) { if (!_status) {
hLog(`AbiEOS.load_abi_hex error for ${contract} at ${block_num}`); hLog(`AbiEOS.load_abi_hex error for ${contract} at ${block_num}`);
if (this.failedAbiMap.has(contract)) { if (this.failedAbiMap.has(contract)) {
this.failedAbiMap.get(contract).add(block_num); this.failedAbiMap.get(contract).add(block_num);
} else { } else {
this.failedAbiMap.set(contract, new Set([block_num])); this.failedAbiMap.set(contract, new Set([block_num]));
} }
} else { } else {
this.removeFromFailed(contract); this.removeFromFailed(contract);
} }
} }
return _status; return _status;
} }
removeFromFailed(contract) { removeFromFailed(contract) {
if (this.failedAbiMap.has(contract)) { if (this.failedAbiMap.has(contract)) {
this.failedAbiMap.delete(contract); this.failedAbiMap.delete(contract);
hLog(`${contract} was removed from the failed map!`); hLog(`${contract} was removed from the failed map!`);
} }
} }
async loadCurrentAbiHex(contract) { async loadCurrentAbiHex(contract) {
let _status; let _status;
if (this.failedAbiMap.has(contract) && this.failedAbiMap.get(contract).has(-1)) { if (this.failedAbiMap.has(contract) && this.failedAbiMap.get(contract).has(-1)) {
_status = false; _status = false;
debugLog('ignore current abi for', contract); debugLog('ignore current abi for', contract);
} else { } else {
const currentAbi = await this.rpc.getRawAbi(contract); const currentAbi = await this.rpc.getRawAbi(contract);
if (currentAbi.abi.byteLength > 0) { if (currentAbi.abi.byteLength > 0) {
const abi_hex = Buffer.from(currentAbi.abi).toString('hex'); const abi_hex = Buffer.from(currentAbi.abi).toString('hex');
_status = AbiEOS.load_abi_hex(contract, abi_hex); _status = AbiEOS.load_abi_hex(contract, abi_hex);
if (!_status) { if (!_status) {
hLog(`AbiEOS.load_abi_hex error for ${contract} at head`); hLog(`AbiEOS.load_abi_hex error for ${contract} at head`);
if (this.failedAbiMap.has(contract)) { if (this.failedAbiMap.has(contract)) {
this.failedAbiMap.get(contract).add(-1); this.failedAbiMap.get(contract).add(-1);
} else { } else {
this.failedAbiMap.set(contract, new Set([-1])); this.failedAbiMap.set(contract, new Set([-1]));
} }
} else { } else {
this.removeFromFailed(contract); this.removeFromFailed(contract);
} }
} else { } else {
_status = false; _status = false;
} }
} }
return _status; return _status;
} }
abstract run(): Promise<void> abstract run(): Promise<void>
abstract assertQueues(): void abstract assertQueues(): void
abstract onIpcMessage(msg: any): void abstract onIpcMessage(msg: any): void
} }
workers/indexer.ts
+3 -5
View File
@@ -48,11 +48,9 @@ export default class IndexerWorker extends HyperionWorker {
this.indexQueue.pause(); this.indexQueue.pause();
this.ch_ready = false; this.ch_ready = false;
}); });
this.ch.assertQueue(process.env.queue, {durable: true}).then(r => { this.ch.assertQueue(process.env.queue, {durable: true});
this.ch.consume(process.env.queue, this.indexQueue.push, { this.ch.prefetch(this.conf.prefetch.index);
prefetch: this.conf.prefetch.index this.ch.consume(process.env.queue, this.indexQueue.push);
}).catch(console.log);
});
} }
} catch (e) { } catch (e) {
console.error('rabbitmq error!'); console.error('rabbitmq error!');