C9S/explorer-api
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v3.3.8 stable

Browse Source 
- fix readers stopping before having finished
This commit is contained in:
Igor Lins e Silva
2023-01-18 14:20:56 -03:00
parent 2ee60bc276
commit 2827f85f3e
5 changed files with 166 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
connections/state-history.ts
+11 -3
View File
@@ -6,6 +6,8 @@ export class StateHistorySocket {
private ws;
private readonly shipUrl;
private readonly max_payload_mb;
retryOnDisconnect = true;
connected = false;
constructor(ship_url, max_payload_mb) {
this.shipUrl = ship_url;
@@ -17,13 +19,13 @@ export class StateHistorySocket {
}
connect(onMessage, onDisconnect, onError, onConnected) {
debugLog(`Connecting to ${this.shipUrl}...`);
this.ws = new WebSocket(this.shipUrl, {
perMessageDeflate: false,
maxPayload: this.max_payload_mb * 1024 * 1024,
});
this.ws.on('open', () => {
this.connected = true;
hLog('Websocket connected!');
if (onConnected) {
onConnected();
@@ -31,15 +33,21 @@ export class StateHistorySocket {
});
this.ws.on('message', (data) => onMessage(data));
this.ws.on('close', () => {
this.connected = false;
hLog('Websocket disconnected!');
onDisconnect();
if(this.retryOnDisconnect) {
onDisconnect();
}
});
this.ws.on('error', (err) => {
hLog(`${this.shipUrl} :: ${err.message}`);
});
}
close() {
close(graceful: boolean) {
if(graceful) {
this.retryOnDisconnect = false;
}
this.ws.close();
}
modules/master.ts
+37 -5
View File
@@ -234,6 +234,8 @@ export class HyperionMaster {
tx: msg.trx_ids
});
this.lastProducedBlockNum = msg.block_num;
if (this.conf.settings.bp_monitoring && !this.conf.indexer.abi_scan_mode) {
this.liveBlockQueue.push(msg).catch(reason => {
hLog('Error handling consumed_block:', reason);
@@ -326,17 +328,38 @@ export class HyperionMaster {
if (end > this.head) {
end = this.head;
}
this.lastAssignedBlock += this.maxBatchSize;
const def = {
first_block: start,
last_block: end
};
this.lastAssignedBlock = def.last_block;
this.activeReadersCount++;
messageAllWorkers(cluster, {
event: 'new_range',
target: msg.id,
data: def
});
} else {
if (this.lastAssignedBlock >= this.head) {
hLog(`Parallel readers finished the requested range`);
const readers = this.workerMap.filter(value => value.worker_role === 'reader');
this.workerMap = this.workerMap.filter(value => value.worker_role !== 'reader');
readers.forEach(value => value.wref.kill());
// for (let hyperionWorkerDef of this.workerMap) {
// if (hyperionWorkerDef.worker_role === 'reader') {
// hyperionWorkerDef.wref.kill();
// }
// }
// for (let workersKey in cluster.workers) {
// const w = cluster.workers[workersKey];
// console.log(w);
// if (w.id === parseInt(msg.id)) {
// const idx = this.workerMap.findIndex(value => value.worker_id === w.id);
// this.workerMap.splice(idx, 1);
// w.kill();
// }
// }
}
}
}
},
@@ -1461,6 +1484,12 @@ export class HyperionMaster {
avg_consume_rate = consume_rate;
}
const log_msg = [];
// print current head for live reading
if (this.lastProducedBlockNum > 0) {
log_msg.push(`#${this.lastProducedBlockNum}`);
}
log_msg.push(`W:${_workers}`);
const _r = (this.pushedBlocks + this.livePushedBlocks) / tScale;
@@ -1484,9 +1513,12 @@ export class HyperionMaster {
this.metrics.indexingRate?.set(_ir);
if (this.total_blocks < this.total_range && !this.conf.indexer.live_only_mode) {
const remaining = this.total_range - this.total_blocks;
const estimated_time = Math.round(remaining / avg_consume_rate);
const time_string = moment().add(estimated_time, 'seconds').fromNow(false);
let time_string = 'waiting for indexer';
if (avg_consume_rate > 0) {
const remaining = this.total_range - this.total_blocks;
const estimated_time = Math.round(remaining / avg_consume_rate);
time_string = moment().add(estimated_time, 'seconds').fromNow(false);
}
const pct_parsed = ((this.total_blocks / this.total_range) * 100).toFixed(1);
const pct_read = ((this.total_read / this.total_range) * 100).toFixed(1);
log_msg.push(`${this.total_blocks}/${this.total_read}/${this.total_range}`);
@@ -1510,7 +1542,7 @@ export class HyperionMaster {
hLog(log_msg.join(' | '));
}
if(this.liveConsumedBlocks > 0 && this.consumedBlocks === 0 && this.conf.indexer.abi_scan_mode) {
if (this.liveConsumedBlocks > 0 && this.consumedBlocks === 0 && this.conf.indexer.abi_scan_mode) {
hLog('Warning: Live reading on ABI SCAN mode')
}
package-lock.json
Generated
+78 -78
View File
@@ -1,12 +1,12 @@
{
"name": "hyperion-history",
"version": "3.3.7",
"version": "3.3.8",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "hyperion-history",
"version": "3.3.7",
"version": "3.3.8",
"hasInstallScript": true,
"license": "MIT",
"dependencies": {
@@ -14,8 +14,8 @@
"@eosrio/node-abieos": "^2.1.1",
"@fastify/autoload": "4.0.1",
"@fastify/cors": "7.0.0",
"@fastify/formbody": "^6.0.0",
"@fastify/rate-limit": "^6.0.0",
"@fastify/formbody": "^6.0.1",
"@fastify/rate-limit": "^6.0.1",
"@fastify/redis": "^5.0.0",
"@fastify/swagger": "6.1.0",
"@pm2/io": "^5.0.0",
@@ -35,26 +35,26 @@
"ioredis": "^4.28.5",
"lodash": "^4.17.21",
"moment": "^2.29.4",
"nodemailer": "^6.7.8",
"nodemailer": "^6.9.0",
"pino-pretty": "^9.1.1",
"portfinder": "^1.0.32",
"socket.io": "4.5.4",
"socket.io-client": "4.5.4",
"socket.io-redis": "^6.1.1",
"telegraf": "^4.9.1",
"telegraf": "^4.11.2",
"typescript": "^4.5.2",
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.15.0",
"ws": "^8.11.0"
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.19.0",
"ws": "^8.12.0"
},
"devDependencies": {
"@types/amqplib": "^0.8.2",
"@types/async": "^3.2.16",
"@types/global-agent": "^2.1.1",
"@types/ioredis": "^4.28.10",
"@types/lodash": "^4.14.177",
"@types/node": "^18.11.9",
"@types/nodemailer": "^6.4.4",
"@types/ws": "^8.5.3"
"@types/lodash": "^4.14.191",
"@types/node": "^18.11.18",
"@types/nodemailer": "^6.4.7",
"@types/ws": "^8.5.4"
},
"engines": {
"node": "^18"
@@ -148,17 +148,17 @@
"integrity": "sha512-wI3fpfDT0t7p8E6dA2eTECzzOd+bZsZCJ2Hcv+Onn2b7ZwK3RwD27uW2QDaMtQhAfWQQP+WNK7nKf0twLsBf9w=="
},
"node_modules/@fastify/formbody": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/@fastify/formbody/-/formbody-6.0.0.tgz",
"integrity": "sha512-YzPTXJbB3CzDMqU5K9YGSBt/nc/RDFZSxSWZ4SoqA3T2VRJzCPd7sZFpggdmlBRWhEBlvl0EWW7EX33kfbbFlg==",
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@fastify/formbody/-/formbody-6.0.1.tgz",
"integrity": "sha512-yIwCitoES4Sh0tPc6v+uHBqZEKw3CooSZ4kUvO9NC8Y7oRBCW+aC+pTgUZ8M3r2DOzRkNO+Pq0jMQkyL2k8jZQ==",
"dependencies": {
"fastify-plugin": "^3.0.0"
}
},
"node_modules/@fastify/rate-limit": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-6.0.0.tgz",
"integrity": "sha512-EZ0TI7Mpo5aK/QaooSt3U0cAGSFjrN51vwj6osgBF1KrnLVSmRA7KPYhPimMoL9YFu9GH53aiNU0H1qTLh7s1Q==",
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-6.0.1.tgz",
"integrity": "sha512-LSd6gIhvTMbt+xRG4vhfFKgbxy8x/namtIo9EcdUneE7LnsS6GSQ1NL2eo0Bg8Q61mNk812guS1xCLvO5kSTPQ==",
"dependencies": {
"fastify-plugin": "^3.0.1",
"ms": "^2.1.3",
@@ -424,20 +424,20 @@
}
},
"node_modules/@types/lodash": {
"version": "4.14.177",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.177.tgz",
"integrity": "sha512-0fDwydE2clKe9MNfvXHBHF9WEahRuj+msTuQqOmAApNORFvhMYZKNGGJdCzuhheVjMps/ti0Ak/iJPACMaevvw==",
"version": "4.14.191",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz",
"integrity": "sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==",
"dev": true
},
"node_modules/@types/node": {
"version": "18.11.9",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz",
"integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg=="
"version": "18.11.18",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz",
"integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA=="
},
"node_modules/@types/nodemailer": {
"version": "6.4.4",
"resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.4.tgz",
"integrity": "sha512-Ksw4t7iliXeYGvIQcSIgWQ5BLuC/mljIEbjf615svhZL10PE9t+ei8O9gDaD3FPCasUJn9KTLwz2JFJyiiyuqw==",
"version": "6.4.7",
"resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.7.tgz",
"integrity": "sha512-f5qCBGAn/f0qtRcd4SEn88c8Fp3Swct1731X4ryPKqS61/A3LmmzN8zaEz7hneJvpjFbUUgY7lru/B/7ODTazg==",
"dev": true,
"dependencies": {
"@types/node": "*"
@@ -452,9 +452,9 @@
}
},
"node_modules/@types/ws": {
"version": "8.5.3",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz",
"integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==",
"version": "8.5.4",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz",
"integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==",
"dev": true,
"dependencies": {
"@types/node": "*"
@@ -1940,9 +1940,9 @@
}
},
"node_modules/nodemailer": {
"version": "6.7.8",
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.7.8.tgz",
"integrity": "sha512-2zaTFGqZixVmTxpJRCFC+Vk5eGRd/fYtvIR+dl5u9QXLTQWGIf48x/JXvo58g9sa0bU6To04XUv554Paykum3g==",
"version": "6.9.0",
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.9.0.tgz",
"integrity": "sha512-jFaCEGTeT3E/m/5R2MHWiyQH3pSARECRUDM+1hokOYc3lQAAG7ASuy+2jIsYVf+RVa9zePopSQwKNVFH8DKUpA==",
"engines": {
"node": ">=6.0.0"
}
@@ -2711,18 +2711,18 @@
}
},
"node_modules/telegraf": {
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/telegraf/-/telegraf-4.9.1.tgz",
"integrity": "sha512-MukWpKvAZ6/HpT3yHXz+jwUf2HsPa9TcsqPLQjJ+kHNGUS2PLgaNX690ExdWmWPuxjVjC4wNHmZ9JetO3C/tVA==",
"version": "4.11.2",
"resolved": "https://registry.npmjs.org/telegraf/-/telegraf-4.11.2.tgz",
"integrity": "sha512-RGEh+NXkHbq1KcSSbJeVYhHMrEN4rymd9DSe3SoIV0886bJPBHLzYCNrOqnk9aeZE2Idwh5uK0X/xbR6jScQKQ==",
"dependencies": {
"abort-controller": "^3.0.0",
"debug": "^4.3.3",
"debug": "^4.3.4",
"mri": "^1.2.0",
"node-fetch": "^2.6.7",
"p-timeout": "^4.1.0",
"safe-compare": "^1.1.4",
"sandwich-stream": "^2.0.2",
"typegram": "^3.11.0"
"typegram": "^4.1.0"
},
"bin": {
"telegraf": "lib/cli.mjs"
@@ -2769,9 +2769,9 @@
}
},
"node_modules/typegram": {
"version": "3.11.0",
"resolved": "https://registry.npmjs.org/typegram/-/typegram-3.11.0.tgz",
"integrity": "sha512-4p6u+AFognlsDgBue8Hla2jO7Ax+UQXcLa27LC7xDdAeR9LTe+Cr4vJrYpoO1wgj/BFWgXTeboaH/+1YgWyfpA=="
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/typegram/-/typegram-4.2.0.tgz",
"integrity": "sha512-rObfoGpDlmWUhmggpWd2I/4xLsdGPDvfvrLpLxV4pBTBL2BBjm+7x7IOmTwJRV6Qe4UAkWdHq0ZYoTYAAPE5YA=="
},
"node_modules/typescript": {
"version": "4.5.2",
@@ -2866,15 +2866,15 @@
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"node_modules/ws": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
"integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
"version": "8.12.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz",
"integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==",
"engines": {
"node": ">=10.0.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": "^5.0.2"
"utf-8-validate": ">=5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
@@ -2986,17 +2986,17 @@
"integrity": "sha512-wI3fpfDT0t7p8E6dA2eTECzzOd+bZsZCJ2Hcv+Onn2b7ZwK3RwD27uW2QDaMtQhAfWQQP+WNK7nKf0twLsBf9w=="
},
"@fastify/formbody": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/@fastify/formbody/-/formbody-6.0.0.tgz",
"integrity": "sha512-YzPTXJbB3CzDMqU5K9YGSBt/nc/RDFZSxSWZ4SoqA3T2VRJzCPd7sZFpggdmlBRWhEBlvl0EWW7EX33kfbbFlg==",
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@fastify/formbody/-/formbody-6.0.1.tgz",
"integrity": "sha512-yIwCitoES4Sh0tPc6v+uHBqZEKw3CooSZ4kUvO9NC8Y7oRBCW+aC+pTgUZ8M3r2DOzRkNO+Pq0jMQkyL2k8jZQ==",
"requires": {
"fastify-plugin": "^3.0.0"
}
},
"@fastify/rate-limit": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-6.0.0.tgz",
"integrity": "sha512-EZ0TI7Mpo5aK/QaooSt3U0cAGSFjrN51vwj6osgBF1KrnLVSmRA7KPYhPimMoL9YFu9GH53aiNU0H1qTLh7s1Q==",
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-6.0.1.tgz",
"integrity": "sha512-LSd6gIhvTMbt+xRG4vhfFKgbxy8x/namtIo9EcdUneE7LnsS6GSQ1NL2eo0Bg8Q61mNk812guS1xCLvO5kSTPQ==",
"requires": {
"fastify-plugin": "^3.0.1",
"ms": "^2.1.3",
@@ -3234,20 +3234,20 @@
}
},
"@types/lodash": {
"version": "4.14.177",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.177.tgz",
"integrity": "sha512-0fDwydE2clKe9MNfvXHBHF9WEahRuj+msTuQqOmAApNORFvhMYZKNGGJdCzuhheVjMps/ti0Ak/iJPACMaevvw==",
"version": "4.14.191",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz",
"integrity": "sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==",
"dev": true
},
"@types/node": {
"version": "18.11.9",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz",
"integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg=="
"version": "18.11.18",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz",
"integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA=="
},
"@types/nodemailer": {
"version": "6.4.4",
"resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.4.tgz",
"integrity": "sha512-Ksw4t7iliXeYGvIQcSIgWQ5BLuC/mljIEbjf615svhZL10PE9t+ei8O9gDaD3FPCasUJn9KTLwz2JFJyiiyuqw==",
"version": "6.4.7",
"resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.7.tgz",
"integrity": "sha512-f5qCBGAn/f0qtRcd4SEn88c8Fp3Swct1731X4ryPKqS61/A3LmmzN8zaEz7hneJvpjFbUUgY7lru/B/7ODTazg==",
"dev": true,
"requires": {
"@types/node": "*"
@@ -3262,9 +3262,9 @@
}
},
"@types/ws": {
"version": "8.5.3",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.3.tgz",
"integrity": "sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==",
"version": "8.5.4",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz",
"integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==",
"dev": true,
"requires": {
"@types/node": "*"
@@ -4410,9 +4410,9 @@
"optional": true
},
"nodemailer": {
"version": "6.7.8",
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.7.8.tgz",
"integrity": "sha512-2zaTFGqZixVmTxpJRCFC+Vk5eGRd/fYtvIR+dl5u9QXLTQWGIf48x/JXvo58g9sa0bU6To04XUv554Paykum3g=="
"version": "6.9.0",
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.9.0.tgz",
"integrity": "sha512-jFaCEGTeT3E/m/5R2MHWiyQH3pSARECRUDM+1hokOYc3lQAAG7ASuy+2jIsYVf+RVa9zePopSQwKNVFH8DKUpA=="
},
"normalize-url": {
"version": "6.1.0",
@@ -5016,18 +5016,18 @@
"integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="
},
"telegraf": {
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/telegraf/-/telegraf-4.9.1.tgz",
"integrity": "sha512-MukWpKvAZ6/HpT3yHXz+jwUf2HsPa9TcsqPLQjJ+kHNGUS2PLgaNX690ExdWmWPuxjVjC4wNHmZ9JetO3C/tVA==",
"version": "4.11.2",
"resolved": "https://registry.npmjs.org/telegraf/-/telegraf-4.11.2.tgz",
"integrity": "sha512-RGEh+NXkHbq1KcSSbJeVYhHMrEN4rymd9DSe3SoIV0886bJPBHLzYCNrOqnk9aeZE2Idwh5uK0X/xbR6jScQKQ==",
"requires": {
"abort-controller": "^3.0.0",
"debug": "^4.3.3",
"debug": "^4.3.4",
"mri": "^1.2.0",
"node-fetch": "^2.6.7",
"p-timeout": "^4.1.0",
"safe-compare": "^1.1.4",
"sandwich-stream": "^2.0.2",
"typegram": "^3.11.0"
"typegram": "^4.1.0"
}
},
"tiny-lru": {
@@ -5056,9 +5056,9 @@
"integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="
},
"typegram": {
"version": "3.11.0",
"resolved": "https://registry.npmjs.org/typegram/-/typegram-3.11.0.tgz",
"integrity": "sha512-4p6u+AFognlsDgBue8Hla2jO7Ax+UQXcLa27LC7xDdAeR9LTe+Cr4vJrYpoO1wgj/BFWgXTeboaH/+1YgWyfpA=="
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/typegram/-/typegram-4.2.0.tgz",
"integrity": "sha512-rObfoGpDlmWUhmggpWd2I/4xLsdGPDvfvrLpLxV4pBTBL2BBjm+7x7IOmTwJRV6Qe4UAkWdHq0ZYoTYAAPE5YA=="
},
"typescript": {
"version": "4.5.2",
@@ -5108,7 +5108,7 @@
},
"uWebSockets.js": {
"version": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#806df48c9da86af7b3341f3e443388c7cd15c3de",
"from": "uWebSockets.js@github:uNetworking/uWebSockets.js#v20.15.0"
"from": "uWebSockets.js@github:uNetworking/uWebSockets.js#v20.19.0"
},
"vary": {
"version": "1.1.2",
@@ -5135,9 +5135,9 @@
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"ws": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
"integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==",
"version": "8.12.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz",
"integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==",
"requires": {}
},
"xmlhttprequest-ssl": {
package.json
+10 -10
View File
@@ -30,8 +30,8 @@
"@eosrio/node-abieos": "^2.1.1",
"@fastify/autoload": "4.0.1",
"@fastify/cors": "7.0.0",
"@fastify/formbody": "^6.0.0",
"@fastify/rate-limit": "^6.0.0",
"@fastify/formbody": "^6.0.1",
"@fastify/rate-limit": "^6.0.1",
"@fastify/redis": "^5.0.0",
"@fastify/swagger": "6.1.0",
"@pm2/io": "^5.0.0",
@@ -51,26 +51,26 @@
"ioredis": "^4.28.5",
"lodash": "^4.17.21",
"moment": "^2.29.4",
"nodemailer": "^6.7.8",
"nodemailer": "^6.9.0",
"portfinder": "^1.0.32",
"pino-pretty": "^9.1.1",
"socket.io": "4.5.4",
"socket.io-client": "4.5.4",
"socket.io-redis": "^6.1.1",
"telegraf": "^4.9.1",
"telegraf": "^4.11.2",
"typescript": "^4.5.2",
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.15.0",
"ws": "^8.11.0"
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.19.0",
"ws": "^8.12.0"
},
"devDependencies": {
"@types/amqplib": "^0.8.2",
"@types/async": "^3.2.16",
"@types/global-agent": "^2.1.1",
"@types/ioredis": "^4.28.10",
"@types/lodash": "^4.14.177",
"@types/node": "^18.11.9",
"@types/nodemailer": "^6.4.4",
"@types/ws": "^8.5.3"
"@types/lodash": "^4.14.191",
"@types/node": "^18.11.18",
"@types/nodemailer": "^6.4.7",
"@types/ws": "^8.5.4"
},
"optionalDependencies": {
"bufferutil": "^4.0.7",
workers/state-reader.ts
+30 -18
View File
@@ -40,6 +40,7 @@ export default class StateReader extends HyperionWorker {
private shipInitStatus: any;
private forkedBlocks = new Map<string, number>();
private idle = true;
constructor() {
super();
@@ -122,7 +123,7 @@ export default class StateReader extends HyperionWorker {
console.log('Message nacked!');
console.log(err.message);
} else {
process.send({event: 'read_block', live: this.isLiveReader});
process.send({event: 'read_block', live: this.isLiveReader, block_num: d.num});
}
});
@@ -231,7 +232,7 @@ export default class StateReader extends HyperionWorker {
case 'stop': {
if (this.isLiveReader) {
console.log('[LIVE READER] Closing Websocket');
this.ship.close();
this.ship.close(true);
setTimeout(() => {
console.log('[LIVE READER] Process killed');
process.exit(1);
@@ -262,9 +263,7 @@ export default class StateReader extends HyperionWorker {
pending++;
}
});
if (pending === this.lastPendingCount && pending > 0) {
// console.log(`[${process.env['worker_id']}] Pending blocks: ${pending}`);
} else {
if (!(pending === this.lastPendingCount && pending > 0)) {
this.lastPendingCount = pending;
}
}
@@ -310,7 +309,7 @@ export default class StateReader extends HyperionWorker {
if (!process.env.worker_role) {
console.log("[FATAL ERROR] undefined role! Exiting now!");
this.ship.close();
this.ship.close(false);
process.exit(1);
return;
}
@@ -359,7 +358,7 @@ export default class StateReader extends HyperionWorker {
}
} else {
this.ship.close();
this.ship.close(true);
process.exit(1);
}
@@ -369,6 +368,7 @@ export default class StateReader extends HyperionWorker {
const res = result[1];
if (res['this_block']) {
this.idle = false;
const blk_num = res['this_block']['block_num'];
const lib = res['last_irreversible'];
const task_payload = {num: blk_num, content: data};
@@ -449,12 +449,15 @@ export default class StateReader extends HyperionWorker {
}
}
} else {
hLog(`Reader is idle! - Head at: ${result[1].head.block_num}`);
this.ship.close();
process.send({
event: 'kill_worker',
id: process.env['worker_id']
});
this.idle = true;
// hLog(`Reader is idle! - Head at: ${result[1].head.block_num}`);
// this.ship.close(true);
// const queueSize = [this.stageOneDistQueue.length(), this.blockReadingQueue.length()];
// hLog(queueSize);
// process.send({
// event: 'kill_worker',
// id: process.env['worker_id']
// });
}
}
@@ -514,8 +517,12 @@ export default class StateReader extends HyperionWorker {
request.start_block_num = parseInt(first_block > 0 ? first_block.toString() : '1', 10);
request.end_block_num = parseInt(last_block.toString(), 10);
const reqType = 'get_blocks_request_' + this.shipRev;
debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`);
this.send([reqType, request]);
if (this.ship.connected) {
debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`);
this.send([reqType, request]);
} else {
hLog('[Warning] Request failed - SHIP is not online!');
}
}
private send(req_data: (string | any)[]) {
@@ -528,8 +535,12 @@ export default class StateReader extends HyperionWorker {
this.local_block_num = request.start_block_num - 1;
request.end_block_num = parseInt(last, 10);
const reqType = 'get_blocks_request_' + this.shipRev;
debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`);
this.send([reqType, request]);
if (this.ship.connected) {
debugLog(`Reader ${process.env.worker_id} sending ${reqType} from: ${request.start_block_num} to: ${request.end_block_num}`);
this.send([reqType, request]);
} else {
hLog('[Warning] Request failed - SHIP is not online!');
}
}
private async deleteForkedBlock(block_id: string) {
@@ -577,6 +588,7 @@ export default class StateReader extends HyperionWorker {
id: targetBlock
});
targetBlock++;
console.log(blockData);
if (blockData.body) {
const targetBlockId = blockData.body._source.block_id;
this.forkedBlocks.set(targetBlockId, Date.now());
@@ -659,7 +671,7 @@ export default class StateReader extends HyperionWorker {
private handleLostConnection() {
this.recovery = true;
this.ship.close();
this.ship.close(false);
hLog(`Retrying connection in 5 seconds... [attempt: ${this.reconnectCount + 1}]`);
debugLog(`PENDING REQUESTS:', ${this.pendingRequest}`);
debugLog(`LOCAL BLOCK:', ${this.local_block_num}`);