Merge pull request #142 from eosrio/large-tx-fix

Large tx-sig fix

api/helpers/cacheManager.ts

@@ -28,19 +28,19 @@ export class CacheManager {
         setInterval(() => {
             try {
                 // remove expired entries
-                let removeCount = 0;
+                // let removeCount = 0;
                 const now = Date.now();
                 this.v1Caches.forEach((pathCacheMap: Map<string, CachedEntry>, pathKey: string) => {
                     pathCacheMap.forEach((cache: CachedEntry, entryKey: string, map: Map<string, CachedEntry>) => {
                         if (cache.exp < now) {
                             map.delete(entryKey);
-                            removeCount++;
+                            // removeCount++;
                         }
                     });
                 });
-                if (removeCount > 0) {
-                    console.log(`${removeCount} expired cache entries removed`);
-                }
+                // if (removeCount > 0) {
+                //     console.log(`${removeCount} expired cache entries removed`);
+                // }
             } catch (e) {
                 console.log(e);
             }

api/helpers/functions.ts

@@ -584,7 +584,7 @@ export function chainApiHandler(fastify: FastifyInstance) {
         // check cache
         const [cachedData, hash, path] = fastify.cacheManager.getCachedData(request);
         if (cachedData) {
-            // console.log('cache hit:', path, hash);
+            console.log('cache hit:', path, hash);
             reply.headers({'hyperion-cached': true}).send(cachedData);
         } else {
             // call actual request

api/routes.ts

@@ -62,7 +62,7 @@ export function registerRoutes(server: FastifyInstance) {
     // chain api redirects
     addRoute(server, 'v1-chain', '/v1/chain');
 
-	// other v1 requests
+    // other v1 requests
     server.route({
         url: '/v1/chain/*',
         method: ["GET", "POST"],
@@ -95,6 +95,25 @@ export function registerRoutes(server: FastifyInstance) {
         }
     });
 
+    // global onRequest hook to collect hourly statistics for each path on redis
+    server.addHook('onResponse', (request: FastifyRequest, reply: FastifyReply, done) => {
+        const path = request.url.split('?')[0].replace(/\/$/, '');
+        const now = new Date();
+        const ts = now.getTime();
+        // normalized time in hours
+        const factor = 1000 * 60 * 60;
+        const ts_hour_unix = Math.floor(ts / factor) * factor;
+        const key = `stats:${server.manager.chain}:H:${ts_hour_unix}`;
+        // console.log(reply.statusCode, key, new Date(ts_hour_unix));
+        server.redis.multi()
+            .incr(`stats:${server.manager.chain}:total:${reply.statusCode}:${path}`)
+            .zincrby(key, 1, `[${reply.statusCode}]${path}`)
+            .expire(key, 60 * 60 * 24 * 7)
+            .exec()
+            .catch(console.log);
+        done();
+    });
+
     server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
         console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
         done();

api/routes/v1-chain/abi_bin_to_json/index.ts

@@ -9,11 +9,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         {
             "code": {$ref: 'AccountName#'},
             "action": {$ref: 'AccountName#'},
-            "binargs": {
-                "type": "string",
-                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
-                "title": "Hex"
-            }
+            "binargs": {"type": "string"}
         },
         ["code", "action", "binargs"]
     );

api/routes/v1-chain/abi_json_to_bin/index.ts

@@ -7,13 +7,14 @@ export default function (fastify: FastifyInstance, opts: any, next) {
         getRouteName(__filename),
         'Convert JSON object to binary',
         {
-            "binargs": {
-                "type": "string",
-                "pattern": "^(0x)(([0-9a-f][0-9a-f])+)?$",
-                "title": "Hex"
+            "code": {$ref: 'AccountName#'},
+            "action": {$ref: 'AccountName#'},
+            "args": {
+                type: 'object',
+                additionalProperties: true
             }
         },
-        ["binargs"]
+        ["code", "action", "args"]
     );
     next();
 }

api/routes/v1-chain/get_table_rows/index.ts

@@ -37,7 +37,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                 "type": "string"
             }
         },
-        ["code", "table"]
+        ["code", "table", "scope"]
     );
     next();
 }

api/routes/v2-stats/get_api_usage/get_api_usage.ts

@@ -0,0 +1,53 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {timedQuery} from "../../../helpers/functions";
+import {parseInt} from "lodash";
+
+async function getApiUsage(fastify: FastifyInstance, request: FastifyRequest) {
+    const response = {
+        total: {responses: {}},
+        buckets: []
+    } as any;
+    const data = await fastify.redis.keys(`stats:${fastify.manager.chain}:*`);
+    const buckets = [];
+    for (const key of data) {
+        const parts = key.split(':');
+        if (parts[2] === 'H') {
+            const bucket = {
+                timestamp: new Date(parseInt(parts[3])),
+                responses: {}
+            };
+            const sortedSet = await fastify.redis.zrange(key, 0, -1, 'WITHSCORES');
+            for (let i = 0; i < sortedSet.length; i += 2) {
+                const [status, path] = sortedSet[i].split(']');
+                if (status) {
+                    const statusCode = status.slice(1);
+                    if (!bucket.responses[statusCode]) {
+                        bucket.responses[statusCode] = {};
+                    }
+                    if (statusCode) {
+                        bucket.responses[statusCode][path] = parseInt(sortedSet[i + 1]);
+                    }
+                }
+            }
+            response.buckets.push(bucket);
+        } else {
+            if (parts[2] === 'total' && parts[4]) {
+                const statusCode = parts[3];
+                if (!response.total.responses[statusCode]) {
+                    response.total.responses[statusCode] = {};
+                }
+                const value = await fastify.redis.get(key);
+                response.total.responses[statusCode][parts[4]] = parseInt(value);
+            }
+        }
+    }
+    // reverse sorte by date
+    response.buckets.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+    return response;
+}
+
+export function getApiUsageHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getApiUsage, fastify, request, route));
+    }
+}

api/routes/v2-stats/get_api_usage/index.ts

@@ -0,0 +1,41 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getApiUsageHandler} from "./get_api_usage";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get hyperion api usage statistics',
+        summary: 'get hyperion api usage statistics',
+        tags: ['stats'],
+        response: extendResponseSchema({
+            total: {
+                type: 'object',
+                properties: {
+                    responses: {
+                        additionalProperties: true
+                    }
+                }
+            },
+            buckets: {
+                type: 'array',
+                items: {
+                    type: 'object',
+                    properties: {
+                        timestamp: {type: 'string'},
+                        responses: {
+                            additionalProperties: true
+                        }
+                    }
+                }
+            }
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getApiUsageHandler,
+        schema
+    );
+    next();
+}

api/server.ts

@@ -16,7 +16,6 @@ import {io, Socket} from "socket.io-client";
 import {CacheManager} from "./helpers/cacheManager";
 
 import {bootstrap} from 'global-agent';
-import {Api} from "eosjs";
 
 class HyperionApiServer {
 
@@ -227,8 +226,8 @@ class HyperionApiServer {
             hLog('Failed to check elasticsearch version!');
             process.exit();
         });
-        hLog('Elasticsearch validated!');
 
+        hLog('Elasticsearch validated!');
         hLog('Registering plugins...');
 
         registerPlugins(this.fastify, this.pluginParams);

connections/amqp.ts

@@ -17,17 +17,16 @@ export async function createConnection(config): Promise<Connection> {
 }
 
 export function getAmpqUrl(config): string {
-	let frameMaxValue = '0x10000';
-	if (config.frameMax) {
-		frameMaxValue = config.frameMax;
-	}
-	const u = encodeURIComponent(config.user);
-	const p = encodeURIComponent(config.pass);
-	const v = encodeURIComponent(config.vhost)
-	return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
+    let frameMaxValue = '0x10000';
+    if (config.frameMax) {
+        frameMaxValue = config.frameMax;
+    }
+    const u = encodeURIComponent(config.user);
+    const p = encodeURIComponent(config.pass);
+    const v = encodeURIComponent(config.vhost);
+    return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
 }
 
-
 async function createChannels(connection) {
 	try {
 		const channel = await connection.createChannel();

connections/manager.class.ts

@@ -9,7 +9,6 @@ import {StateHistorySocket} from "./state-history";
 import fetch from 'cross-fetch';
 import {exec} from "child_process";
 import {hLog} from "../helpers/common_functions";
-import {readFileSync} from "fs";
 
 export class ConnectionManager {
 
@@ -20,8 +19,8 @@ export class ConnectionManager {
     last_commit_hash: string;
     current_version: string;
 
-    private readonly esIngestClients: Client[];
-    private esIngestClient: Client;
+    esIngestClients: Client[];
+    esIngestClient: Client;
 
     constructor(private cm: ConfigurationModule) {
         this.config = cm.config;

ecosystem.config.js

@@ -20,4 +20,11 @@ readdirSync(chainsRoot)
         }
     });
 
+apps.push({
+    name: 'hyperion-governor',
+    namespace: 'hyperion',
+    script: 'governor/server/index.js',
+    watch: false,
+});
+
 module.exports = {apps};

modules/parsers/base-parser.ts

@@ -192,7 +192,6 @@ export abstract class BaseParser {
         if (ds_act) {
 
             if (ds_act.account && ds_act.name && ds_act.authorization) {
-                console.log(ds_act);
                 action.act.data = ds_act.data;
             }
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "hyperion-history",
-  "version": "3.3.9-6",
+  "version": "3.3.9-7",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "hyperion-history",
-      "version": "3.3.9-6",
+      "version": "3.3.9-7",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
@@ -24,7 +24,7 @@
         "base-x": "^4.0.0",
         "cli-progress": "^3.12.0",
         "commander": "^8.3.0",
-        "cross-fetch": "^3.1.5",
+        "cross-fetch": "^3.1.8",
         "eosjs": "^22.1.0",
         "fast-json-stringify": "2.7.13",
         "fastify": "3.29.4",
@@ -37,32 +37,32 @@
         "lodash": "^4.17.21",
         "moment": "^2.29.4",
         "nodemailer": "^6.9.0",
-        "pino-pretty": "^9.1.1",
+        "pino-pretty": "^10.2.0",
         "portfinder": "^1.0.32",
-        "socket.io": "4.6.1",
-        "socket.io-client": "4.6.1",
+        "socket.io": "4.7.2",
+        "socket.io-client": "4.7.2",
         "socket.io-redis": "^6.1.1",
         "telegraf": "^4.12.3-canary.1",
         "typescript": "^4.5.2",
-        "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.24.0",
-        "ws": "^8.12.1"
+        "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.31.0",
+        "ws": "^8.14.2"
       },
       "devDependencies": {
-        "@types/amqplib": "^0.10.1",
+        "@types/amqplib": "^0.10.2",
         "@types/async": "^3.2.18",
         "@types/global-agent": "^2.1.1",
         "@types/ioredis": "^4.28.10",
-        "@types/lodash": "^4.14.191",
-        "@types/node": "^20.1.2",
-        "@types/nodemailer": "^6.4.7",
-        "@types/ws": "^8.5.4"
+        "@types/lodash": "^4.14.197",
+        "@types/node": "^20.5.7",
+        "@types/nodemailer": "^6.4.9",
+        "@types/ws": "^8.5.5"
       },
       "engines": {
         "node": ">=18"
       },
       "optionalDependencies": {
         "bufferutil": "^4.0.7",
-        "utf-8-validate": "^5.0.10"
+        "utf-8-validate": "^6.0.3"
       }
     },
     "node_modules/@acuminous/bitsyntax": {
@@ -199,9 +199,9 @@
       }
     },
     "node_modules/@opencensus/core/node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
       "bin": {
         "semver": "bin/semver"
       }
@@ -234,17 +234,17 @@
       }
     },
     "node_modules/@opencensus/propagation-b3/node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
       "bin": {
         "semver": "bin/semver"
       }
     },
     "node_modules/@pm2/io": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/@pm2/io/-/io-5.0.0.tgz",
-      "integrity": "sha512-3rToDVJaRoob5Lq8+7Q2TZFruoEkdORxwzFpZaqF4bmH6Bkd7kAbdPrI/z8X6k1Meq5rTtScM7MmDgppH6aLlw==",
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/@pm2/io/-/io-5.0.2.tgz",
+      "integrity": "sha512-XAvrNoQPKOyO/jJyCu8jPhLzlyp35MEf7w/carHXmWKddPzeNOFSEpSEqMzPDawsvpxbE+i918cNN+MwgVsStA==",
       "dependencies": {
         "@opencensus/core": "0.0.9",
         "@opencensus/propagation-b3": "0.0.8",
@@ -252,7 +252,7 @@
         "debug": "~4.3.1",
         "eventemitter2": "^6.3.1",
         "require-in-the-middle": "^5.0.0",
-        "semver": "6.3.0",
+        "semver": "~7.5.4",
         "shimmer": "^1.2.0",
         "signal-exit": "^3.0.3",
         "tslib": "1.9.3"
@@ -269,14 +269,6 @@
         "lodash": "^4.17.14"
       }
     },
-    "node_modules/@pm2/io/node_modules/semver": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-      "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
-      "bin": {
-        "semver": "bin/semver.js"
-      }
-    },
     "node_modules/@sindresorhus/is": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz",
@@ -305,9 +297,9 @@
       }
     },
     "node_modules/@types/amqplib": {
-      "version": "0.10.1",
-      "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.1.tgz",
-      "integrity": "sha512-j6ANKT79ncUDnAs/+9r9eDujxbeJoTjoVu33gHHcaPfmLQaMhvfbH2GqSe8KUM444epAp1Vl3peVOQfZk3UIqA==",
+      "version": "0.10.2",
+      "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.2.tgz",
+      "integrity": "sha512-K0qC2YR0ZcQWWMOifg4yvCAu5wi/V6wY6MnMS4LSvqx6qwXBAhxno6OBN8D76FIaajLNfgHKOXobZBL/uAwXAQ==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
@@ -372,20 +364,20 @@
       }
     },
     "node_modules/@types/lodash": {
-      "version": "4.14.194",
-      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.194.tgz",
-      "integrity": "sha512-r22s9tAS7imvBt2lyHC9B8AGwWnXaYb1tY09oyLkXDs4vArpYJzw09nj8MLx5VfciBPGIb+ZwG0ssYnEPJxn/g==",
+      "version": "4.14.197",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.197.tgz",
+      "integrity": "sha512-BMVOiWs0uNxHVlHBgzTIqJYmj+PgCo4euloGF+5m4okL3rEYzM2EEv78mw8zWSMM57dM7kVIgJ2QDvwHSoCI5g==",
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "20.1.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.2.tgz",
-      "integrity": "sha512-CTO/wa8x+rZU626cL2BlbCDzydgnFNgc19h4YvizpTO88MFQxab8wqisxaofQJ/9bLGugRdWIuX/TbIs6VVF6g=="
+      "version": "20.5.7",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.7.tgz",
+      "integrity": "sha512-dP7f3LdZIysZnmvP3ANJYTSwg+wLLl8p7RqniVlV7j+oXSXAbt9h0WIBFmJy5inWZoX9wZN6eXx+YXd9Rh3RBA=="
     },
     "node_modules/@types/nodemailer": {
-      "version": "6.4.7",
-      "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.7.tgz",
-      "integrity": "sha512-f5qCBGAn/f0qtRcd4SEn88c8Fp3Swct1731X4ryPKqS61/A3LmmzN8zaEz7hneJvpjFbUUgY7lru/B/7ODTazg==",
+      "version": "6.4.9",
+      "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.9.tgz",
+      "integrity": "sha512-XYG8Gv+sHjaOtUpiuytahMy2mM3rectgroNbs6R3djZEKmPNiIJwe9KqOJBGzKKnNZNKvnuvmugBgpq3w/S0ig==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
@@ -400,9 +392,9 @@
       }
     },
     "node_modules/@types/ws": {
-      "version": "8.5.4",
-      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz",
-      "integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==",
+      "version": "8.5.5",
+      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.5.tgz",
+      "integrity": "sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
@@ -517,9 +509,9 @@
       }
     },
     "node_modules/async-listener/node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
       "bin": {
         "semver": "bin/semver"
       }
@@ -810,11 +802,11 @@
       }
     },
     "node_modules/cross-fetch": {
-      "version": "3.1.5",
-      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
-      "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==",
+      "version": "3.1.8",
+      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz",
+      "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==",
       "dependencies": {
-        "node-fetch": "2.6.7"
+        "node-fetch": "^2.6.12"
       }
     },
     "node_modules/dateformat": {
@@ -990,9 +982,9 @@
       }
     },
     "node_modules/engine.io": {
-      "version": "6.4.2",
-      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.4.2.tgz",
-      "integrity": "sha512-FKn/3oMiJjrOEOeUub2WCox6JhxBXq/Zn3fZOMCBxKnNYtsdKjxhl7yR3fZhM9PV+rdE75SU5SYMc+2PGzo+Tg==",
+      "version": "6.5.2",
+      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.5.2.tgz",
+      "integrity": "sha512-IXsMcGpw/xRfjra46sVZVHiSWo/nJ/3g1337q9KNXtS6YRzbW5yIzTCb9DjhrBe7r3GZQR0I4+nq+4ODk5g/cA==",
       "dependencies": {
         "@types/cookie": "^0.4.1",
         "@types/cors": "^2.8.12",
@@ -1002,25 +994,39 @@
         "cookie": "~0.4.1",
         "cors": "~2.8.5",
         "debug": "~4.3.1",
-        "engine.io-parser": "~5.0.3",
+        "engine.io-parser": "~5.2.1",
         "ws": "~8.11.0"
       },
       "engines": {
-        "node": ">=10.0.0"
+        "node": ">=10.2.0"
       }
     },
     "node_modules/engine.io-client": {
-      "version": "6.4.0",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.4.0.tgz",
-      "integrity": "sha512-GyKPDyoEha+XZ7iEqam49vz6auPnNJ9ZBfy89f+rMMas8AuiMWOZ9PVzu8xb9ZC6rafUqiGHSCfu22ih66E+1g==",
+      "version": "6.5.2",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.2.tgz",
+      "integrity": "sha512-CQZqbrpEYnrpGqC07a9dJDz4gePZUgTPMU3NKJPSeQOyw27Tst4Pl3FemKoFGAlHzgZmKjoRmiJvbWfhCXUlIg==",
       "dependencies": {
         "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.1",
-        "engine.io-parser": "~5.0.3",
+        "engine.io-parser": "~5.2.1",
         "ws": "~8.11.0",
         "xmlhttprequest-ssl": "~2.0.0"
       }
     },
+    "node_modules/engine.io-client/node_modules/utf-8-validate": {
+      "version": "5.0.10",
+      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
+      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
+      "hasInstallScript": true,
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "node-gyp-build": "^4.3.0"
+      },
+      "engines": {
+        "node": ">=6.14.2"
+      }
+    },
     "node_modules/engine.io-client/node_modules/ws": {
       "version": "8.11.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
@@ -1042,9 +1048,9 @@
       }
     },
     "node_modules/engine.io-parser": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.0.6.tgz",
-      "integrity": "sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw==",
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.1.tgz",
+      "integrity": "sha512-9JktcM3u18nU9N2Lz3bWeBgxVgOKpw7yhRaoxQA3FUDZzzw+9WlA6p4G4u0RixNkg14fH7EfEc/RhpurtiROTQ==",
       "engines": {
         "node": ">=10.0.0"
       }
@@ -1057,6 +1063,20 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/engine.io/node_modules/utf-8-validate": {
+      "version": "5.0.10",
+      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
+      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
+      "hasInstallScript": true,
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "node-gyp-build": "^4.3.0"
+      },
+      "engines": {
+        "node": ">=6.14.2"
+      }
+    },
     "node_modules/engine.io/node_modules/ws": {
       "version": "8.11.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
@@ -1888,9 +1908,9 @@
       }
     },
     "node_modules/node-fetch": {
-      "version": "2.6.7",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
-      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
       "dependencies": {
         "whatwg-url": "^5.0.0"
       },
@@ -2124,9 +2144,9 @@
       }
     },
     "node_modules/pino-pretty": {
-      "version": "9.4.0",
-      "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-9.4.0.tgz",
-      "integrity": "sha512-NIudkNLxnl7MGj1XkvsqVyRgo6meFP82ECXF2PlOI+9ghmbGuBUUqKJ7IZPIxpJw4vhhSva0IuiDSAuGh6TV9g==",
+      "version": "10.2.0",
+      "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.2.0.tgz",
+      "integrity": "sha512-tRvpyEmGtc2D+Lr3FulIZ+R1baggQ4S3xD2Ar93KixFEDx6SEAUP3W5aYuEw1C73d6ROrNcB2IXLteW8itlwhA==",
       "dependencies": {
         "colorette": "^2.0.7",
         "dateformat": "^4.6.3",
@@ -2490,9 +2510,9 @@
       "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
     },
     "node_modules/semver": {
-      "version": "7.5.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz",
-      "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dependencies": {
         "lru-cache": "^6.0.0"
       },
@@ -2584,19 +2604,20 @@
       "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="
     },
     "node_modules/socket.io": {
-      "version": "4.6.1",
-      "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.6.1.tgz",
-      "integrity": "sha512-KMcaAi4l/8+xEjkRICl6ak8ySoxsYG+gG6/XfRCPJPQ/haCRIJBTL4wIl8YCsmtaBovcAXGLOShyVWQ/FG8GZA==",
+      "version": "4.7.2",
+      "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.7.2.tgz",
+      "integrity": "sha512-bvKVS29/I5fl2FGLNHuXlQaUH/BlzX1IN6S+NKLNZpBsPZIDH+90eQmCs2Railn4YUiww4SzUedJ6+uzwFnKLw==",
       "dependencies": {
         "accepts": "~1.3.4",
         "base64id": "~2.0.0",
+        "cors": "~2.8.5",
         "debug": "~4.3.2",
-        "engine.io": "~6.4.1",
+        "engine.io": "~6.5.2",
         "socket.io-adapter": "~2.5.2",
-        "socket.io-parser": "~4.2.1"
+        "socket.io-parser": "~4.2.4"
       },
       "engines": {
-        "node": ">=10.0.0"
+        "node": ">=10.2.0"
       }
     },
     "node_modules/socket.io-adapter": {
@@ -2607,6 +2628,20 @@
         "ws": "~8.11.0"
       }
     },
+    "node_modules/socket.io-adapter/node_modules/utf-8-validate": {
+      "version": "5.0.10",
+      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
+      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
+      "hasInstallScript": true,
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "node-gyp-build": "^4.3.0"
+      },
+      "engines": {
+        "node": ">=6.14.2"
+      }
+    },
     "node_modules/socket.io-adapter/node_modules/ws": {
       "version": "8.11.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
@@ -2628,23 +2663,23 @@
       }
     },
     "node_modules/socket.io-client": {
-      "version": "4.6.1",
-      "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.6.1.tgz",
-      "integrity": "sha512-5UswCV6hpaRsNg5kkEHVcbBIXEYoVbMQaHJBXJCyEQ+CiFPV1NIOY0XOFWG4XR4GZcB8Kn6AsRs/9cy9TbqVMQ==",
+      "version": "4.7.2",
+      "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.7.2.tgz",
+      "integrity": "sha512-vtA0uD4ibrYD793SOIAwlo8cj6haOeMHrGvwPxJsxH7CeIksqJ+3Zc06RvWTIFgiSqx4A3sOnTXpfAEE2Zyz6w==",
       "dependencies": {
         "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.2",
-        "engine.io-client": "~6.4.0",
-        "socket.io-parser": "~4.2.1"
+        "engine.io-client": "~6.5.2",
+        "socket.io-parser": "~4.2.4"
       },
       "engines": {
         "node": ">=10.0.0"
       }
     },
     "node_modules/socket.io-parser": {
-      "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.3.tgz",
-      "integrity": "sha512-JMafRntWVO2DCJimKsRTh/wnqVvO4hrfwOqtO7f+uzwsQMuxO6VwImtYxaQ+ieoyshWOTJyV0fA21lccEXRPpQ==",
+      "version": "4.2.4",
+      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.4.tgz",
+      "integrity": "sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==",
       "dependencies": {
         "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.1"
@@ -2809,25 +2844,6 @@
         "node": "^12.20.0 || >=14.13.1"
       }
     },
-    "node_modules/telegraf/node_modules/node-fetch": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz",
-      "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==",
-      "dependencies": {
-        "whatwg-url": "^5.0.0"
-      },
-      "engines": {
-        "node": "4.x || >=6.0.0"
-      },
-      "peerDependencies": {
-        "encoding": "^0.1.0"
-      },
-      "peerDependenciesMeta": {
-        "encoding": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/tiny-lru": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz",
@@ -2905,9 +2921,9 @@
       }
     },
     "node_modules/utf-8-validate": {
-      "version": "5.0.10",
-      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
-      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-6.0.3.tgz",
+      "integrity": "sha512-uIuGf9TWQ/y+0Lp+KGZCMuJWc3N9BHA+l/UmHd/oUHwJJDeysyTRxNQVkbzsIWfGFbRe3OcgML/i0mvVRPOyDA==",
       "hasInstallScript": true,
       "optional": true,
       "dependencies": {
@@ -2963,9 +2979,9 @@
       "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
     },
     "node_modules/ws": {
-      "version": "8.13.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
-      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
+      "version": "8.14.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
+      "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
       "engines": {
         "node": ">=10.0.0"
       },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperion-history",
-  "version": "3.3.9-6",
+  "version": "3.3.9-8",
   "description": "Scalable Full History API Solution for EOSIO based blockchains",
   "main": "launcher.js",
   "scripts": {
@@ -40,7 +40,7 @@
     "base-x": "^4.0.0",
     "cli-progress": "^3.12.0",
     "commander": "^8.3.0",
-    "cross-fetch": "^3.1.5",
+    "cross-fetch": "^3.1.8",
     "eosjs": "^22.1.0",
     "fast-json-stringify": "2.7.13",
     "fastify": "3.29.4",
@@ -53,28 +53,28 @@
     "lodash": "^4.17.21",
     "moment": "^2.29.4",
     "nodemailer": "^6.9.0",
-    "pino-pretty": "^9.1.1",
+    "pino-pretty": "^10.2.0",
     "portfinder": "^1.0.32",
-    "socket.io": "4.6.1",
-    "socket.io-client": "4.6.1",
+    "socket.io": "4.7.2",
+    "socket.io-client": "4.7.2",
     "socket.io-redis": "^6.1.1",
     "telegraf": "^4.12.3-canary.1",
     "typescript": "^4.5.2",
-    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.24.0",
-    "ws": "^8.12.1"
+    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.31.0",
+    "ws": "^8.14.2"
   },
   "devDependencies": {
-    "@types/amqplib": "^0.10.1",
+    "@types/amqplib": "^0.10.2",
     "@types/async": "^3.2.18",
     "@types/global-agent": "^2.1.1",
     "@types/ioredis": "^4.28.10",
-    "@types/lodash": "^4.14.191",
-    "@types/node": "^20.1.2",
-    "@types/nodemailer": "^6.4.7",
-    "@types/ws": "^8.5.4"
+    "@types/lodash": "^4.14.197",
+    "@types/node": "^20.5.7",
+    "@types/nodemailer": "^6.4.9",
+    "@types/ws": "^8.5.5"
   },
   "optionalDependencies": {
     "bufferutil": "^4.0.7",
-    "utf-8-validate": "^5.0.10"
+    "utf-8-validate": "^6.0.3"
   }
 }

scripts/hyp-repair.ts

@@ -509,43 +509,73 @@ async function repairChain(chain: string, file: string, args: any) {
     await fillMissingBlocksFromFile(args.host, chain, file, args.dry);
 }
 
-async function fillMissingBlocksFromFile(host: string, chain: string, file: string, dryRun: string) {
+async function fillMissingBlocksFromFile(host, chain, file, dryRun) {
     const config = readConnectionConfig();
     const controlPort = config.chains[chain].control_port;
     let hyperionIndexer = `ws://localhost:${controlPort}`;
     if (host) {
-        hyperionIndexer = 'ws://' + host + ':' + controlPort;
+        hyperionIndexer = `ws://${host}:${controlPort}`;
     }
     const controller = new WebSocket(hyperionIndexer + '/local');
+
+    // Function to send Chunk
+    async function sendChunk(chunk): Promise<void> {
+        return new Promise((resolve, reject) => {
+            const payload = {
+                "event": "fill_missing_blocks",
+                "data": chunk
+            };
+            controller.send(JSON.stringify(payload));
+            
+            // Wait repair_completed confirmation
+            controller.once('message', (data) => {
+                const parsed = JSON.parse(data.toString());
+                if (parsed.event === 'repair_completed') {
+                    // console.log(`Hyperion repair completed for chunk!`);
+                    resolve();
+                }
+            });
+        });
+    }
+
     controller.on('open', async () => {
         console.log('Connected to Hyperion Controller');
         const parsedFile = JSON.parse(readFileSync(file).toString());
+        const chunkSize = 20; // Chunk size
+        
         if (!dryRun) {
-            const payload = {
-                "event": "fill_missing_blocks",
-                "data": parsedFile
-            };
-            controller.send(JSON.stringify(payload));
+            const totalChunks = Math.ceil(parsedFile.length / chunkSize);
+            let completedChunks = 0;
+            for (let i = 0; i < parsedFile.length; i += chunkSize) {
+                const progress = (completedChunks / totalChunks) * 100;
+                const progressBar = Array(Math.round(progress / 2)).fill('#').join('');
+                process.stdout.write(`Progress: [${progressBar}] ${progress.toFixed(2)}%     \r`);
+                //  console.log(`Progress: [${progressBar}] ${progress.toFixed(2)}%`);
+
+                const chunk = parsedFile.slice(i, i + chunkSize);
+                await sendChunk(chunk);
+
+                 // Update progess bar
+                 completedChunks++;
+            }
+            console.log();
+            controller.close();
         } else {
             console.log('Dry run, skipping repair');
             controller.close();
         }
     });
-    controller.on('message', (data) => {
-        const parsed = JSON.parse(data.toString());
-        if (parsed.event === 'repair_completed') {
-            console.log(`Hyperion repair completed!`);
-            controller.close();
-        }
-    });
+
     controller.on('close', () => {
         console.log('Disconnected from Hyperion Controller');
     });
+
     controller.on('error', (err) => {
         console.log(err);
     });
 }
 
+
 function viewFile(file: string) {
     const data = readFileSync(file, 'utf8');
     const parsed = JSON.parse(data);

workers/deserializer.ts

@@ -8,6 +8,7 @@ import {Type} from "../addons/eosjs-native/eosjs-serialize";
 import {debugLog, hLog} from "../helpers/common_functions";
 import {createHash} from "crypto";
 import flatstr from 'flatstr';
+import {Options} from "amqplib";
 
 const FJS = require('fast-json-stringify');
 
@@ -124,6 +125,9 @@ export default class MainDSWorker extends HyperionWorker {
 
     allowedDynamicContracts: Set<string> = new Set<string>();
 
+    backpressureQueue: any[] = [];
+    waitToSend = false;
+
     constructor() {
 
         super();
@@ -290,6 +294,16 @@ export default class MainDSWorker extends HyperionWorker {
             this.ch.consume(process.env['worker_queue'], (data) => {
                 this.consumerQueue.push(data).catch(console.log);
             });
+            this.ch.on('drain', args => {
+                this.waitToSend = false;
+                while (this.backpressureQueue.length > 0) {
+                    const msg = this.backpressureQueue.shift();
+                    const status = this.controlledSendToQueue(msg.queue, msg.payload, msg.options);
+                    if (!status) {
+                        break;
+                    }
+                }
+            });
         }
     }
 
@@ -478,6 +492,7 @@ export default class MainDSWorker extends HyperionWorker {
                                 hLog(`${block_num} was filtered with ${inline_count} actions!`);
                             }
                             try {
+                                trace[1].signatures = signatures;
                                 this.routeToPool(trace[1], {
                                     block_num,
                                     block_id,
@@ -485,8 +500,7 @@ export default class MainDSWorker extends HyperionWorker {
                                     ts,
                                     inline_count,
                                     filtered,
-                                    live: process.env['live_mode'],
-                                    signatures
+                                    live: process.env['live_mode']
                                 });
                             } catch (e) {
                                 hLog(e);
@@ -622,15 +636,33 @@ export default class MainDSWorker extends HyperionWorker {
         }
 
         const pool_queue = `${this.chain}:ds_pool:${selected_q}`;
-        if (this.ch_ready) {
-            // console.log('selected_q', pool_queue);
-            this.ch.sendToQueue(pool_queue, bufferFromJson(trace, true), {headers});
-            return true;
+        const payload = bufferFromJson(trace, true);
+
+        if (!this.waitToSend) {
+            if (this.ch_ready) {
+                this.controlledSendToQueue(pool_queue, payload, {headers});
+                return true;
+            } else {
+                return false;
+            }
         } else {
+            this.backpressureQueue.push({
+                queue: pool_queue,
+                payload: payload,
+                options: {headers}
+            });
             return false;
         }
     }
 
+    controlledSendToQueue(pool_queue: string, payload: Buffer, options: Options.Publish): boolean {
+        const enqueueResult = this.ch.sendToQueue(pool_queue, payload, options);
+        if (!enqueueResult) {
+            this.waitToSend = true;
+        }
+        return enqueueResult;
+    }
+
     createSerialBuffer(inputArray) {
         return new Serialize.SerialBuffer({textEncoder: this.txEnc, textDecoder: this.txDec, array: inputArray});
     }
@@ -1160,7 +1192,7 @@ export default class MainDSWorker extends HyperionWorker {
                 let jsonRow = await this.processContractRowNative(payload, block_num);
 
                 if (jsonRow?.value && !jsonRow['_blacklisted']) {
-                    console.log(jsonRow);
+                    debugLog(jsonRow);
                     debugLog('Delta DS failed ->>', jsonRow);
                     jsonRow = await this.processContractRowNative(payload, block_num - 1);
                     debugLog('Retry with previous ABI ->>', jsonRow);
@@ -1495,7 +1527,7 @@ export default class MainDSWorker extends HyperionWorker {
                             } catch (e) {
                                 hLog(`Delta struct [${key}] processing error: ${e.message}`);
                                 hLog(e);
-                                console.log(data[1]);
+                                hLog(data[1]);
                             }
                         }
                     }

workers/ds-pool.ts

@@ -454,8 +454,8 @@ export default class DSPoolWorker extends HyperionWorker {
     }
 
     async processTraces(transaction_trace, extra) {
-        const {cpu_usage_us, net_usage_words} = transaction_trace;
-        const {block_num, block_id, producer, ts, inline_count, filtered, live, signatures} = extra;
+        const {cpu_usage_us, net_usage_words, signatures} = transaction_trace;
+        const {block_num, block_id, producer, ts, inline_count, filtered, live} = extra;
 
         if (transaction_trace.status === 0) {
             let action_count = 0;

workers/state-reader.ts

@@ -375,7 +375,7 @@ export default class StateReader extends HyperionWorker {
                 this.shipInitStatus = result[1];
                 hLog(`\n| SHIP Status Report\n| Init block: ${this.shipInitStatus['chain_state_begin_block']}\n| Head block: ${this.shipInitStatus['chain_state_end_block']}`);
                 const chain_state_begin_block = this.shipInitStatus['chain_state_begin_block'];
-                if (!this.conf.indexer.disable_reading) {
+                if (!this.conf.indexer.disable_reading || process.env['worker_role'] === 'repair_reader') {
 
                     switch (process.env['worker_role']) {