fix

Large tx-sig fix

.gitignore

@@ -68,8 +68,8 @@ helpers/**/*.js.map
 interfaces/**/*.js
 interfaces/**/*.js.map
 
-modules/**/*.js
-modules/**/*.js.map
+#modules/**/*.js
+#modules/**/*.js.map
 
 addons/**/*.js
 addons/**/*.js.map

README.md

@@ -91,7 +91,7 @@ to collect action traces and state deltas. Provides data via websocket to the in
 #### 2.7 Hyperion Stream Client (optional)
 
 Web and Node.js client for real-time streaming on enabled hyperion
-providers. [Documentation](https://hyperion.docs.eosrio.io/stream_client/)
+providers. [Documentation](https://hyperion.docs.eosrio.io/dev/stream_client/)
 
 #### 2.8 Hyperion Plugins (optional)
 

api/helpers/cacheManager.ts

@@ -28,19 +28,19 @@ export class CacheManager {
         setInterval(() => {
             try {
                 // remove expired entries
-                let removeCount = 0;
+                // let removeCount = 0;
                 const now = Date.now();
                 this.v1Caches.forEach((pathCacheMap: Map<string, CachedEntry>, pathKey: string) => {
                     pathCacheMap.forEach((cache: CachedEntry, entryKey: string, map: Map<string, CachedEntry>) => {
                         if (cache.exp < now) {
                             map.delete(entryKey);
-                            removeCount++;
+                            // removeCount++;
                         }
                     });
                 });
-                if (removeCount > 0) {
-                    console.log(`${removeCount} expired cache entries removed`);
-                }
+                // if (removeCount > 0) {
+                //     console.log(`${removeCount} expired cache entries removed`);
+                // }
             } catch (e) {
                 console.log(e);
             }

api/helpers/functions.ts

@@ -584,7 +584,7 @@ export function chainApiHandler(fastify: FastifyInstance) {
         // check cache
         const [cachedData, hash, path] = fastify.cacheManager.getCachedData(request);
         if (cachedData) {
-            // console.log('cache hit:', path, hash);
+            console.log('cache hit:', path, hash);
             reply.headers({'hyperion-cached': true}).send(cachedData);
         } else {
             // call actual request

api/routes.ts

@@ -62,7 +62,7 @@ export function registerRoutes(server: FastifyInstance) {
     // chain api redirects
     addRoute(server, 'v1-chain', '/v1/chain');
 
-	// other v1 requests
+    // other v1 requests
     server.route({
         url: '/v1/chain/*',
         method: ["GET", "POST"],
@@ -95,6 +95,25 @@ export function registerRoutes(server: FastifyInstance) {
         }
     });
 
+    // global onRequest hook to collect hourly statistics for each path on redis
+    server.addHook('onResponse', (request: FastifyRequest, reply: FastifyReply, done) => {
+        const path = request.url.split('?')[0].replace(/\/$/, '');
+        const now = new Date();
+        const ts = now.getTime();
+        // normalized time in hours
+        const factor = 1000 * 60 * 60;
+        const ts_hour_unix = Math.floor(ts / factor) * factor;
+        const key = `stats:${server.manager.chain}:H:${ts_hour_unix}`;
+        // console.log(reply.statusCode, key, new Date(ts_hour_unix));
+        server.redis.multi()
+            .incr(`stats:${server.manager.chain}:total:${reply.statusCode}:${path}`)
+            .zincrby(key, 1, `[${reply.statusCode}]${path}`)
+            .expire(key, 60 * 60 * 24 * 7)
+            .exec()
+            .catch(console.log);
+        done();
+    });
+
     server.addHook('onError', (request: FastifyRequest, reply: FastifyReply, error: FastifyError, done) => {
         console.log(`[${request.headers['x-real-ip'] || request.ip}] ${request.method} ${request.url} failed >> ${error.message}`);
         done();

api/routes/v1-chain/get_table_rows/index.ts

@@ -37,7 +37,7 @@ export default function (fastify: FastifyInstance, opts: any, next) {
                 "type": "string"
             }
         },
-        ["code", "table"]
+        ["code", "table", "scope"]
     );
     next();
 }

api/routes/v2-history/get_actions/definitions.ts

@@ -12,7 +12,14 @@ export const extendedActions = new Set([
     "buyrambytes",
     "delegatebw",
     "undelegatebw",
-    "voteproducer"
+    "voteproducer",
+    'statement',
+    'decision',
+    'batch',
+    'act',
+    'exec',
+    'votefor',
+    'voteagainst'
 ]);
 
 export const primaryTerms = [

api/routes/v2-stats/get_api_usage/get_api_usage.ts

@@ -0,0 +1,53 @@
+import {FastifyInstance, FastifyReply, FastifyRequest} from "fastify";
+import {timedQuery} from "../../../helpers/functions";
+import {parseInt} from "lodash";
+
+async function getApiUsage(fastify: FastifyInstance, request: FastifyRequest) {
+    const response = {
+        total: {responses: {}},
+        buckets: []
+    } as any;
+    const data = await fastify.redis.keys(`stats:${fastify.manager.chain}:*`);
+    const buckets = [];
+    for (const key of data) {
+        const parts = key.split(':');
+        if (parts[2] === 'H') {
+            const bucket = {
+                timestamp: new Date(parseInt(parts[3])),
+                responses: {}
+            };
+            const sortedSet = await fastify.redis.zrange(key, 0, -1, 'WITHSCORES');
+            for (let i = 0; i < sortedSet.length; i += 2) {
+                const [status, path] = sortedSet[i].split(']');
+                if (status) {
+                    const statusCode = status.slice(1);
+                    if (!bucket.responses[statusCode]) {
+                        bucket.responses[statusCode] = {};
+                    }
+                    if (statusCode) {
+                        bucket.responses[statusCode][path] = parseInt(sortedSet[i + 1]);
+                    }
+                }
+            }
+            response.buckets.push(bucket);
+        } else {
+            if (parts[2] === 'total' && parts[4]) {
+                const statusCode = parts[3];
+                if (!response.total.responses[statusCode]) {
+                    response.total.responses[statusCode] = {};
+                }
+                const value = await fastify.redis.get(key);
+                response.total.responses[statusCode][parts[4]] = parseInt(value);
+            }
+        }
+    }
+    // reverse sorte by date
+    response.buckets.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+    return response;
+}
+
+export function getApiUsageHandler(fastify: FastifyInstance, route: string) {
+    return async (request: FastifyRequest, reply: FastifyReply) => {
+        reply.send(await timedQuery(getApiUsage, fastify, request, route));
+    }
+}

api/routes/v2-stats/get_api_usage/index.ts

@@ -0,0 +1,41 @@
+import {FastifyInstance} from "fastify";
+import {addApiRoute, extendResponseSchema, getRouteName} from "../../../helpers/functions";
+import {getApiUsageHandler} from "./get_api_usage";
+
+export default function (fastify: FastifyInstance, opts: any, next) {
+    const schema = {
+        description: 'get hyperion api usage statistics',
+        summary: 'get hyperion api usage statistics',
+        tags: ['stats'],
+        response: extendResponseSchema({
+            total: {
+                type: 'object',
+                properties: {
+                    responses: {
+                        additionalProperties: true
+                    }
+                }
+            },
+            buckets: {
+                type: 'array',
+                items: {
+                    type: 'object',
+                    properties: {
+                        timestamp: {type: 'string'},
+                        responses: {
+                            additionalProperties: true
+                        }
+                    }
+                }
+            }
+        })
+    };
+    addApiRoute(
+        fastify,
+        'GET',
+        getRouteName(__filename),
+        getApiUsageHandler,
+        schema
+    );
+    next();
+}

api/server.ts

@@ -16,7 +16,6 @@ import {io, Socket} from "socket.io-client";
 import {CacheManager} from "./helpers/cacheManager";
 
 import {bootstrap} from 'global-agent';
-import {Api} from "eosjs";
 
 class HyperionApiServer {
 
@@ -227,8 +226,8 @@ class HyperionApiServer {
             hLog('Failed to check elasticsearch version!');
             process.exit();
         });
-        hLog('Elasticsearch validated!');
 
+        hLog('Elasticsearch validated!');
         hLog('Registering plugins...');
 
         registerPlugins(this.fastify, this.pluginParams);

connections/amqp.ts

@@ -17,17 +17,16 @@ export async function createConnection(config): Promise<Connection> {
 }
 
 export function getAmpqUrl(config): string {
-	let frameMaxValue = '0x10000';
-	if (config.frameMax) {
-		frameMaxValue = config.frameMax;
-	}
-	const u = encodeURIComponent(config.user);
-	const p = encodeURIComponent(config.pass);
-	const v = encodeURIComponent(config.vhost)
-	return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
+    let frameMaxValue = '0x10000';
+    if (config.frameMax) {
+        frameMaxValue = config.frameMax;
+    }
+    const u = encodeURIComponent(config.user);
+    const p = encodeURIComponent(config.pass);
+    const v = encodeURIComponent(config.vhost);
+    return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
 }
 
-
 async function createChannels(connection) {
 	try {
 		const channel = await connection.createChannel();

connections/manager.class.ts

@@ -9,7 +9,6 @@ import {StateHistorySocket} from "./state-history";
 import fetch from 'cross-fetch';
 import {exec} from "child_process";
 import {hLog} from "../helpers/common_functions";
-import {readFileSync} from "fs";
 
 export class ConnectionManager {
 
@@ -20,8 +19,8 @@ export class ConnectionManager {
     last_commit_hash: string;
     current_version: string;
 
-    private readonly esIngestClients: Client[];
-    private esIngestClient: Client;
+    esIngestClients: Client[];
+    esIngestClient: Client;
 
     constructor(private cm: ConfigurationModule) {
         this.config = cm.config;

definitions/index-templates.ts

@@ -183,7 +183,74 @@ export const action = {
                     "unstake_net_quantity": {"type": "float"},
                     "amount": {"type": "float"}
                 }
-            }
+            },
+
+            // eosio::undelegatebw
+            // "@exec": {
+            //     "properties": {
+            //         "executer": {"type": "keyword"},
+            //         "coopname": {"type": "keyword"},
+            //         "decision_id": {"type": "long"},
+            //     }
+            // },
+
+            // "@statement": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "username": {"type": "keyword"},
+            //         "action": {"type": "keyword"},
+            //         "decision_id": {"type": "long"},
+            //         "batch_id": {"type": "long"},
+            //         "statement": {"enabled": false},
+            //     }
+            // },
+
+            // "@act": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "username": {"type": "keyword"},
+            //         "action": {"type": "keyword"},
+            //         "decision_id": {"type": "long"},
+            //         "batch_id": {"type": "long"},
+            //         "act": {"enabled": false},
+            //     }
+            // },
+
+            // "@decision": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "username": {"type": "keyword"},
+            //         "action": {"type": "keyword"},
+            //         "decision_id": {"type": "long"},
+            //         "batch_id": {"type": "long"},
+            //         "decision": {"enabled": false},
+            //     }
+            // },
+            // "@batch": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "action": {"type": "keyword"},
+            //         "batch_id": {"type": "long"},
+            //     }
+            // },
+
+
+            // // eosio::votefor
+            // "@votefor": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "member": {"type": "keyword"},
+            //         "decision_id": {"type": "long"}
+            //     }
+            // },
+
+            // "@voteagainst": {
+            //     "properties": {
+            //         "coopname": {"type": "keyword"},
+            //         "member": {"type": "keyword"},
+            //         "decision_id": {"type": "long"}
+            //     }
+            // }
         }
     }
 };

ecosystem.config.js

@@ -20,4 +20,11 @@ readdirSync(chainsRoot)
         }
     });
 
+// apps.push({
+//     name: 'hyperion-governor',
+//     namespace: 'hyperion',
+//     script: 'governor/server/index.js',
+//     watch: false,
+// });
+
 module.exports = {apps};

modules/action_data/act.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'act',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'act',
+
+    mappings: {
+        action: {
+            "@act": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "act": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@act'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            act: data.act
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/batch.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'batch',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'batch',
+        
+    mappings: {
+        action: {
+            "@batch": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "batch_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@batch'] = {
+            coopname: data.coopname,
+            action: data.action,
+            batch_id: data.batch_id,
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/decision.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'decision',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'decision',
+    
+    mappings: {
+        action: {
+            "@decision": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "decision": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+           
+        action['@decision'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            decision: data.decision
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/exec.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'exec',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'exec',
+
+    mappings: {
+        action: {
+            "@exec": {
+                "properties": {
+                    "executer": {"type": "keyword"},
+                    "coopname": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@exec'] = {
+            executer: data.executer,
+            coopname: data.coopname,
+            decision_id: data.decision_id,
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/statement.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'statement',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'statement',
+
+    mappings: {
+        action: {
+            "@statement": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "statement": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+           
+        action['@statement'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            statement: data.statement
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/voteagainst.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'voteagainst',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'voteagainst',
+
+    mappings: {
+        action: {
+            "@voteagainst": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "member": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@voteagainst'] = {
+            coopname: data.coopname,
+            member: data.member,
+            decision_id: data.decision_id
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/action_data/votefor.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'votefor',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'votefor',
+
+    mappings: {
+        action: {
+            "@votefor": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "member": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@votefor'] = {
+            coopname: data.coopname,
+            member: data.member,
+            decision_id: data.decision_id
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

modules/index.js

@@ -53,12 +53,12 @@ class HyperionModuleLoader {
                     const key = `${_module.contract}::${_module.action}`;
                     if (this.chainMappings.has(key)) {
                         if (this.chainMappings.get(key) === '*' && _module.chain === chainID) {
-                            // console.log('Overwriting module ' + key + ' for ' + _module.chain);
+                            console.log('Overwriting module ' + key + ' for ' + _module.chain);
                             this.includeModule(_module);
                             this.chainMappings.set(key, _module.chain);
                         }
                     } else {
-                        // console.log('Including module ' + key + ' for ' + _module.chain);
+                        console.log('Including module ' + key + ' for ' + _module.chain);
                         this.includeModule(_module);
                         this.chainMappings.set(key, _module.chain);
                     }

modules/parsers/base-parser.ts

@@ -192,7 +192,6 @@ export abstract class BaseParser {
         if (ds_act) {
 
             if (ds_act.account && ds_act.name && ds_act.authorization) {
-                console.log(ds_act);
                 action.act.data = ds_act.data;
             }
 

package-lock.json

@@ -24,7 +24,7 @@
         "base-x": "^4.0.0",
         "cli-progress": "^3.12.0",
         "commander": "^8.3.0",
-        "cross-fetch": "^3.1.5",
+        "cross-fetch": "^3.1.8",
         "eosjs": "^22.1.0",
         "fast-json-stringify": "2.7.13",
         "fastify": "3.29.4",
@@ -37,32 +37,32 @@
         "lodash": "^4.17.21",
         "moment": "^2.29.4",
         "nodemailer": "^6.9.0",
-        "pino-pretty": "^9.1.1",
+        "pino-pretty": "^10.2.0",
         "portfinder": "^1.0.32",
-        "socket.io": "4.7.0",
-        "socket.io-client": "4.7.0",
+        "socket.io": "4.7.2",
+        "socket.io-client": "4.7.2",
         "socket.io-redis": "^6.1.1",
         "telegraf": "^4.12.3-canary.1",
         "typescript": "^4.5.2",
-        "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.24.0",
-        "ws": "^8.12.1"
+        "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.31.0",
+        "ws": "^8.14.2"
       },
       "devDependencies": {
-        "@types/amqplib": "^0.10.1",
+        "@types/amqplib": "^0.10.2",
         "@types/async": "^3.2.18",
         "@types/global-agent": "^2.1.1",
         "@types/ioredis": "^4.28.10",
-        "@types/lodash": "^4.14.191",
-        "@types/node": "^20.1.2",
-        "@types/nodemailer": "^6.4.7",
-        "@types/ws": "^8.5.4"
+        "@types/lodash": "^4.14.197",
+        "@types/node": "^20.5.7",
+        "@types/nodemailer": "^6.4.9",
+        "@types/ws": "^8.5.5"
       },
       "engines": {
         "node": ">=18"
       },
       "optionalDependencies": {
         "bufferutil": "^4.0.7",
-        "utf-8-validate": "^5.0.10"
+        "utf-8-validate": "^6.0.3"
       }
     },
     "node_modules/@acuminous/bitsyntax": {
@@ -199,9 +199,9 @@
       }
     },
     "node_modules/@opencensus/core/node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
       "bin": {
         "semver": "bin/semver"
       }
@@ -234,17 +234,17 @@
       }
     },
     "node_modules/@opencensus/propagation-b3/node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
       "bin": {
         "semver": "bin/semver"
       }
     },
     "node_modules/@pm2/io": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/@pm2/io/-/io-5.0.0.tgz",
-      "integrity": "sha512-3rToDVJaRoob5Lq8+7Q2TZFruoEkdORxwzFpZaqF4bmH6Bkd7kAbdPrI/z8X6k1Meq5rTtScM7MmDgppH6aLlw==",
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/@pm2/io/-/io-5.0.2.tgz",
+      "integrity": "sha512-XAvrNoQPKOyO/jJyCu8jPhLzlyp35MEf7w/carHXmWKddPzeNOFSEpSEqMzPDawsvpxbE+i918cNN+MwgVsStA==",
       "dependencies": {
         "@opencensus/core": "0.0.9",
         "@opencensus/propagation-b3": "0.0.8",
@@ -252,7 +252,7 @@
         "debug": "~4.3.1",
         "eventemitter2": "^6.3.1",
         "require-in-the-middle": "^5.0.0",
-        "semver": "6.3.0",
+        "semver": "~7.5.4",
         "shimmer": "^1.2.0",
         "signal-exit": "^3.0.3",
         "tslib": "1.9.3"
@@ -269,14 +269,6 @@
         "lodash": "^4.17.14"
       }
     },
-    "node_modules/@pm2/io/node_modules/semver": {
-      "version": "6.3.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
-      "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==",
-      "bin": {
-        "semver": "bin/semver.js"
-      }
-    },
     "node_modules/@sindresorhus/is": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz",
@@ -305,9 +297,9 @@
       }
     },
     "node_modules/@types/amqplib": {
-      "version": "0.10.1",
-      "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.1.tgz",
-      "integrity": "sha512-j6ANKT79ncUDnAs/+9r9eDujxbeJoTjoVu33gHHcaPfmLQaMhvfbH2GqSe8KUM444epAp1Vl3peVOQfZk3UIqA==",
+      "version": "0.10.2",
+      "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.2.tgz",
+      "integrity": "sha512-K0qC2YR0ZcQWWMOifg4yvCAu5wi/V6wY6MnMS4LSvqx6qwXBAhxno6OBN8D76FIaajLNfgHKOXobZBL/uAwXAQ==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
@@ -372,20 +364,20 @@
       }
     },
     "node_modules/@types/lodash": {
-      "version": "4.14.194",
-      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.194.tgz",
-      "integrity": "sha512-r22s9tAS7imvBt2lyHC9B8AGwWnXaYb1tY09oyLkXDs4vArpYJzw09nj8MLx5VfciBPGIb+ZwG0ssYnEPJxn/g==",
+      "version": "4.14.197",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.197.tgz",
+      "integrity": "sha512-BMVOiWs0uNxHVlHBgzTIqJYmj+PgCo4euloGF+5m4okL3rEYzM2EEv78mw8zWSMM57dM7kVIgJ2QDvwHSoCI5g==",
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "20.1.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.2.tgz",
-      "integrity": "sha512-CTO/wa8x+rZU626cL2BlbCDzydgnFNgc19h4YvizpTO88MFQxab8wqisxaofQJ/9bLGugRdWIuX/TbIs6VVF6g=="
+      "version": "20.5.7",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.7.tgz",
+      "integrity": "sha512-dP7f3LdZIysZnmvP3ANJYTSwg+wLLl8p7RqniVlV7j+oXSXAbt9h0WIBFmJy5inWZoX9wZN6eXx+YXd9Rh3RBA=="
     },
     "node_modules/@types/nodemailer": {
-      "version": "6.4.7",
-      "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.7.tgz",
-      "integrity": "sha512-f5qCBGAn/f0qtRcd4SEn88c8Fp3Swct1731X4ryPKqS61/A3LmmzN8zaEz7hneJvpjFbUUgY7lru/B/7ODTazg==",
+      "version": "6.4.9",
+      "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.9.tgz",
+      "integrity": "sha512-XYG8Gv+sHjaOtUpiuytahMy2mM3rectgroNbs6R3djZEKmPNiIJwe9KqOJBGzKKnNZNKvnuvmugBgpq3w/S0ig==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
@@ -400,9 +392,9 @@
       }
     },
     "node_modules/@types/ws": {
-      "version": "8.5.4",
-      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.4.tgz",
-      "integrity": "sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==",
+      "version": "8.5.5",
+      "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.5.tgz",
+      "integrity": "sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
@@ -517,9 +509,9 @@
       }
     },
     "node_modules/async-listener/node_modules/semver": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz",
+      "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==",
       "bin": {
         "semver": "bin/semver"
       }
@@ -810,11 +802,11 @@
       }
     },
     "node_modules/cross-fetch": {
-      "version": "3.1.5",
-      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
-      "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==",
+      "version": "3.1.8",
+      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz",
+      "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==",
       "dependencies": {
-        "node-fetch": "2.6.7"
+        "node-fetch": "^2.6.12"
       }
     },
     "node_modules/dateformat": {
@@ -990,9 +982,9 @@
       }
     },
     "node_modules/engine.io": {
-      "version": "6.5.0",
-      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.5.0.tgz",
-      "integrity": "sha512-UlfoK1iD62Hkedw2TmuHdhDsZCGaAyp+LZ/AvnImjYBeWagA3qIEETum90d6shMeFZiDuGT66zVCdx1wKYKGGg==",
+      "version": "6.5.2",
+      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.5.2.tgz",
+      "integrity": "sha512-IXsMcGpw/xRfjra46sVZVHiSWo/nJ/3g1337q9KNXtS6YRzbW5yIzTCb9DjhrBe7r3GZQR0I4+nq+4ODk5g/cA==",
       "dependencies": {
         "@types/cookie": "^0.4.1",
         "@types/cors": "^2.8.12",
@@ -1002,25 +994,39 @@
         "cookie": "~0.4.1",
         "cors": "~2.8.5",
         "debug": "~4.3.1",
-        "engine.io-parser": "~5.1.0",
+        "engine.io-parser": "~5.2.1",
         "ws": "~8.11.0"
       },
       "engines": {
-        "node": ">=10.0.0"
+        "node": ">=10.2.0"
       }
     },
     "node_modules/engine.io-client": {
-      "version": "6.5.0",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.0.tgz",
-      "integrity": "sha512-C7eN3OKggSfd5g8IDgUA9guC8TNS6CEganKT7dL6Fp3q+FobcQ/WBn2Qq2XTL1vNTiFZfDzXohvqLuR9dWejdg==",
+      "version": "6.5.2",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.5.2.tgz",
+      "integrity": "sha512-CQZqbrpEYnrpGqC07a9dJDz4gePZUgTPMU3NKJPSeQOyw27Tst4Pl3FemKoFGAlHzgZmKjoRmiJvbWfhCXUlIg==",
       "dependencies": {
         "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.1",
-        "engine.io-parser": "~5.1.0",
+        "engine.io-parser": "~5.2.1",
         "ws": "~8.11.0",
         "xmlhttprequest-ssl": "~2.0.0"
       }
     },
+    "node_modules/engine.io-client/node_modules/utf-8-validate": {
+      "version": "5.0.10",
+      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
+      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
+      "hasInstallScript": true,
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "node-gyp-build": "^4.3.0"
+      },
+      "engines": {
+        "node": ">=6.14.2"
+      }
+    },
     "node_modules/engine.io-client/node_modules/ws": {
       "version": "8.11.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
@@ -1042,9 +1048,9 @@
       }
     },
     "node_modules/engine.io-parser": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.1.0.tgz",
-      "integrity": "sha512-enySgNiK5tyZFynt3z7iqBR+Bto9EVVVvDFuTT0ioHCGbzirZVGDGiQjZzEp8hWl6hd5FSVytJGuScX1C1C35w==",
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.1.tgz",
+      "integrity": "sha512-9JktcM3u18nU9N2Lz3bWeBgxVgOKpw7yhRaoxQA3FUDZzzw+9WlA6p4G4u0RixNkg14fH7EfEc/RhpurtiROTQ==",
       "engines": {
         "node": ">=10.0.0"
       }
@@ -1057,6 +1063,20 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/engine.io/node_modules/utf-8-validate": {
+      "version": "5.0.10",
+      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
+      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
+      "hasInstallScript": true,
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "node-gyp-build": "^4.3.0"
+      },
+      "engines": {
+        "node": ">=6.14.2"
+      }
+    },
     "node_modules/engine.io/node_modules/ws": {
       "version": "8.11.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
@@ -1888,9 +1908,9 @@
       }
     },
     "node_modules/node-fetch": {
-      "version": "2.6.7",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
-      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
       "dependencies": {
         "whatwg-url": "^5.0.0"
       },
@@ -2124,9 +2144,9 @@
       }
     },
     "node_modules/pino-pretty": {
-      "version": "9.4.0",
-      "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-9.4.0.tgz",
-      "integrity": "sha512-NIudkNLxnl7MGj1XkvsqVyRgo6meFP82ECXF2PlOI+9ghmbGuBUUqKJ7IZPIxpJw4vhhSva0IuiDSAuGh6TV9g==",
+      "version": "10.2.0",
+      "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.2.0.tgz",
+      "integrity": "sha512-tRvpyEmGtc2D+Lr3FulIZ+R1baggQ4S3xD2Ar93KixFEDx6SEAUP3W5aYuEw1C73d6ROrNcB2IXLteW8itlwhA==",
       "dependencies": {
         "colorette": "^2.0.7",
         "dateformat": "^4.6.3",
@@ -2490,9 +2510,9 @@
       "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="
     },
     "node_modules/semver": {
-      "version": "7.5.3",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz",
-      "integrity": "sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==",
+      "version": "7.5.4",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
+      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
       "dependencies": {
         "lru-cache": "^6.0.0"
       },
@@ -2584,20 +2604,20 @@
       "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="
     },
     "node_modules/socket.io": {
-      "version": "4.7.0",
-      "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.7.0.tgz",
-      "integrity": "sha512-eOpu7oCNiPGBHn9Falg0cAGivp6TpDI3Yt596fbsf+vln8kRLFWxXKrecFrybn/xNYVn9HcdJNAkYToCmTjsyg==",
+      "version": "4.7.2",
+      "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.7.2.tgz",
+      "integrity": "sha512-bvKVS29/I5fl2FGLNHuXlQaUH/BlzX1IN6S+NKLNZpBsPZIDH+90eQmCs2Railn4YUiww4SzUedJ6+uzwFnKLw==",
       "dependencies": {
         "accepts": "~1.3.4",
         "base64id": "~2.0.0",
         "cors": "~2.8.5",
         "debug": "~4.3.2",
-        "engine.io": "~6.5.0",
+        "engine.io": "~6.5.2",
         "socket.io-adapter": "~2.5.2",
         "socket.io-parser": "~4.2.4"
       },
       "engines": {
-        "node": ">=10.0.0"
+        "node": ">=10.2.0"
       }
     },
     "node_modules/socket.io-adapter": {
@@ -2608,6 +2628,20 @@
         "ws": "~8.11.0"
       }
     },
+    "node_modules/socket.io-adapter/node_modules/utf-8-validate": {
+      "version": "5.0.10",
+      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
+      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
+      "hasInstallScript": true,
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "node-gyp-build": "^4.3.0"
+      },
+      "engines": {
+        "node": ">=6.14.2"
+      }
+    },
     "node_modules/socket.io-adapter/node_modules/ws": {
       "version": "8.11.0",
       "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz",
@@ -2629,13 +2663,13 @@
       }
     },
     "node_modules/socket.io-client": {
-      "version": "4.7.0",
-      "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.7.0.tgz",
-      "integrity": "sha512-7Q8CeDrhuZzg4QLXl3tXlk5yb086oxYzehAVZRLiGCzCmtDneiHz1qHyyWcxhTgxXiokVpWQXoG/u60HoXSQew==",
+      "version": "4.7.2",
+      "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.7.2.tgz",
+      "integrity": "sha512-vtA0uD4ibrYD793SOIAwlo8cj6haOeMHrGvwPxJsxH7CeIksqJ+3Zc06RvWTIFgiSqx4A3sOnTXpfAEE2Zyz6w==",
       "dependencies": {
         "@socket.io/component-emitter": "~3.1.0",
         "debug": "~4.3.2",
-        "engine.io-client": "~6.5.0",
+        "engine.io-client": "~6.5.2",
         "socket.io-parser": "~4.2.4"
       },
       "engines": {
@@ -2810,25 +2844,6 @@
         "node": "^12.20.0 || >=14.13.1"
       }
     },
-    "node_modules/telegraf/node_modules/node-fetch": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz",
-      "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==",
-      "dependencies": {
-        "whatwg-url": "^5.0.0"
-      },
-      "engines": {
-        "node": "4.x || >=6.0.0"
-      },
-      "peerDependencies": {
-        "encoding": "^0.1.0"
-      },
-      "peerDependenciesMeta": {
-        "encoding": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/tiny-lru": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz",
@@ -2906,9 +2921,9 @@
       }
     },
     "node_modules/utf-8-validate": {
-      "version": "5.0.10",
-      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
-      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-6.0.3.tgz",
+      "integrity": "sha512-uIuGf9TWQ/y+0Lp+KGZCMuJWc3N9BHA+l/UmHd/oUHwJJDeysyTRxNQVkbzsIWfGFbRe3OcgML/i0mvVRPOyDA==",
       "hasInstallScript": true,
       "optional": true,
       "dependencies": {
@@ -2964,9 +2979,9 @@
       "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
     },
     "node_modules/ws": {
-      "version": "8.13.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz",
-      "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
+      "version": "8.14.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
+      "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
       "engines": {
         "node": ">=10.0.0"
       },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hyperion-history",
-  "version": "3.3.9-7",
+  "version": "3.3.9-8",
   "description": "Scalable Full History API Solution for EOSIO based blockchains",
   "main": "launcher.js",
   "scripts": {
@@ -40,7 +40,7 @@
     "base-x": "^4.0.0",
     "cli-progress": "^3.12.0",
     "commander": "^8.3.0",
-    "cross-fetch": "^3.1.5",
+    "cross-fetch": "^3.1.8",
     "eosjs": "^22.1.0",
     "fast-json-stringify": "2.7.13",
     "fastify": "3.29.4",
@@ -53,28 +53,28 @@
     "lodash": "^4.17.21",
     "moment": "^2.29.4",
     "nodemailer": "^6.9.0",
-    "pino-pretty": "^9.1.1",
+    "pino-pretty": "^10.2.0",
     "portfinder": "^1.0.32",
-    "socket.io": "4.7.0",
-    "socket.io-client": "4.7.0",
+    "socket.io": "4.7.2",
+    "socket.io-client": "4.7.2",
     "socket.io-redis": "^6.1.1",
     "telegraf": "^4.12.3-canary.1",
     "typescript": "^4.5.2",
-    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.24.0",
-    "ws": "^8.12.1"
+    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.31.0",
+    "ws": "^8.14.2"
   },
   "devDependencies": {
-    "@types/amqplib": "^0.10.1",
+    "@types/amqplib": "^0.10.2",
     "@types/async": "^3.2.18",
     "@types/global-agent": "^2.1.1",
     "@types/ioredis": "^4.28.10",
-    "@types/lodash": "^4.14.191",
-    "@types/node": "^20.1.2",
-    "@types/nodemailer": "^6.4.7",
-    "@types/ws": "^8.5.4"
+    "@types/lodash": "^4.14.197",
+    "@types/node": "^20.5.7",
+    "@types/nodemailer": "^6.4.9",
+    "@types/ws": "^8.5.5"
   },
   "optionalDependencies": {
     "bufferutil": "^4.0.7",
-    "utf-8-validate": "^5.0.10"
+    "utf-8-validate": "^6.0.3"
   }
 }

scripts/hyp-repair.ts

@@ -1,36 +1,47 @@
-import {Command} from 'commander';
-import {readFileSync} from "node:fs";
-import {Client, estypes} from "@elastic/elasticsearch";
+import { Command } from 'commander';
+import { readFileSync } from 'node:fs';
+import { Client, estypes } from '@elastic/elasticsearch';
 // @ts-ignore
-import cliProgress from "cli-progress";
-import {JsonRpc} from 'eosjs';
+import cliProgress from 'cli-progress';
+import { JsonRpc } from 'eosjs';
 import fetch from 'cross-fetch';
-import {existsSync, mkdirSync, writeFileSync} from "fs";
-import {HyperionBlock} from "./repair-cli/interfaces.js";
+import { existsSync, mkdirSync, writeFileSync } from 'fs';
+import { HyperionBlock } from './repair-cli/interfaces.js';
 import {
     getBlocks,
     getFirstIndexedBlock,
     getLastIndexedBlock,
     initESClient,
     readChainConfig,
-    readConnectionConfig
-} from "./repair-cli/functions.js";
-import {SearchResponse} from "@elastic/elasticsearch/api/types";
+    readConnectionConfig,
+} from './repair-cli/functions.js';
+import { SearchResponse } from '@elastic/elasticsearch/api/types';
 
-import {WebSocket} from 'ws';
-import {block} from "../definitions/index-templates";
+import { WebSocket } from 'ws';
+import { block } from '../definitions/index-templates';
 
-const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+const progressBar = new cliProgress.SingleBar(
+    {},
+    cliProgress.Presets.shades_classic
+);
 const program = new Command();
 const errorRanges: any[] = [];
 let pendingBlock: HyperionBlock | null = null;
 
 const missingBlocks: {
-    start: number,
-    end: number
+    start: number;
+    end: number;
 }[] = [];
 
-async function run(client: Client, rpc: JsonRpc, indexName: string, blockInit: number, firstBlock: number, qtdTotal: any, loop: any = 1) {
+async function run(
+    client: Client,
+    rpc: JsonRpc,
+    indexName: string,
+    blockInit: number,
+    firstBlock: number,
+    qtdTotal: any,
+    loop: any = 1
+) {
     let blockInitial = blockInit;
     let blockFinal = blockInitial - qtdTotal;
     const tRef = process.hrtime.bigint();
@@ -40,33 +51,44 @@ async function run(client: Client, rpc: JsonRpc, indexName: string, blockInit: n
             blockFinal = firstBlock;
         }
         try {
-            let {body} = await getBlocks(client, indexName, blockInitial, blockFinal, qtdTotal);
-            let {hits: {hits}} = body;
+            let { body } = await getBlocks(
+                client,
+                indexName,
+                blockInitial,
+                blockFinal,
+                qtdTotal
+            );
+            let {
+                hits: { hits },
+            } = body;
             const blocks = hits.map((obj: any) => obj._source);
             await findForksOnRange(blocks, rpc);
             blockInitial = blockFinal;
             blockFinal = blockInitial - qtdTotal;
             progressBar.update(i);
         } catch (e: any) {
-            console.log("Error: ", e);
+            console.log('Error: ', e);
             process.exit(1);
         }
     }
     progressBar.stop();
-    console.log("=========== Forked Ranges ===========");
+    console.log('=========== Forked Ranges ===========');
     console.table(errorRanges);
-    console.log("=========== Missing Ranges ==========");
+    console.log('=========== Missing Ranges ==========');
     console.table(missingBlocks);
     const tDiff = Number(process.hrtime.bigint() - tRef) / 1000000;
     console.log(`Total time: ${Math.round(tDiff / 1000)}s`);
 }
 
 async function findForksOnRange(blocks: HyperionBlock[], rpc: JsonRpc) {
-
     const removals: Set<string> = new Set();
     let start = null;
     let end = null;
-    if (blocks.length > 0 && pendingBlock && blocks[0].block_num !== pendingBlock.block_num) {
+    if (
+        blocks.length > 0 &&
+        pendingBlock &&
+        blocks[0].block_num !== pendingBlock.block_num
+    ) {
         blocks.unshift(pendingBlock);
     }
 
@@ -83,10 +105,14 @@ async function findForksOnRange(blocks: HyperionBlock[], rpc: JsonRpc) {
         i++;
         // console.log('previous -> ', previousBlock.block_num);
         if (previousBlock.block_num !== currentBlockNumber - 1) {
-            console.log(`\nBlock number mismatch, expected: ${currentBlockNumber - 1} got ${previousBlock.block_num}`);
+            console.log(
+                `\nBlock number mismatch, expected: ${
+                    currentBlockNumber - 1
+                } got ${previousBlock.block_num}`
+            );
             missingBlocks.push({
                 start: previousBlock.block_num + 1,
-                end: currentBlockNumber - 1
+                end: currentBlockNumber - 1,
             });
             continue;
         }
@@ -107,7 +133,7 @@ async function findForksOnRange(blocks: HyperionBlock[], rpc: JsonRpc) {
                         removals.add(currentBlock.block_id);
                     } else {
                         end = currentBlockNumber + 1;
-                        const range = {start, end, ids: [...removals]};
+                        const range = { start, end, ids: [...removals] };
                         errorRanges.push(range);
                         removals.clear();
                         // console.log(`\n ⚠️⚠️ Forked at ${range.start} to ${range.end}`);
@@ -129,7 +155,7 @@ async function scanChain(chain: string, args: any) {
 
     const client = initESClient(config);
 
-    const jsonRpc = new JsonRpc(config.chains[chain].http, {fetch});
+    const jsonRpc = new JsonRpc(config.chains[chain].http, { fetch });
 
     const ping = await client.ping();
 
@@ -165,12 +191,20 @@ async function scanChain(chain: string, args: any) {
 
     const numberOfBatches = Math.ceil(totalBlocks / batchSize);
 
-    console.log("Range:", firstBlock, lastBlock);
-    console.log("Total Blocks:", totalBlocks);
-    console.log("Batch Size:", batchSize);
-    console.log("Number of Batches:", numberOfBatches);
+    console.log('Range:', firstBlock, lastBlock);
+    console.log('Total Blocks:', totalBlocks);
+    console.log('Batch Size:', batchSize);
+    console.log('Number of Batches:', numberOfBatches);
 
-    await run(client, jsonRpc, blockIndex, lastBlock, firstBlock, batchSize, numberOfBatches);
+    await run(
+        client,
+        jsonRpc,
+        blockIndex,
+        lastBlock,
+        firstBlock,
+        batchSize,
+        numberOfBatches
+    );
     console.log(`Finished checking forked blocks!`);
 
     if (errorRanges.length > 0 || missingBlocks.length > 0) {
@@ -180,7 +214,9 @@ async function scanChain(chain: string, args: any) {
     }
 
     if (errorRanges.length > 0) {
-        let path = `.repair/${chain}-${firstBlock + 2}-${lastBlock}-forked-blocks.json`;
+        let path = `.repair/${chain}-${
+            firstBlock + 2
+        }-${lastBlock}-forked-blocks.json`;
         if (args.outFile) {
             path = args.outFile + '-forked-blocks.json';
         }
@@ -188,7 +224,9 @@ async function scanChain(chain: string, args: any) {
     }
 
     if (missingBlocks.length > 0) {
-        let path = `.repair/${chain}-${firstBlock + 2}-${lastBlock}-missing-blocks.json`;
+        let path = `.repair/${chain}-${
+            firstBlock + 2
+        }-${lastBlock}-missing-blocks.json`;
         if (args.outFile) {
             path = args.outFile + '-missing-blocks.json';
         }
@@ -209,7 +247,6 @@ async function repairMissing(chain: string, file: string, args: any) {
 }
 
 async function repairChain(chain: string, file: string, args: any) {
-
     const chainConfig = readChainConfig(chain);
     const config = readConnectionConfig();
     const client = initESClient(config);
@@ -243,7 +280,6 @@ async function repairChain(chain: string, file: string, args: any) {
     let deletePermissions = 0;
 
     for (const range of forkedBlocks) {
-
         // ACTIONS
         const searchActions = {
             index: `${chain}-action-${chainConfig.settings.index_version}-*`,
@@ -252,22 +288,52 @@ async function repairChain(chain: string, file: string, args: any) {
             body: {
                 query: {
                     bool: {
-                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
-                    }
-                }
-            }
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
         };
 
         if (args.dry) {
             const resultActions = await client.search<any>(searchActions);
-            if ((resultActions.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
-                deleteActions += (resultActions.body.hits.total as estypes.SearchTotalHits)?.value;
+            if (
+                (resultActions.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                deleteActions += (
+                    resultActions.body.hits.total as estypes.SearchTotalHits
+                )?.value;
             }
         } else {
-            delete searchActions.size;
-            const deletedActionsResult = await client.deleteByQuery(searchActions);
-            if (deletedActionsResult && deletedActionsResult.body.deleted && deletedActionsResult.body.deleted > 0) {
-                deleteActions += deletedActionsResult.body.deleted;
+            const indexExists = await client.indices.exists({
+                index: searchActions.index,
+            });
+
+            if (indexExists.body) {
+                delete searchActions.size;
+                const deletedActionsResult = await client.deleteByQuery(
+                    searchActions
+                );
+                if (
+                    deletedActionsResult &&
+                    deletedActionsResult.body.deleted &&
+                    deletedActionsResult.body.deleted > 0
+                ) {
+                    deleteActions += deletedActionsResult.body.deleted;
+                }
+            } else {
+                console.log(
+                    `Index ${searchActions.index} doesn't exist. Unable to delete.`
+                );
             }
         }
 
@@ -279,22 +345,51 @@ async function repairChain(chain: string, file: string, args: any) {
             body: {
                 query: {
                     bool: {
-                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
-                    }
-                }
-            }
-        }
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
 
         if (args.dry) {
             const resultDeltas = await client.search<any>(searchDeltas);
-            if ((resultDeltas.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
-                deleteDeltas += (resultDeltas.body.hits.total as estypes.SearchTotalHits)?.value;
+            if (
+                (resultDeltas.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                deleteDeltas += (
+                    resultDeltas.body.hits.total as estypes.SearchTotalHits
+                )?.value;
             }
         } else {
-            delete searchDeltas.size;
-            const deletedDeltasResult = await client.deleteByQuery(searchDeltas);
-            if (deletedDeltasResult && deletedDeltasResult.body.deleted && deletedDeltasResult.body.deleted > 0) {
-                deleteDeltas += deletedDeltasResult.body.deleted;
+            const indexExists = await client.indices.exists({
+                index: searchDeltas.index,
+            });
+            if (indexExists.body) {
+                delete searchDeltas.size;
+                const deletedDeltasResult = await client.deleteByQuery(
+                    searchDeltas
+                );
+                if (
+                    deletedDeltasResult &&
+                    deletedDeltasResult.body.deleted &&
+                    deletedDeltasResult.body.deleted > 0
+                ) {
+                    deleteDeltas += deletedDeltasResult.body.deleted;
+                }
+            } else {
+                console.log(
+                    `Index ${searchDeltas.index} doesn't exist. Unable to delete.`
+                );
             }
         }
 
@@ -306,23 +401,49 @@ async function repairChain(chain: string, file: string, args: any) {
             body: {
                 query: {
                     bool: {
-                        must: [{range: {block: {lte: range.start, gte: range.end}}}]
-                    }
-                }
-            }
-        }
+                        must: [
+                            {
+                                range: {
+                                    block: { lte: range.start, gte: range.end },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
 
         if (args.dry) {
             const resultAbis = await client.search<any>(searchAbis);
-            if ((resultAbis.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
-                deleteAbis += (resultAbis.body.hits.total as estypes.SearchTotalHits)?.value;
-                console.log('ABIs', {lte: range.start, gte: range.end});
+            if (
+                (resultAbis.body.hits.total as estypes.SearchTotalHits)?.value >
+                0
+            ) {
+                deleteAbis += (
+                    resultAbis.body.hits.total as estypes.SearchTotalHits
+                )?.value;
+                console.log('ABIs', { lte: range.start, gte: range.end });
             }
         } else {
-            delete searchAbis.size;
-            const deletedAbisResult = await client.deleteByQuery(searchAbis);
-            if (deletedAbisResult && deletedAbisResult.body.deleted && deletedAbisResult.body.deleted > 0) {
-                deleteAbis += deletedAbisResult.body.deleted;
+            const indexExists = await client.indices.exists({
+                index: searchAbis.index,
+            });
+            if (indexExists.body) {
+                delete searchAbis.size;
+                const deletedAbisResult = await client.deleteByQuery(
+                    searchAbis
+                );
+                if (
+                    deletedAbisResult &&
+                    deletedAbisResult.body.deleted &&
+                    deletedAbisResult.body.deleted > 0
+                ) {
+                    deleteAbis += deletedAbisResult.body.deleted;
+                }
+            } else {
+                console.log(
+                    `Index ${searchAbis.index} doesn't exist. Unable to delete.`
+                );
             }
         }
 
@@ -334,25 +455,61 @@ async function repairChain(chain: string, file: string, args: any) {
             body: {
                 query: {
                     bool: {
-                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
-                    }
-                }
-            }
-        }
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
 
         if (args.dry) {
             const resultAccounts = await client.search<any>(searchAccounts);
-            if ((resultAccounts.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
-                console.log('[WARNING]', (resultAccounts.body.hits.total as estypes.SearchTotalHits)?.value, 'accounts needs to be updated');
-                deleteAccounts += (resultAccounts.body.hits.total as estypes.SearchTotalHits)?.value;
+            if (
+                (resultAccounts.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (resultAccounts.body.hits.total as estypes.SearchTotalHits)
+                        ?.value,
+                    'accounts needs to be updated'
+                );
+                deleteAccounts += (
+                    resultAccounts.body.hits.total as estypes.SearchTotalHits
+                )?.value;
             }
         } else {
+
+            const indexExists = await client.indices.exists({
+                index: searchAccounts.index,
+            });
+
+            if (indexExists.body) {
             delete searchAccounts.size;
-            const deletedAccountsResult = await client.deleteByQuery(searchAccounts);
-            if (deletedAccountsResult && deletedAccountsResult.body.deleted && deletedAccountsResult.body.deleted > 0) {
+            const deletedAccountsResult = await client.deleteByQuery(
+                searchAccounts
+            );
+            if (
+                deletedAccountsResult &&
+                deletedAccountsResult.body.deleted &&
+                deletedAccountsResult.body.deleted > 0
+            ) {
                 deleteAccounts += deletedAccountsResult.body.deleted;
             }
+        }else {
+            console.log(
+                `Index ${searchAccounts.index} doesn't exist. Unable to delete.`
+            );
         }
+    }
 
         // VOTERS
         const searchVoters = {
@@ -362,25 +519,61 @@ async function repairChain(chain: string, file: string, args: any) {
             body: {
                 query: {
                     bool: {
-                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
-                    }
-                }
-            }
-        }
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
 
         if (args.dry) {
             const resultVoters = await client.search<any>(searchVoters);
-            if ((resultVoters.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
-                console.log('[WARNING]', (resultVoters.body.hits.total as estypes.SearchTotalHits)?.value, 'voters needs to be updated');
-                deleteVoters += (resultVoters.body.hits.total as estypes.SearchTotalHits)?.value;
+            if (
+                (resultVoters.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (resultVoters.body.hits.total as estypes.SearchTotalHits)
+                        ?.value,
+                    'voters needs to be updated'
+                );
+                deleteVoters += (
+                    resultVoters.body.hits.total as estypes.SearchTotalHits
+                )?.value;
             }
         } else {
+            const indexExists = await client.indices.exists({
+                index: searchVoters.index,
+            });
+
+            if (indexExists.body) {
+
             delete searchVoters.size;
-            const deletedVotersResult = await client.deleteByQuery(searchVoters);
-            if (deletedVotersResult && deletedVotersResult.body.deleted && deletedVotersResult.body.deleted > 0) {
+            const deletedVotersResult = await client.deleteByQuery(
+                searchVoters
+            );
+            if (
+                deletedVotersResult &&
+                deletedVotersResult.body.deleted &&
+                deletedVotersResult.body.deleted > 0
+            ) {
                 deleteVoters += deletedVotersResult.body.deleted;
             }
+        }else {
+            console.log(
+                `Index ${searchVoters.index} doesn't exist. Unable to delete.`
+            );
         }
+    }
 
         // PROPOSALS
         const searchProposals = {
@@ -390,25 +583,56 @@ async function repairChain(chain: string, file: string, args: any) {
             body: {
                 query: {
                     bool: {
-                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
-                    }
-                }
-            }
-        }
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
 
         if (args.dry) {
             const resultProposals = await client.search<any>(searchProposals);
-            if ((resultProposals.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
-                console.log('[WARNING]', (resultProposals.body.hits.total as estypes.SearchTotalHits)?.value, 'proposals needs to be updated');
-                deleteProposals += (resultProposals.body.hits.total as estypes.SearchTotalHits)?.value;
+            if (
+                (resultProposals.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (resultProposals.body.hits.total as estypes.SearchTotalHits)
+                        ?.value,
+                    'proposals needs to be updated'
+                );
+                deleteProposals += (
+                    resultProposals.body.hits.total as estypes.SearchTotalHits
+                )?.value;
             }
         } else {
+
+
+            const indexExists = await client.indices.exists({index: searchProposals.index});
+
+            if (indexExists.body) {
             delete searchProposals.size;
-            const deletedProposalsResult = await client.deleteByQuery(searchProposals);
-            if (deletedProposalsResult && deletedProposalsResult.body.deleted && deletedProposalsResult.body.deleted > 0) {
+            const deletedProposalsResult = await client.deleteByQuery(
+                searchProposals
+            );
+            if (
+                deletedProposalsResult &&
+                deletedProposalsResult.body.deleted &&
+                deletedProposalsResult.body.deleted > 0
+            ) {
                 deleteProposals += deletedProposalsResult.body.deleted;
             }
-        }
+        }else { console.log(`Index ${searchProposals.index} doesn't exist. Unable to delete.`);}
+    }
 
         // LINKS
         const searchLinks = {
@@ -418,25 +642,52 @@ async function repairChain(chain: string, file: string, args: any) {
             body: {
                 query: {
                     bool: {
-                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
-                    }
-                }
-            }
-        }
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
 
         if (args.dry) {
             const resultLinks = await client.search<any>(searchLinks);
-            if ((resultLinks.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
-                console.log('[WARNING]', (resultLinks.body.hits.total as estypes.SearchTotalHits)?.value, 'links needs to be updated');
-                deleteLinks += (resultLinks.body.hits.total as estypes.SearchTotalHits)?.value;
+            if (
+                (resultLinks.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (resultLinks.body.hits.total as estypes.SearchTotalHits)
+                        ?.value,
+                    'links needs to be updated'
+                );
+                deleteLinks += (
+                    resultLinks.body.hits.total as estypes.SearchTotalHits
+                )?.value;
             }
         } else {
+            const indexExists = await client.indices.exists({index: searchLinks.index});
+
+            if (indexExists.body) {
             delete searchLinks.size;
             const deletedLinksResult = await client.deleteByQuery(searchLinks);
-            if (deletedLinksResult && deletedLinksResult.body.deleted && deletedLinksResult.body.deleted > 0) {
+            if (
+                deletedLinksResult &&
+                deletedLinksResult.body.deleted &&
+                deletedLinksResult.body.deleted > 0
+            ) {
                 deleteLinks += deletedLinksResult.body.deleted;
             }
-        }
+        }else { console.log(`Index ${searchLinks.index} doesn't exist. Unable to delete.`);}
+    }
 
         // PERMISSIONS
         const searchPermissions = {
@@ -446,38 +697,78 @@ async function repairChain(chain: string, file: string, args: any) {
             body: {
                 query: {
                     bool: {
-                        must: [{range: {block_num: {lte: range.start, gte: range.end}}}]
-                    }
-                }
-            }
-        }
+                        must: [
+                            {
+                                range: {
+                                    block_num: {
+                                        lte: range.start,
+                                        gte: range.end,
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            },
+        };
 
         if (args.dry) {
-            const resultPermissions = await client.search<any>(searchPermissions);
-            if ((resultPermissions.body.hits.total as estypes.SearchTotalHits)?.value > 0) {
-                console.log('[WARNING]', (resultPermissions.body.hits.total as estypes.SearchTotalHits)?.value, 'permissions needs to be updated');
-                console.log({lte: range.start, gte: range.end});
-                deletePermissions += (resultPermissions.body.hits.total as estypes.SearchTotalHits)?.value;
+            const resultPermissions = await client.search<any>(
+                searchPermissions
+            );
+            if (
+                (resultPermissions.body.hits.total as estypes.SearchTotalHits)
+                    ?.value > 0
+            ) {
+                console.log(
+                    '[WARNING]',
+                    (
+                        resultPermissions.body.hits
+                            .total as estypes.SearchTotalHits
+                    )?.value,
+                    'permissions needs to be updated'
+                );
+                console.log({ lte: range.start, gte: range.end });
+                deletePermissions += (
+                    resultPermissions.body.hits.total as estypes.SearchTotalHits
+                )?.value;
             }
         } else {
+
+            const indexExists = await client.indices.exists({index: searchPermissions.index});
+            if (indexExists.body) {
             delete searchPermissions.size;
-            const deletedPermissionsResult = await client.deleteByQuery(searchPermissions);
-            if (deletedPermissionsResult && deletedPermissionsResult.body.deleted && deletedPermissionsResult.body.deleted > 0) {
+            const deletedPermissionsResult = await client.deleteByQuery(
+                searchPermissions
+            );
+            if (
+                deletedPermissionsResult &&
+                deletedPermissionsResult.body.deleted &&
+                deletedPermissionsResult.body.deleted > 0
+            ) {
                 deletePermissions += deletedPermissionsResult.body.deleted;
             }
-        }
+        }else { console.log(`Index ${searchPermissions.index} doens't exist. Não foi possível realizar a exclusão.`);}
+    }
 
         for (const id of range.ids) {
             const searchBlocks = {
                 index: blockIndex,
                 body: {
-                    query: {bool: {must: [{term: {block_id: {value: id}}}]}}
-                }
+                    query: {
+                        bool: { must: [{ term: { block_id: { value: id } } }] },
+                    },
+                },
             };
             if (args.dry) {
                 console.log(`Deleting block ${id}...`);
-                const resultBlocks = await client.search<SearchResponse<HyperionBlock>>(searchBlocks);
-                if (resultBlocks.body.hits.hits.length > 0 && resultBlocks.body.hits.hits[0]._source) {
+                const resultBlocks = await client.search<
+                    SearchResponse<HyperionBlock>
+                >(searchBlocks);
+                if (
+                    resultBlocks.body.hits.hits.length > 0 &&
+                    resultBlocks.body.hits.hits[0]._source
+                ) {
                     deleteBlocks++;
                 }
             } else {
@@ -490,7 +781,9 @@ async function repairChain(chain: string, file: string, args: any) {
     }
 
     if (args.dry) {
-        console.log(`DRY-RUN: Would have deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`);
+        console.log(
+            `DRY-RUN: Would have deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`
+        );
         console.dir({
             blocks: deleteBlocks,
             actions: deleteActions,
@@ -500,47 +793,83 @@ async function repairChain(chain: string, file: string, args: any) {
             voters: deleteVoters,
             proposals: deleteProposals,
             links: deleteLinks,
-            permissions: deletePermissions
+            permissions: deletePermissions,
         });
     } else {
-        console.log(`Deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`);
+        console.log(
+            `Deleted ${deleteBlocks} blocks, ${deleteActions} actions, ${deleteDeltas} deltas and ${deleteAbis} ABIs`
+        );
     }
 
     await fillMissingBlocksFromFile(args.host, chain, file, args.dry);
 }
 
-async function fillMissingBlocksFromFile(host: string, chain: string, file: string, dryRun: string) {
+async function fillMissingBlocksFromFile(host, chain, file, dryRun) {
     const config = readConnectionConfig();
     const controlPort = config.chains[chain].control_port;
     let hyperionIndexer = `ws://localhost:${controlPort}`;
     if (host) {
-        hyperionIndexer = 'ws://' + host + ':' + controlPort;
+        hyperionIndexer = `ws://${host}:${controlPort}`;
     }
     const controller = new WebSocket(hyperionIndexer + '/local');
+
+    // Function to send Chunk
+    async function sendChunk(chunk): Promise<void> {
+        return new Promise((resolve, reject) => {
+            const payload = {
+                event: 'fill_missing_blocks',
+                data: chunk,
+            };
+            controller.send(JSON.stringify(payload));
+
+            // Wait repair_completed confirmation
+            controller.once('message', (data) => {
+                const parsed = JSON.parse(data.toString());
+                if (parsed.event === 'repair_completed') {
+                    // console.log(`Hyperion repair completed for chunk!`);
+                    resolve();
+                }
+            });
+        });
+    }
+
     controller.on('open', async () => {
         console.log('Connected to Hyperion Controller');
         const parsedFile = JSON.parse(readFileSync(file).toString());
+        const chunkSize = 20; // Chunk size
+        const totalLines = parsedFile.length;
+
         if (!dryRun) {
-            const payload = {
-                "event": "fill_missing_blocks",
-                "data": parsedFile
-            };
-            controller.send(JSON.stringify(payload));
+            let completedLines = 0;
+
+            for (let i = 0; i < parsedFile.length; i += chunkSize) {
+                const chunk = parsedFile.slice(i, i + chunkSize);
+                await sendChunk(chunk);
+
+                // Atualizar o progresso com base no número total de linhas
+                completedLines += chunk.length;
+                const progress = (completedLines / totalLines) * 100;
+                const progressBar = Array(Math.round(progress / 2))
+                    .fill('#')
+                    .join('');
+                process.stdout.clearLine(0); // Limpar a linha anterior
+                process.stdout.cursorTo(0); // Mover o cursor para o início da linha
+                process.stdout.write(
+                    `Progress: [${progressBar}] ${progress.toFixed(2)}%`
+                );
+            }
+            console.log(); // Pule para a próxima linha após a conclusão
+            controller.close();
         } else {
             console.log('Dry run, skipping repair');
             controller.close();
         }
     });
-    controller.on('message', (data) => {
-        const parsed = JSON.parse(data.toString());
-        if (parsed.event === 'repair_completed') {
-            console.log(`Hyperion repair completed!`);
-            controller.close();
-        }
-    });
+
     controller.on('close', () => {
         console.log('Disconnected from Hyperion Controller');
     });
+
     controller.on('error', (err) => {
         console.log(err);
     });
@@ -559,8 +888,8 @@ program
     .description('CLI to find and repair forked and missing blocks on Hyperion')
     .version('0.2.2');
 
-
-program.command('scan <chain>')
+program
+    .command('scan <chain>')
     .description('scan for forked blocks')
     .option('-d, --dry', 'dry-run, do not delete or repair blocks')
     .option('-o, --out-file <file>', 'forked-blocks.json output file')
@@ -569,24 +898,28 @@ program.command('scan <chain>')
     .option('-b, --batch <number>', 'batch size to process')
     .action(scanChain);
 
-program.command('repair <chain> <file>')
+program
+    .command('repair <chain> <file>')
     .description('repair forked blocks')
     .option('-h, --host <host>', 'Hyperion local control api')
     .option('-d, --dry', 'dry-run, do not delete or repair blocks')
     .option('-t, --check-tasks', 'check for running tasks')
     .action(repairChain);
 
-program.command('fill-missing <chain> <file>')
+program
+    .command('fill-missing <chain> <file>')
     .description('write missing blocks')
     .option('-h, --host <host>', 'Hyperion local control api')
     .option('-d, --dry', 'dry-run, do not delete or repair blocks')
     .action(repairMissing);
 
-program.command('view <file>')
+program
+    .command('view <file>')
     .description('view forked blocks')
     .action(viewFile);
 
-program.command('connect')
+program
+    .command('connect')
     .option('-h, --host <host>', 'Hyperion local control api')
     .action(async (args) => {
         let hyperionIndexer = 'ws://localhost:4321';
@@ -602,12 +935,16 @@ program.command('connect')
             });
             controller.on('close', () => {
                 if (valid) {
-                    console.log(`✅  Hyperion Indexer Online - ${hyperionIndexer}`);
+                    console.log(
+                        `✅  Hyperion Indexer Online - ${hyperionIndexer}`
+                    );
                 }
             });
             controller.on('error', (err) => {
-                console.log("Error:", err.message);
-                console.log(`Failed to connect on Hyperion Indexer at ${hyperionIndexer}, please use "--host ws://ADDRESS:PORT" to specify a remote indexer connection`);
+                console.log('Error:', err.message);
+                console.log(
+                    `Failed to connect on Hyperion Indexer at ${hyperionIndexer}, please use "--host ws://ADDRESS:PORT" to specify a remote indexer connection`
+                );
             });
         } catch (e) {
             console.log(e);
@@ -615,8 +952,3 @@ program.command('connect')
     });
 
 program.parse();
-
-
-
-
-

temp/act.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'act',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'act',
+
+    mappings: {
+        action: {
+            "@act": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "act": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@act'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            act: data.act
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/batch.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'batch',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'batch',
+        
+    mappings: {
+        action: {
+            "@batch": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "batch_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@batch'] = {
+            coopname: data.coopname,
+            action: data.action,
+            batch_id: data.batch_id,
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/decision.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'decision',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'decision',
+    
+    mappings: {
+        action: {
+            "@decision": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "decision": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+           
+        action['@decision'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            decision: data.decision
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/exec.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'exec',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'exec',
+
+    mappings: {
+        action: {
+            "@exec": {
+                "properties": {
+                    "executer": {"type": "keyword"},
+                    "coopname": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@exec'] = {
+            executer: data.executer,
+            coopname: data.coopname,
+            decision_id: data.decision_id,
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/statement.js

@@ -0,0 +1,39 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'statement',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'statement',
+
+    mappings: {
+        action: {
+            "@statement": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "username": {"type": "keyword"},
+                    "action": {"type": "keyword"},
+                    "decision_id": {"type": "long"},
+                    "batch_id": {"type": "long"},
+                    "statement": {"enabled": false}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+           
+        action['@statement'] = {
+            coopname: data.coopname,
+            username: data.username,
+            action: data.action,
+            decision_id: data.decision_id,
+            batch_id: data.batch_id,
+            statement: data.statement
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/voteagainst.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'voteagainst',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'voteagainst',
+
+    mappings: {
+        action: {
+            "@voteagainst": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "member": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@voteagainst'] = {
+            coopname: data.coopname,
+            member: data.member,
+            decision_id: decision_id
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

temp/votefor.js

@@ -0,0 +1,33 @@
+const hyperionModule = {
+    chain: "*",
+    contract: '*',
+    action: 'votefor',
+    parser_version: ['3.2', '2.1','1.8','1.7'],
+    defineQueryPrefix: 'votefor',
+
+    mappings: {
+        action: {
+            "@votefor": {
+                "properties": {
+                    "coopname": {"type": "keyword"},
+                    "member": {"type": "keyword"},
+                    "decision_id": {"type": "long"}
+                }
+            }
+        }
+    },
+
+    handler: (action) => {
+        const data = action['act']['data'];
+        
+        action['@votefor'] = {
+            coopname: data.coopname,
+            member: data.member,
+            decision_id: decision_id
+        };
+
+        
+    }
+};
+
+module.exports = {hyperionModule};

workers/deserializer.ts

@@ -8,6 +8,7 @@ import {Type} from "../addons/eosjs-native/eosjs-serialize";
 import {debugLog, hLog} from "../helpers/common_functions";
 import {createHash} from "crypto";
 import flatstr from 'flatstr';
+import {Options} from "amqplib";
 
 const FJS = require('fast-json-stringify');
 
@@ -124,6 +125,9 @@ export default class MainDSWorker extends HyperionWorker {
 
     allowedDynamicContracts: Set<string> = new Set<string>();
 
+    backpressureQueue: any[] = [];
+    waitToSend = false;
+
     constructor() {
 
         super();
@@ -290,6 +294,16 @@ export default class MainDSWorker extends HyperionWorker {
             this.ch.consume(process.env['worker_queue'], (data) => {
                 this.consumerQueue.push(data).catch(console.log);
             });
+            this.ch.on('drain', args => {
+                this.waitToSend = false;
+                while (this.backpressureQueue.length > 0) {
+                    const msg = this.backpressureQueue.shift();
+                    const status = this.controlledSendToQueue(msg.queue, msg.payload, msg.options);
+                    if (!status) {
+                        break;
+                    }
+                }
+            });
         }
     }
 
@@ -478,6 +492,7 @@ export default class MainDSWorker extends HyperionWorker {
                                 hLog(`${block_num} was filtered with ${inline_count} actions!`);
                             }
                             try {
+                                trace[1].signatures = signatures;
                                 this.routeToPool(trace[1], {
                                     block_num,
                                     block_id,
@@ -485,8 +500,7 @@ export default class MainDSWorker extends HyperionWorker {
                                     ts,
                                     inline_count,
                                     filtered,
-                                    live: process.env['live_mode'],
-                                    signatures
+                                    live: process.env['live_mode']
                                 });
                             } catch (e) {
                                 hLog(e);
@@ -622,15 +636,33 @@ export default class MainDSWorker extends HyperionWorker {
         }
 
         const pool_queue = `${this.chain}:ds_pool:${selected_q}`;
-        if (this.ch_ready) {
-            // console.log('selected_q', pool_queue);
-            this.ch.sendToQueue(pool_queue, bufferFromJson(trace, true), {headers});
-            return true;
+        const payload = bufferFromJson(trace, true);
+
+        if (!this.waitToSend) {
+            if (this.ch_ready) {
+                this.controlledSendToQueue(pool_queue, payload, {headers});
+                return true;
+            } else {
+                return false;
+            }
         } else {
+            this.backpressureQueue.push({
+                queue: pool_queue,
+                payload: payload,
+                options: {headers}
+            });
             return false;
         }
     }
 
+    controlledSendToQueue(pool_queue: string, payload: Buffer, options: Options.Publish): boolean {
+        const enqueueResult = this.ch.sendToQueue(pool_queue, payload, options);
+        if (!enqueueResult) {
+            this.waitToSend = true;
+        }
+        return enqueueResult;
+    }
+
     createSerialBuffer(inputArray) {
         return new Serialize.SerialBuffer({textEncoder: this.txEnc, textDecoder: this.txDec, array: inputArray});
     }
@@ -1160,7 +1192,7 @@ export default class MainDSWorker extends HyperionWorker {
                 let jsonRow = await this.processContractRowNative(payload, block_num);
 
                 if (jsonRow?.value && !jsonRow['_blacklisted']) {
-                    console.log(jsonRow);
+                    debugLog(jsonRow);
                     debugLog('Delta DS failed ->>', jsonRow);
                     jsonRow = await this.processContractRowNative(payload, block_num - 1);
                     debugLog('Retry with previous ABI ->>', jsonRow);
@@ -1495,7 +1527,7 @@ export default class MainDSWorker extends HyperionWorker {
                             } catch (e) {
                                 hLog(`Delta struct [${key}] processing error: ${e.message}`);
                                 hLog(e);
-                                console.log(data[1]);
+                                hLog(data[1]);
                             }
                         }
                     }

workers/ds-pool.ts

@@ -454,8 +454,8 @@ export default class DSPoolWorker extends HyperionWorker {
     }
 
     async processTraces(transaction_trace, extra) {
-        const {cpu_usage_us, net_usage_words} = transaction_trace;
-        const {block_num, block_id, producer, ts, inline_count, filtered, live, signatures} = extra;
+        const {cpu_usage_us, net_usage_words, signatures} = transaction_trace;
+        const {block_num, block_id, producer, ts, inline_count, filtered, live} = extra;
 
         if (transaction_trace.status === 0) {
             let action_count = 0;