fix tests

docs/10_utilities/eosio-blocklog.md

@@ -1,38 +0,0 @@
----
-content_title: eosio-blocklog
-link_text: eosio-blocklog
----
-
-`eosio-blocklog` is a command-line interface (CLI) utility that allows node operators to perform low-level tasks on the block logs created by a `nodeos` instance. `eosio-blocklog` can perform one of the following operations:
-
-* Convert a range of blocks to JSON format, as single objects or array.
-* Generate `blocks.index` from `blocks.log` in blocks directory.
-* Trim `blocks.log` and `blocks.index` between a range of blocks.
-* Perform consistency test between `blocks.log` and `blocks.index`.
-* Output the results of the operation to a file or `stdout` (default).
-
-## Usage
-```sh
-eosio-blocklog <options> ...
-```
-
-## Options
-
-Option (=default) | Description
--|-
-`--blocks-dir arg (="blocks")` | The location of the blocks directory (absolute path or relative to the current directory)
-`-o [ --output-file ] arg` | The file to write the generated output to (absolute or relative path). If not specified then output is to `stdout`
-`-f [ --first ] arg (=0)` | The first block number to log or the first block to keep if `trim-blocklog` specified
-`-l [ --last ] arg (=4294967295)` | the last block number to log or the last block to keep if `trim-blocklog` specified
-`--no-pretty-print` | Do not pretty print the output. Useful if piping to `jq` to improve performance
-`--as-json-array` | Print out JSON blocks wrapped in JSON array (otherwise the output is free-standing JSON objects)
-`--make-index` | Create `blocks.index` from `blocks.log`. Must give `blocks-dir` location. Give `output-file` relative to current directory or absolute path (default is `<blocks-dir>/blocks.index`)
-`--trim-blocklog` | Trim `blocks.log` and `blocks.index`. Must give `blocks-dir` and `first` and/or `last` options.
-`--smoke-test` | Quick test that `blocks.log` and `blocks.index` are well formed and agree with each other
-`-h [ --help ]` | Print this help message and exit
-
-## Remarks
-
-When `eosio-blocklog` is launched, the utility attempts to perform the specified operation, then yields the following possible outcomes:
-* If successful, the selected operation is performed and the utility terminates with a zero error code (no error).
-* If unsuccessful, the utility outputs an error to `stderr` and terminates with a non-zero error code (indicating an error).

docs/10_utilities/index.md

@@ -5,5 +5,4 @@ link_text: Antelope Utilities
 
 This section contains documentation for additional utilities that complement or extend `nodeos` and potentially other Antelope software:
 
-* [eosio-blocklog](eosio-blocklog.md) - Low-level utility for node operators to interact with block log files.
 * [trace_api_util](trace_api_util.md) - Low-level utility for performing tasks associated with the [Trace API](../01_nodeos/03_plugins/trace_api_plugin/index.md).

libraries/chain/block_log.cpp

@@ -1450,18 +1450,6 @@ namespace eosio { namespace chain {
       adjust_block_positions(index, new_block_file, block_log_preamble::nbytes_with_chain_id, -nbytes_to_trim);
    }
 
-   // static
-   bool block_log::extract_block_range(const fc::path& block_dir, const fc::path& output_dir, block_num_type& start,
-                                       block_num_type& end, bool rename_input) {
-      block_log_bundle bundle(block_dir);
-      fc::path         output_block_name = rename_input ? output_dir / "old.log" : output_dir / "blocks.log";
-      fc::path         output_index_name = rename_input ? output_dir / "old.index" : output_dir / "blocks.index";
-      if (!fc::exists(output_dir))
-         fc::create_directories(output_dir);
-      extract_blocklog_i(bundle, output_block_name, output_index_name, start, end - start + 1);
-      return true;
-   }
-
    // static
    bool block_log::trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir,
                                        uint32_t truncate_at_block) {
@@ -1517,6 +1505,8 @@ namespace eosio { namespace chain {
          dlog("There are no blocks after block ${n} so do nothing", ("n", n));
          return 2;
       }
+      if (n == log_bundle.log_data.last_block_num())
+         return 0;
 
       const auto to_trim_block_index    = n + 1 - log_bundle.log_data.first_block_num();
       const auto to_trim_block_position = log_bundle.log_index.nth_block_position(to_trim_block_index);
@@ -1557,22 +1547,21 @@ namespace eosio { namespace chain {
    }
 
    // static
-   void block_log::extract_blocklog(const fc::path& log_filename, const fc::path& index_filename,
-                                    const fc::path& dest_dir, uint32_t start_block_num, uint32_t num_blocks) {
+   void block_log::extract_block_range(const fc::path& block_dir, const fc::path& dest_dir,
+                                       block_num_type start_block_num, block_num_type end_block_num) {
 
-      block_log_bundle log_bundle(log_filename, index_filename);
+
+      block_log_bundle log_bundle(block_dir);
 
       EOS_ASSERT(start_block_num >= log_bundle.log_data.first_block_num(), block_log_exception,
                  "The first available block is block ${first_block}.",
                  ("first_block", log_bundle.log_data.first_block_num()));
 
-      EOS_ASSERT(start_block_num + num_blocks - 1 <= log_bundle.log_data.last_block_num(), block_log_exception,
-                 "The last available block is block ${last_block}.",
-                 ("last_block", log_bundle.log_data.last_block_num()));
-
       if (!fc::exists(dest_dir))
          fc::create_directories(dest_dir);
 
+      uint32_t num_blocks = end_block_num - start_block_num + 1;
+
       auto [new_block_filename, new_index_filename] = blocklog_files(dest_dir, start_block_num, num_blocks);
 
       extract_blocklog_i(log_bundle, new_block_filename, new_index_filename, start_block_num, num_blocks);

libraries/chain/include/eosio/chain/block_log.hpp

@@ -95,7 +95,7 @@ namespace eosio { namespace chain {
 
          static bool is_pruned_log(const fc::path& data_dir);
 
-         static bool extract_block_range(const fc::path& block_dir, const fc::path&output_dir, block_num_type& start, block_num_type& end, bool rename_input=false);
+         static void extract_block_range(const fc::path& block_dir, const fc::path&output_dir, block_num_type start, block_num_type end);
 
          static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block);
          static int  trim_blocklog_end(const fc::path& block_dir, uint32_t n);
@@ -110,8 +110,6 @@ namespace eosio { namespace chain {
           */
          static void smoke_test(const fc::path& block_dir, uint32_t n);
 
-         static void extract_blocklog(const fc::path& log_filename, const fc::path& index_filename,
-                                      const fc::path& dest_dir, uint32_t start_block, uint32_t num_blocks);
          static void split_blocklog(const fc::path& block_dir, const fc::path& dest_dir, uint32_t stride);
          static void merge_blocklogs(const fc::path& block_dir, const fc::path& dest_dir);
    private:

programs/leap-util/actions/blocklog.cpp

@@ -38,7 +38,7 @@ struct report_time {
 
    void report() {
       const auto duration = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - _start).count() / 1000;
-      ilog("eosio-blocklog - ${desc} took ${t} msec", ("desc", _desc)("t", duration));
+      ilog("leap-util - ${desc} took ${t} msec", ("desc", _desc)("t", duration));
    }
 
    const std::chrono::high_resolution_clock::time_point _start;
@@ -164,8 +164,7 @@ int blocklog_actions::trim_blocklog() {
 }
 
 int blocklog_actions::extract_blocks() {
-   if(!extract_block_range(opt->blocks_dir, opt->output_dir, opt->first_block, opt->last_block))
-      return -1;
+   extract_block_range(opt->blocks_dir, opt->output_dir, opt->first_block, opt->last_block);
    return 0;
 }
 
@@ -215,18 +214,16 @@ int blocklog_actions::trim_blocklog_end(bfs::path block_dir, uint32_t n) {//n is
 
 bool blocklog_actions::trim_blocklog_front(bfs::path block_dir, uint32_t n) {//n is first block to keep (remove prior blocks)
    report_time rt("trimming blocklog start");
-   block_num_type end = std::numeric_limits<block_num_type>::max();
-   const bool status = block_log::extract_block_range(block_dir, block_dir / "old", n, end, true);
+   const bool status = block_log::trim_blocklog_front(block_dir, block_dir / "old", n);
    rt.report();
    return status;
 }
 
-bool blocklog_actions::extract_block_range(bfs::path block_dir, bfs::path output_dir, uint32_t start, uint32_t end) {
+void blocklog_actions::extract_block_range(bfs::path block_dir, bfs::path output_dir, uint32_t start, uint32_t end) {
    report_time rt("extracting block range");
    EOS_ASSERT(end > start, block_log_exception, "extract range end must be greater than start");
-   const bool status = block_log::extract_block_range(block_dir, output_dir, start, end, false);
+   block_log::extract_block_range(block_dir, output_dir, start, end);
    rt.report();
-   return status;
 }
 
 int blocklog_actions::smoke_test() {

programs/leap-util/actions/blocklog.hpp

@@ -31,7 +31,7 @@ protected:
    void initialize();
    int trim_blocklog_end(bfs::path block_dir, uint32_t n);
    bool trim_blocklog_front(bfs::path block_dir, uint32_t n);
-   bool extract_block_range(bfs::path block_dir, bfs::path output_dir, uint32_t start, uint32_t end);
+   void extract_block_range(bfs::path block_dir, bfs::path output_dir, uint32_t start, uint32_t end);
 
    int make_index();
    int trim_blocklog();

tests/TestHarness/testUtils.py

@@ -68,8 +68,6 @@ class Utils:
     EosLauncherPath="programs/eosio-launcher/eosio-launcher"
     ShuttingDown=False
 
-    EosBlockLogPath="programs/eosio-blocklog/eosio-blocklog"
-
     FileDivider="================================================================="
     DataRoot="var"
     DataDir="%s/lib/" % (DataRoot)
@@ -408,18 +406,18 @@ class Utils:
         blockLogActionStr=None
         returnType=ReturnType.raw
         if blockLogAction==BlockLogAction.return_blocks:
-            blockLogActionStr=""
+            blockLogActionStr=" print-log --as-json-array "
             returnType=ReturnType.json
         elif blockLogAction==BlockLogAction.make_index:
-            blockLogActionStr=" --make-index "
+            blockLogActionStr=" make-index "
         elif blockLogAction==BlockLogAction.trim:
-            blockLogActionStr=" --trim "
+            blockLogActionStr=" trim-blocklog "
         elif blockLogAction==BlockLogAction.smoke_test:
-            blockLogActionStr=" --smoke-test "
+            blockLogActionStr=" smoke-test "
         else:
             unhandledEnumType(blockLogAction)
 
-        cmd="%s --blocks-dir %s --as-json-array %s%s%s%s" % (Utils.EosBlockLogPath, blockLogLocation, outputFileStr, firstStr, lastStr, blockLogActionStr)
+        cmd="%s block-log %s --blocks-dir %s  %s%s%s" % (Utils.LeapClientPath, blockLogActionStr, blockLogLocation, outputFileStr, firstStr, lastStr)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         rtn=None
         try:
@@ -572,11 +570,11 @@ class Utils:
 
     @staticmethod
     def makeHTTPReqStr(host : str, port : str, api_call : str, body : str, keepAlive=False) -> str:
-        hdr = "POST " + api_call + " HTTP/1.1\r\n" 
+        hdr = "POST " + api_call + " HTTP/1.1\r\n"
         hdr += f"Host: {host}:{port}\r\n"
         body += "\r\n"
         body_len = len(body)
-        hdr +=  f"content-length: {body_len}\r\n" 
+        hdr +=  f"content-length: {body_len}\r\n"
         hdr +=  "Accept: */*\r\n"
         hdr += "Connection: "
         if keepAlive:
@@ -613,7 +611,7 @@ class Utils:
     def readSocketDataStr(sock : socket.socket, maxMsgSize : int, enc : str) -> str:
         """Read data from a socket until maxMsgSize is reached or timeout
         Retrusn data as decoded string object"""
-        data = Utils.readSocketData(sock, maxMsgSize) 
+        data = Utils.readSocketData(sock, maxMsgSize)
         return data.decode(enc)
 
     @staticmethod
@@ -636,4 +634,3 @@ class Account(object):
 
     def __str__(self):
         return "Name: %s" % (self.name)
-

tests/block_log_util_test.py

@@ -23,7 +23,7 @@ from core_symbol import CORE_SYMBOL
 def verifyBlockLog(expected_block_num, trimmedBlockLog):
     firstBlockNum = expected_block_num
     for block in trimmedBlockLog:
-        assert 'block_num' in block, print("ERROR: eosio-blocklog didn't return block output")
+        assert 'block_num' in block, print("ERROR: leap-util didn't return block output")
         block_num = block['block_num']
         assert block_num == expected_block_num
         expected_block_num += 1
@@ -128,7 +128,7 @@ try:
 
     try:
         Print("Head block num %d will not be in block log (it will be in reversible DB), so --trim will throw an exception" % (headBlockNum))
-        output=cluster.getBlockLog(0, blockLogAction=BlockLogAction.trim, last=headBlockNum, throwException=True)
+        output=cluster.getBlockLog(0, blockLogAction=BlockLogAction.trim, first=0, last=headBlockNum, throwException=True)
         Utils.errorExit("BlockLogUtil --trim should have indicated error for last value set to lib (%d) " +
                         "which should not do anything since only trimming blocklog and not irreversible blocks" % (lib))
     except subprocess.CalledProcessError as ex:
@@ -136,13 +136,13 @@ try:
 
     beforeEndOfBlockLog=lib-20
     Print("Block num %d will definitely be at least one block behind the most recent entry in block log, so --trim will work" % (beforeEndOfBlockLog))
-    output=cluster.getBlockLog(0, blockLogAction=BlockLogAction.trim, last=beforeEndOfBlockLog, throwException=True)
+    output=cluster.getBlockLog(0, blockLogAction=BlockLogAction.trim, first=0, last=beforeEndOfBlockLog, throwException=True)
 
     Print("Kill the non production node, we want to verify its block log")
     cluster.getNode(2).kill(signal.SIGTERM)
 
     Print("Trim off block num 1 to remove genesis block from block log.")
-    output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.trim, first=2, throwException=True)
+    output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.trim, first=2, last=4294967295, throwException=True)
 
     Print("Smoke test the trimmed block log.")
     output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.smoke_test)
@@ -171,7 +171,7 @@ try:
 
     firstBlock = info["last_irreversible_block_num"]
     Print("Trim off block num %s." % (firstBlock))
-    output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.trim, first=firstBlock, throwException=True)
+    output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.trim, first=firstBlock, last=4294967295, throwException=True)
 
     Print("Smoke test the trimmed block log.")
     output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.smoke_test)

unittests/block_log_extract.cpp

@@ -4,6 +4,7 @@
 
 #include <eosio/chain/block_log.hpp>
 #include <eosio/chain/block.hpp>
+#include <regex>
 
 using namespace eosio::chain;
 
@@ -25,6 +26,22 @@ struct block_log_extract_fixture {
       log->append(p, p->calculate_id());
    }
 
+   static void rename_blocks_files(fc::path dir) {
+   // rename blocks files with block number range with those without
+   // i.e.   blocks-1-100.index  --> blocks.index
+   //        blocks-1-100.log    --> blocks.log
+      namespace bfs = boost::filesystem;
+      for (bfs::directory_iterator itr(dir); itr != bfs::directory_iterator{}; ++itr ) {
+         auto file_path = itr->path();
+         if ( !bfs::is_regular_file( file_path )) continue;
+         std::regex block_range_expression("-\\d+-\\d+");
+         auto new_path = std::regex_replace(file_path.string(), block_range_expression, "");
+         if (new_path != file_path)
+            bfs::rename(file_path, new_path);
+      }
+
+   }
+
    genesis_state gs;
    fc::temp_directory dir;
    std::optional<block_log> log;
@@ -37,7 +54,7 @@ BOOST_FIXTURE_TEST_CASE(extract_from_middle, block_log_extract_fixture) try {
    fc::temp_directory output_dir;
    block_num_type start=3, end=7;
    block_log::extract_block_range(dir.path(), output_dir.path(), start, end);
-
+   rename_blocks_files(output_dir.path());
    block_log new_log(output_dir.path());
 
    auto id = gs.compute_chain_id();
@@ -53,7 +70,7 @@ BOOST_FIXTURE_TEST_CASE(extract_from_start, block_log_extract_fixture) try {
    fc::temp_directory output_dir;
    block_num_type start=1, end=7;
    block_log::extract_block_range(dir.path(), output_dir.path(), start, end);
-
+   rename_blocks_files(output_dir.path());
    block_log new_log(output_dir.path());
 
    auto id = gs.compute_chain_id();
@@ -68,11 +85,11 @@ BOOST_FIXTURE_TEST_CASE(reextract_from_start, block_log_extract_fixture) try {
    fc::temp_directory output_dir;
    block_num_type start=1, end=9;
    block_log::extract_block_range(dir.path(), output_dir.path(), start, end);
-
+   rename_blocks_files(output_dir.path());
    fc::temp_directory output_dir2;
    end=6;
    block_log::extract_block_range(output_dir.path(), output_dir2.path(), start, end);
-
+   rename_blocks_files(output_dir2.path());
    block_log new_log(output_dir2.path());
 
    auto id = gs.compute_chain_id();
@@ -87,7 +104,7 @@ BOOST_FIXTURE_TEST_CASE(extract_to_end, block_log_extract_fixture) try {
    fc::temp_directory output_dir;
    block_num_type start=5, end=std::numeric_limits<block_num_type>::max();
    block_log::extract_block_range(dir.path(), output_dir.path(), start, end);
-
+   rename_blocks_files(output_dir.path());
    block_log new_log(output_dir.path());
 
    auto id = gs.compute_chain_id();