add counter of dropped transactions and size of mempool to block stats

Also colony_basic_test was used to show how dropping looks in practice (transactions are dropped once
when test switches from 2MB blocks to 64kB blocks).

Author: ABW

libraries/chain/block_flow_control.cpp

@@ -72,9 +72,9 @@ void block_flow_control::on_end_of_apply_block() const
   current_phase = phase::APPLIED;
 }
 
-void block_flow_control::on_end_of_processing( uint32_t _exp_txs, uint32_t _fail_txs, uint32_t _ok_txs, uint32_t _post_txs, uint32_t _lib ) const
+void block_flow_control::on_end_of_processing( uint32_t _exp_txs, uint32_t _fail_txs, uint32_t _ok_txs, uint32_t _post_txs, uint32_t _drop_txs, size_t _mempool_size, uint32_t _lib ) const
 {
-  stats.on_cleanup( _exp_txs, _fail_txs, _ok_txs, _post_txs, _lib );
+  stats.on_cleanup( _exp_txs, _fail_txs, _ok_txs, _post_txs, _drop_txs, _mempool_size, _lib );
   if( !except && current_phase == phase::APPLIED )
     current_phase = phase::END;
 }
@@ -173,7 +173,13 @@ fc::variant_object block_flow_control::get_report( report_type rt ) const
       ( "exp", stats.get_txs_expired_after_block() )
       ( "fail", stats.get_txs_failed_after_block() )
       ( "appl", stats.get_txs_reapplied_after_block() )
-      ( "post", stats.get_txs_postponed_after_block() );
+      ( "post", stats.get_txs_postponed_after_block() )
+      ( "drop", stats.get_txs_dropped_after_block() );
+    if( rt == report_type::FULL )
+    {
+      after
+        ( "size", stats.get_size_of_txs_left_after_block() );
+    }
     report( "after", after.get() );
   }
   if( rt != report_type::MINIMAL )

libraries/chain/include/hive/chain/block_flow_control.hpp

@@ -34,13 +34,15 @@ class block_stats
     end_work = fc::time_point::now();
   }
 
-  void on_cleanup( uint32_t _exp_txs, uint32_t _fail_txs, uint32_t _ok_txs, uint32_t _post_txs, uint32_t _lib )
+  void on_cleanup( uint32_t _exp_txs, uint32_t _fail_txs, uint32_t _ok_txs, uint32_t _post_txs, uint32_t _drop_txs, size_t _mempool_size, uint32_t _lib )
   {
     end_cleanup = fc::time_point::now();
     txs_expired = _exp_txs;
     txs_failed = _fail_txs;
     txs_reapplied = _ok_txs;
     txs_postponed = _post_txs;
+    txs_dropped = _drop_txs;
+    txs_size = _mempool_size;
     last_irreversible_block_num = _lib;
   }
 
@@ -62,6 +64,9 @@ class block_stats
   uint32_t get_txs_failed_after_block() const { return txs_failed; }
   uint32_t get_txs_reapplied_after_block() const { return txs_reapplied; }
   uint32_t get_txs_postponed_after_block() const { return txs_postponed; }
+  uint32_t get_txs_dropped_after_block() const { return txs_dropped; }
+
+  size_t get_size_of_txs_left_after_block() const { return txs_size; }
 
   uint32_t get_last_irreversible_block_num() const { return last_irreversible_block_num; }
 
@@ -91,6 +96,12 @@ class block_stats
   uint32_t txs_reapplied = 0;
   //number of transactions that were not touched during pending reapplication due to time limit
   uint32_t txs_postponed = 0;
+  //number of would-be-postponed transactions that were dropped during pending reapplication due to mempool size limit
+  uint32_t txs_dropped = 0;
+
+  //sum of packed sizes of transactions left in mempool after pending reapplication
+  size_t txs_size = 0;
+
   //last irreversible block at the time block work was done
   uint32_t last_irreversible_block_num = 0;
 };
@@ -150,7 +161,7 @@ class block_flow_control
   virtual void on_end_of_apply_block() const;
 
   // after reapplication of pending transactions (only overridden in tests)
-  virtual void on_end_of_processing( uint32_t _exp_txs, uint32_t _fail_txs, uint32_t _ok_txs, uint32_t _post_txs, uint32_t _lib ) const;
+  virtual void on_end_of_processing( uint32_t _exp_txs, uint32_t _fail_txs, uint32_t _ok_txs, uint32_t _post_txs, uint32_t _drop_txs, size_t _mempool_size, uint32_t _lib ) const;
 
   // in case of exception
   virtual void on_failure( const fc::exception& e ) const;

libraries/chain/include/hive/chain/db_with.hpp

@@ -195,7 +195,7 @@ struct pending_transactions_restorer
       _db._popped_tx.clear();
     } );
 
-    _block_ctrl.on_end_of_processing( expired_txs, failed_txs, applied_txs, postponed_txs, _db.get_last_irreversible_block_num() );
+    _block_ctrl.on_end_of_processing( expired_txs, failed_txs, applied_txs, postponed_txs, dropped_txs, pending_size, _db.get_last_irreversible_block_num() );
     if( in_sync && ( postponed_txs || expired_txs ) )
     {
       wlog("Postponed ${postponed_txs} pending transactions. ${applied_txs} were applied. ${expired_txs} expired.",

tests/unit/plugin_tests/witness_tests.cpp

@@ -1282,7 +1282,8 @@ BOOST_AUTO_TEST_CASE( colony_basic_test )
       config_line_t( { "colony-reply", { R"~({"min":30,"max":1000,"weight":110,"exponent":5})~" } } ),
       config_line_t( { "colony-vote", { R"~({"weight":2070})~" } } ),
       config_line_t( { "colony-transfer", { R"~({"min":0,"max":350,"weight":87,"exponent":4})~" } } ),
-      config_line_t( { "colony-custom", { R"~({"min":20,"max":400,"weight":6006,"exponent":1})~" } } )
+      config_line_t( { "colony-custom", { R"~({"min":20,"max":400,"weight":6006,"exponent":1})~" } } ),
+      config_line_t( { "max-mempool-size", { "0" } } ),
     }, "1G" );
 
     fc::thread api_thread;

tests/unit/tests/block_tests.cpp

@@ -1471,11 +1471,11 @@ class test_block_flow_control : public BASE
     verify( expectation::END_OF_BLOCK );
   }
 
-  virtual void on_end_of_processing( uint32_t _exp_txs, uint32_t _fail_txs, uint32_t _ok_txs, uint32_t _post_txs, uint32_t _lib ) const override
+  virtual void on_end_of_processing( uint32_t _exp_txs, uint32_t _fail_txs, uint32_t _ok_txs, uint32_t _post_txs, uint32_t _drop_txs, size_t _mempool_size, uint32_t _lib ) const override
   {
     // pending transactions are actually reapplied even after failed block and that is before exception
     // is passed to the block flow control; it means we can't be sure on what is current phase
-    BASE::on_end_of_processing( _exp_txs, _fail_txs, _ok_txs, _post_txs, _lib );
+    BASE::on_end_of_processing( _exp_txs, _fail_txs, _ok_txs, _post_txs, _drop_txs, _mempool_size, _lib );
     verify( expectation::END_PROCESSING );
   }