Commit 8703690c authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

Benchmarks: refactored JSON logging format 

Instead of writing the whole log as a valid JSON object, it is much
easier to write one JSON object per line. This can be processed in
Python with the Pandas package using the following code:

df = pandas.read_json(open("foo.log", "r"), orient="records", lines=True)

Also removed the outputMode parameter from the Benchmark and Logging
classes, since it was pretty useless.
parent c31b2268

src/Benchmarks/Benchmarks.h

+1 −4
Original line number Diff line number Diff line
@@ -67,10 +67,7 @@ class Benchmark 
      using MetadataColumns = typename Logger::MetadataColumns;

      using SolverMonitorType = Solvers::IterativeSolverMonitor< double, int >;



      Benchmark( int loops = 10,

               bool verbose = true,

               String outputMode = "",

               bool logFileAppend = false );

      Benchmark( int loops = 10, bool verbose = true );



      static void configSetup( Config::ConfigDescription& config );

src/Benchmarks/Benchmarks.hpp

+2 −5
Original line number Diff line number Diff line
@@ -25,11 +25,8 @@ namespace Benchmarks { 


template< typename Logger >

Benchmark< Logger >::

Benchmark( int loops,

           bool verbose,

           String outputMode,

           bool logFileAppend )

: logger(verbose, outputMode, logFileAppend), loops(loops)

Benchmark( int loops, bool verbose )

: logger(verbose), loops(loops)

{}



template< typename Logger >

src/Benchmarks/CustomLogging.h

+2 −7
Original line number Diff line number Diff line
@@ -23,11 +23,8 @@ class CustomLogging 
: public Logging

{

public:

   CustomLogging( int verbose = true,

                  std::string outputMode = "",

                  bool logFileAppend = false )

   : Logging(verbose), outputMode( outputMode )

   {}

   // inherit constructors

   using Logging::Logging;



   virtual void

   writeTitle( const std::string & title ) override
@@ -224,8 +221,6 @@ protected: 


   MetadataColumns metadataColumns;

   bool header_changed = true;



   std::string outputMode;

};



} // namespace Benchmarks

src/Benchmarks/JsonLogging.h

+19 −49
Original line number Diff line number Diff line
@@ -23,18 +23,12 @@ class JsonLogging 
: public Logging

{

public:

   JsonLogging( int verbose = true,

                std::string outputMode = "",

                bool logFileAppend = false )

   : Logging(verbose), outputMode( outputMode ), logFileAppend( logFileAppend )

   {}

   // inherit constructors

   using Logging::Logging;



   virtual void

   writeTitle( const std::string & title ) override

   {

      if( outputMode == "append" )

         return;



      if( verbose )

         std::cout << std::endl << "== " << title << " ==" << std::endl << std::endl;

   }
@@ -42,20 +36,13 @@ public: 
   virtual void

   writeMetadata( const MetadataMap & metadata ) override

   {

      if( outputMode == "append" )

         return;



      if( verbose )

      if( verbose ) {

         std::cout << "properties:" << std::endl;



      for( auto & it : metadata ) {

         if( verbose )

         for( auto & it : metadata )

            std::cout << "   " << it.first << " = " << it.second << std::endl;

      }



      if( verbose )

         std::cout << std::endl;

      }

   }



   virtual void setMetadataColumns( const MetadataColumns& elements ) override

   {
@@ -113,10 +100,8 @@ public: 
   {

      TNL_ASSERT_EQ( headerElements.size(), rowElements.size(), "elements must have equal sizes" );

      TNL_ASSERT_EQ( headerElements.size(), widths.size(), "elements must have equal sizes" );

      if( this->lineStarted )

         log << "," << std::endl;



      log << "      {" << std::endl;

      log << "{";



      // write common logs

      int idx( 0 );
@@ -125,7 +110,7 @@ public: 
         if( verbose )

            std::cout << std::setw( 20 ) << lg.second;

         if( idx++ > 0 )

            log << "," << std::endl;

            log << ", ";

         log << "\"" << lg.first << "\": \"" << lg.second << "\"";

      }
@@ -135,17 +120,16 @@ public: 
         if( verbose )

            std::cout << std::setw( widths[ i ] ) << el;

         if( idx++ > 0 )

            log << "," << std::endl;

            log << ", ";

         log << "\"" << headerElements[ i ] << "\": \"" << el << "\"";

         i++;

      }

      if( ! errorMessage.empty() ) {

         if( idx++ > 0 )

            log << "," << std::endl;

            log << ", ";

         log << "\"error\": \"" << errorMessage << "\"";

      }

      log << std::endl << "      }";

      this->lineStarted = true;

      log << "}" << std::endl;

      if( verbose )

         std::cout << std::endl;

   }
@@ -164,26 +148,22 @@ public: 
   virtual void

   writeErrorMessage( const std::string& message ) override

   {

      if( this->lineStarted )

         log << "," << std::endl;



      log << "      {" << std::endl;

      log << "{";



      // write common logs

      int idx( 0 );

      for( auto lg : this->metadataColumns )

      {

         if( idx++ > 0 )

            log << "," << std::endl;

            log << ", ";

         log << "\"" << lg.first << "\": \"" << lg.second << "\"";

      }



      if( idx++ > 0 )

         log << "," << std::endl;

         log << ", ";

      log << "\"error\": \"" << message << "\"";



      log << std::endl << "      }";

      this->lineStarted = true;

      log << "}" << std::endl;

   }



   virtual void
@@ -194,11 +174,6 @@ public: 


   virtual bool save( std::ostream & logFile ) override

   {

      if( ! this->logFileAppend )

      {

         logFile << "{" << std::endl;

         logFile << "   \"results\" : [ " << std::endl;

      }

      logFile << log.str();

      if( logFile.good() ) {

         log.str() = "";
@@ -225,11 +200,6 @@ protected: 


   MetadataColumns metadataColumns;

   bool header_changed = true;



   std::string outputMode;



   bool lineStarted = false;

   bool logFileAppend = false;

};



} // namespace Benchmarks

src/Benchmarks/SpMV/tnl-benchmark-spmv.h

+2 −12
Original line number Diff line number Diff line
@@ -80,10 +80,9 @@ setupConfig( Config::ConfigDescription & config ) 
   config.addEntry< bool >( "with-legacy-matrices", "Perform benchmark even for legacy TNL matrix formats.", true );

   config.addEntry< bool >( "with-all-cpu-tests", "All matrix formats are tested on both CPU and GPU. ", false );

   config.addEntry< String >( "log-file", "Log file name.", "tnl-benchmark-spmv::" + getCurrDateTime() + ".log");

   config.addEntry< String >( "output-mode", "Mode for opening the log file - 'close' will only finalize the log file.", "append" );

   config.addEntry< String >( "output-mode", "Mode for opening the log file.", "append" );

   config.addEntryEnum( "append" );

   config.addEntryEnum( "overwrite" );

   config.addEntryEnum( "close" );

   config.addEntry< String >( "precision", "Precision of the arithmetics.", "double" );

   config.addEntryEnum( "float" );

   config.addEntryEnum( "double" );
@@ -134,22 +133,13 @@ main( int argc, char* argv[] ) 
   const int verboseMR = parameters.getParameter< int >( "verbose-MReader" );



   // open log file

   if( outputMode == "close" )

   {

      std::fstream file;

      file.open( logFileName.getString(), std::ios::out | std::ios::app );

      file << std::endl << "   ]" << std::endl << "}";

      return EXIT_SUCCESS;

   }

   if( inputFileName == "" )

   {

      std::cerr << "ERROR: Input file name is required." << std::endl;

      return EXIT_FAILURE;

   }

   bool logFileAppend( false );

   if( std::experimental::filesystem::exists(logFileName.getString()) )

   {

      logFileAppend = true;

      std::cout << "Log file " << logFileName << " exists and ";

      if( outputMode == "append" )

         std::cout << "new logs will be appended." << std::endl;
@@ -163,7 +153,7 @@ main( int argc, char* argv[] ) 
   std::ofstream logFile( logFileName.getString(), mode );



   // init benchmark and common metadata

   TNL::Benchmarks::SpMV::BenchmarkType benchmark( loops, verbose, outputMode, logFileAppend );

   TNL::Benchmarks::SpMV::BenchmarkType benchmark( loops, verbose );



   // prepare global metadata

   Logging::MetadataMap metadata = getHardwareMetadata();