Splitting source code of Benchmark into two files, added test for existence of the log file. (d4ea98b3) · Commits · TNL / tnl-dev

src/Benchmarks/Benchmark.hpp

0 → 100644

+312 −0

Original line number	Diff line number	Diff line
		/***************************************************************************
		Benchmarks.hpp - description
		-------------------
		begin : Jun 7, 2021
		copyright : (C) 2021 by Tomas Oberhuber et al.
		email : tomas.oberhuber@fjfi.cvut.cz
		***************************************************************************/

		/* See Copyright Notice in tnl/Copyright */

		// Implemented by: Jakub Klinkovsky,
		// Tomas Oberhuber

		#pragma once

		#include "FunctionTimer.h"
		#include "Logging.h"

		#include <iostream>
		#include <exception>
		#include <limits>

		#include <TNL/String.h>

		#include <TNL/Devices/Host.h>
		#include <TNL/SystemInfo.h>
		#include <TNL/Cuda/DeviceInfo.h>
		#include <TNL/Config/ConfigDescription.h>
		#include <TNL/MPI/Wrappers.h>

		namespace TNL {
		namespace Benchmarks {


		template< typename Logger >
		Benchmark< Logger >::
		Benchmark( int loops,
		bool verbose,
		String outputMode,
		bool logFileAppend )
		: Logger(verbose, outputMode, logFileAppend), loops(loops)
		{}

		template< typename Logger >
		void
		Benchmark< Logger >::
		configSetup( Config::ConfigDescription& config )
		{
		config.addEntry< int >( "loops", "Number of iterations for every computation.", 10 );
		config.addEntry< bool >( "reset", "Call reset function between loops.", true );
		config.addEntry< double >( "min-time", "Minimal real time in seconds for every computation.", 0.0 );
		config.addEntry< int >( "verbose", "Verbose mode, the higher number the more verbosity.", 1 );
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		setup( const Config::ParameterContainer& parameters )
		{
		this->loops = parameters.getParameter< int >( "loops" );
		this->reset = parameters.getParameter< bool >( "reset" );
		this->minTime = parameters.getParameter< double >( "min-time" );
		const int verbose = parameters.getParameter< int >( "verbose" );
		Logger::setVerbose( verbose );
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		setLoops( int loops )
		{
		this->loops = loops;
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		setMinTime( const double& minTime )
		{
		this->minTime = minTime;
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		newBenchmark( const String & title )
		{
		Logger::closeTable();
		Logger::writeTitle( title );
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		newBenchmark( const String & title,
		MetadataMap metadata )
		{
		Logger::closeTable();
		Logger::writeTitle( title );
		// add loops and reset flag to metadata
		metadata["loops"] = convertToString(loops);
		metadata["reset"] = convertToString( reset );
		metadata["minimal test time"] = convertToString( minTime );
		Logger::writeMetadata( metadata );
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		setMetadataColumns( const MetadataColumns & metadata )
		{
		if( Logger::metadataColumns != metadata )
		Logger::header_changed = true;
		Logger::metadataColumns = metadata;
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		setOperation( const String & operation,
		const double datasetSize,
		const double baseTime )
		{
		monitor.setStage( operation.getString() );
		if( Logger::metadataColumns.size() > 0 && String(Logger::metadataColumns[ 0 ].first) == "operation" ) {
		Logger::metadataColumns[ 0 ].second = operation;
		}
		else {
		Logger::metadataColumns.insert( Logger::metadataColumns.begin(), {"operation", operation} );
		}
		setOperation( datasetSize, baseTime );
		Logger::header_changed = true;
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		setOperation( const double datasetSize,
		const double baseTime )
		{
		this->datasetSize = datasetSize;
		this->baseTime = baseTime;
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		createHorizontalGroup( const String & name,
		int subcolumns )
		{
		if( Logger::horizontalGroups.size() == 0 ) {
		Logger::horizontalGroups.push_back( {name, subcolumns} );
		}
		else {
		auto & last = Logger::horizontalGroups.back();
		if( last.first != name && last.second > 0 ) {
		Logger::horizontalGroups.push_back( {name, subcolumns} );
		}
		else {
		last.first = name;
		last.second = subcolumns;
		}
		}
		}

		template< typename Logger >
		template< typename Device,
		typename ResetFunction,
		typename ComputeFunction >
		double
		Benchmark< Logger >::
		time( ResetFunction reset,
		const String & performer,
		ComputeFunction & compute,
		BenchmarkResult< Logger > & result )
		{
		result.time = std::numeric_limits<double>::quiet_NaN();
		result.stddev = std::numeric_limits<double>::quiet_NaN();
		FunctionTimer< Device > functionTimer;
		try {
		if( Logger::verbose > 1 ) {
		// run the monitor main loop
		Solvers::SolverMonitorThread monitor_thread( monitor );
		if( this->reset )
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, reset, loops, minTime, Logger::verbose, monitor );
		else
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, loops, minTime, Logger::verbose, monitor );
		}
		else {
		if( this->reset )
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, reset, loops, minTime, Logger::verbose, monitor );
		else
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, loops, minTime, Logger::verbose, monitor );
		}
		this->performedLoops = functionTimer.getPerformedLoops();
		}
		catch ( const std::exception& e ) {
		std::cerr << "timeFunction failed due to a C++ exception with description: " << e.what() << std::endl;
		}

		result.bandwidth = datasetSize / result.time;
		result.speedup = this->baseTime / result.time;
		if( this->baseTime == 0.0 )
		this->baseTime = result.time;

		Logger::writeTableHeader( performer, result.getTableHeader() );
		Logger::writeTableRow( performer, result.getRowElements() );

		return this->baseTime;
		}

		template< typename Logger >
		template< typename Device,
		typename ResetFunction,
		typename ComputeFunction >
		inline double
		Benchmark< Logger >::
		time( ResetFunction reset,
		const String& performer,
		ComputeFunction& compute )
		{
		BenchmarkResult< Logger > result;
		return time< Device, ResetFunction, ComputeFunction >( reset, performer, compute, result );
		}

		template< typename Logger >
		template< typename Device,
		typename ComputeFunction >
		double
		Benchmark< Logger >::
		time( const String & performer,
		ComputeFunction & compute,
		BenchmarkResult< Logger > & result )
		{
		result.time = std::numeric_limits<double>::quiet_NaN();
		result.stddev = std::numeric_limits<double>::quiet_NaN();
		FunctionTimer< Device > functionTimer;
		try {
		if( Logger::verbose > 1 ) {
		// run the monitor main loop
		Solvers::SolverMonitorThread monitor_thread( monitor );
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, loops, minTime, Logger::verbose, monitor );
		}
		else {
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, loops, minTime, Logger::verbose, monitor );
		}
		}
		catch ( const std::exception& e ) {
		std::cerr << "Function timer failed due to a C++ exception with description: " << e.what() << std::endl;
		}

		result.bandwidth = datasetSize / result.time;
		result.speedup = this->baseTime / result.time;
		if( this->baseTime == 0.0 )
		this->baseTime = result.time;

		Logger::writeTableHeader( performer, result.getTableHeader() );
		Logger::writeTableRow( performer, result.getRowElements() );

		return this->baseTime;
		}

		template< typename Logger >
		template< typename Device,
		typename ComputeFunction >
		inline double
		Benchmark< Logger >::
		time( const String & performer,
		ComputeFunction & compute )
		{
		BenchmarkResult< Logger > result;
		return time< Device, ComputeFunction >( performer, compute, result );
		}

		template< typename Logger >
		void
		Benchmark< Logger >::
		addErrorMessage( const char* msg,
		int numberOfComputations )
		{
		// each computation has 3 subcolumns
		const int colspan = 3 * numberOfComputations;
		Logger::writeErrorMessage( msg, colspan );
		std::cerr << msg << std::endl;
		}

		template< typename Logger >
		auto
		Benchmark< Logger >::
		getMonitor() -> SolverMonitorType&
		{
		return monitor;
		}

		template< typename Logger >
		int
		Benchmark< Logger >::
		getPerformedLoops() const
		{
		return this->performedLoops;
		}

		template< typename Logger >
		bool
		Benchmark< Logger >::
		isResetingOn() const
		{
		return reset;
		}

		} // namespace Benchmarks
		} // namespace TNL

src/Benchmarks/Benchmarks.h

+126 −273

Original line number	Diff line number	Diff line
		/***************************************************************************
		benchmarks.h - description
		Benchmarks.h - description
		-------------------
		begin : Dec 30, 2015
		copyright : (C) 2015 by Tomas Oberhuber et al.
		@@ -79,70 +79,29 @@ public:

		Benchmark( int loops = 10,
		bool verbose = true,
		String outputMode = "" )
		: Logger(verbose, outputMode), loops(loops)
		{}
		String outputMode = "",
		bool logFileAppend = false );

		static void configSetup( Config::ConfigDescription& config )
		{
		config.addEntry< int >( "loops", "Number of iterations for every computation.", 10 );
		config.addEntry< bool >( "reset", "Call reset function between loops.", true );
		config.addEntry< double >( "min-time", "Minimal real time in seconds for every computation.", 0.0 );
		config.addEntry< int >( "verbose", "Verbose mode, the higher number the more verbosity.", 1 );
		}
		static void configSetup( Config::ConfigDescription& config );

		void setup( const Config::ParameterContainer& parameters );

		void setup( const Config::ParameterContainer& parameters )
		{
		this->loops = parameters.getParameter< int >( "loops" );
		this->reset = parameters.getParameter< bool >( "reset" );
		this->minTime = parameters.getParameter< double >( "min-time" );
		const int verbose = parameters.getParameter< int >( "verbose" );
		Logger::setVerbose( verbose );
		}
		// TODO: ensure that this is not called in the middle of the benchmark
		// (or just remove it completely?)
		void
		setLoops( int loops )
		{
		this->loops = loops;
		}
		void setLoops( int loops );

		void setMinTime( const double& minTime )
		{
		this->minTime = minTime;
		}
		void setMinTime( const double& minTime );

		// Marks the start of a new benchmark
		void
		newBenchmark( const String & title )
		{
		Logger::closeTable();
		Logger::writeTitle( title );
		}
		void newBenchmark( const String & title );

		// Marks the start of a new benchmark (with custom metadata)
		void
		newBenchmark( const String & title,
		MetadataMap metadata )
		{
		Logger::closeTable();
		Logger::writeTitle( title );
		// add loops and reset flag to metadata
		metadata["loops"] = convertToString(loops);
		metadata["reset"] = convertToString( reset );
		metadata["minimal test time"] = convertToString( minTime );
		Logger::writeMetadata( metadata );
		}
		void newBenchmark( const String & title,
		MetadataMap metadata );

		// Sets metadata columns -- values used for all subsequent rows until
		// the next call to this function.
		void
		setMetadataColumns( const MetadataColumns & metadata )
		{
		if( Logger::metadataColumns != metadata )
		Logger::header_changed = true;
		Logger::metadataColumns = metadata;
		}
		void setMetadataColumns( const MetadataColumns & metadata );

		// TODO: maybe should be renamed to createVerticalGroup and ensured that vertical and horizontal groups are not used within the same "Benchmark"
		// Sets current operation -- operations expand the table vertically
		@@ -153,48 +112,17 @@ public:
		void
		setOperation( const String & operation,
		const double datasetSize = 0.0, // in GB
		const double baseTime = 0.0 )
		{
		monitor.setStage( operation.getString() );
		if( Logger::metadataColumns.size() > 0 && String(Logger::metadataColumns[ 0 ].first) == "operation" ) {
		Logger::metadataColumns[ 0 ].second = operation;
		}
		else {
		Logger::metadataColumns.insert( Logger::metadataColumns.begin(), {"operation", operation} );
		}
		setOperation( datasetSize, baseTime );
		Logger::header_changed = true;
		}
		const double baseTime = 0.0 );

		void
		setOperation( const double datasetSize = 0.0,
		const double baseTime = 0.0 )
		{
		this->datasetSize = datasetSize;
		this->baseTime = baseTime;
		}
		void setOperation( const double datasetSize = 0.0,
		const double baseTime = 0.0 );

		// Creates new horizontal groups inside a benchmark -- increases the number
		// of columns in the "Benchmark", implies column spanning.
		// (Useful e.g. for SpMV formats, different configurations etc.)
		void
		createHorizontalGroup( const String & name,
		int subcolumns )
		{
		if( Logger::horizontalGroups.size() == 0 ) {
		Logger::horizontalGroups.push_back( {name, subcolumns} );
		}
		else {
		auto & last = Logger::horizontalGroups.back();
		if( last.first != name && last.second > 0 ) {
		Logger::horizontalGroups.push_back( {name, subcolumns} );
		}
		else {
		last.first = name;
		last.second = subcolumns;
		}
		}
		}
		int subcolumns );

		// Times a single ComputeFunction. Subsequent calls implicitly split
		// the current "horizontal group" into sub-columns identified by
		@@ -205,138 +133,61 @@ public:
		template< typename Device,
		typename ResetFunction,
		typename ComputeFunction >
		double
		time( ResetFunction reset,
		double time( ResetFunction reset,
		const String & performer,
		ComputeFunction & compute,
		BenchmarkResult< Logger > & result )
		{
		result.time = std::numeric_limits<double>::quiet_NaN();
		result.stddev = std::numeric_limits<double>::quiet_NaN();
		FunctionTimer< Device > functionTimer;
		try {
		if( Logger::verbose > 1 ) {
		// run the monitor main loop
		Solvers::SolverMonitorThread monitor_thread( monitor );
		if( this->reset )
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, reset, loops, minTime, Logger::verbose, monitor );
		else
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, loops, minTime, Logger::verbose, monitor );
		}
		else {
		if( this->reset )
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, reset, loops, minTime, Logger::verbose, monitor );
		else
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, loops, minTime, Logger::verbose, monitor );
		}
		this->performedLoops = functionTimer.getPerformedLoops();
		}
		catch ( const std::exception& e ) {
		std::cerr << "timeFunction failed due to a C++ exception with description: " << e.what() << std::endl;
		}

		result.bandwidth = datasetSize / result.time;
		result.speedup = this->baseTime / result.time;
		if( this->baseTime == 0.0 )
		this->baseTime = result.time;

		Logger::writeTableHeader( performer, result.getTableHeader() );
		Logger::writeTableRow( performer, result.getRowElements() );

		return this->baseTime;
		}
		BenchmarkResult< Logger > & result );

		template< typename Device,
		typename ResetFunction,
		typename ComputeFunction >
		inline double
		time( ResetFunction reset,
		inline double time( ResetFunction reset,
		const String & performer,
		ComputeFunction & compute )
		{
		ComputeFunction & compute );
		/*{
		BenchmarkResult< Logger > result;
		return time< Device, ResetFunction, ComputeFunction >( reset, performer, compute, result );
		}
		}*/

		/****
		* The same methods as above but without reset function
		*/
		template< typename Device,
		typename ComputeFunction >
		double
		time( const String & performer,
		double time( const String & performer,
		ComputeFunction & compute,
		BenchmarkResult< Logger > & result )
		{
		result.time = std::numeric_limits<double>::quiet_NaN();
		result.stddev = std::numeric_limits<double>::quiet_NaN();
		FunctionTimer< Device > functionTimer;
		try {
		if( Logger::verbose > 1 ) {
		// run the monitor main loop
		Solvers::SolverMonitorThread monitor_thread( monitor );
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, loops, minTime, Logger::verbose, monitor );
		}
		else {
		std::tie( result.time, result.stddev ) = functionTimer.timeFunction( compute, loops, minTime, Logger::verbose, monitor );
		}
		}
		catch ( const std::exception& e ) {
		std::cerr << "Function timer failed due to a C++ exception with description: " << e.what() << std::endl;
		}

		result.bandwidth = datasetSize / result.time;
		result.speedup = this->baseTime / result.time;
		if( this->baseTime == 0.0 )
		this->baseTime = result.time;

		Logger::writeTableHeader( performer, result.getTableHeader() );
		Logger::writeTableRow( performer, result.getRowElements() );

		return this->baseTime;
		}
		BenchmarkResult< Logger > & result );

		template< typename Device,
		typename ComputeFunction >
		inline double
		time( const String & performer,
		ComputeFunction & compute )
		{
		BenchmarkResult< Logger > result;
		return time< Device, ComputeFunction >( performer, compute, result );
		}
		inline double time( const String & performer,
		ComputeFunction & compute );

		// Adds an error message to the log. Should be called in places where the
		// "time" method could not be called (e.g. due to failed allocation).
		void
		addErrorMessage( const char* msg,
		int numberOfComputations = 1 ) {
		// each computation has 3 subcolumns
		const int colspan = 3 * numberOfComputations;
		Logger::writeErrorMessage( msg, colspan );
		std::cerr << msg << std::endl;
		}
		void addErrorMessage( const char* msg,
		int numberOfComputations = 1 );

		using Logger::save;

		SolverMonitorType& getMonitor() {
		return monitor;
		}
		SolverMonitorType& getMonitor();

		int getPerformedLoops() const {
		return this->performedLoops;
		}
		int getPerformedLoops() const;

		bool isResetingOn() const {
		return reset;
		}
		bool isResetingOn() const;

		protected:

		int loops = 1, performedLoops = 0;

		double minTime = 0.0;

		double datasetSize = 0.0;

		double baseTime = 0.0;

		bool reset = true;

		SolverMonitorType monitor;
		};

		@@ -396,3 +247,5 @@ inline typename Benchmark< Logger >::MetadataMap getHardwareMetadata()

		} // namespace Benchmarks
		} // namespace TNL

		#include <Benchmarks/Benchmark.hpp>

src/Benchmarks/JsonLogging.h

+4 −2

Original line number	Diff line number	Diff line
		@@ -89,8 +89,9 @@ public:
		using RowElements = JsonLoggingRowElements;

		JsonLogging( int verbose = true,
		String outputMode = "" )
		: verbose(verbose), outputMode( outputMode )
		String outputMode = "",
		bool logFileAppend = false )
		: verbose(verbose), outputMode( outputMode ), logFileAppend( logFileAppend )
		{}

		void
		@@ -313,6 +314,7 @@ protected:

		bool lineStarted = false;
		bool resultsStarted = false;
		bool logFileAppend = false;
		};

		} // namespace Benchmarks

src/Benchmarks/Logging.h

+2 −1

Original line number	Diff line number	Diff line
		@@ -87,7 +87,8 @@ public:
		using RowElements = LoggingRowElements;

		Logging( int verbose = true,
		String outputMode = "" )
		String outputMode = "",
		bool logFileAppend = false )
		: verbose(verbose), outputMode( outputMode )
		{}

src/Benchmarks/SpMV/tnl-benchmark-spmv.h

+14 −6

Original line number	Diff line number	Diff line
		@@ -74,7 +74,7 @@ setupConfig( Config::ConfigDescription & config )
		config.addEntry< bool >( "with-symmetric-matrices", "Perform benchmark even for symmetric matrix formats.", true );
		config.addEntry< bool >( "with-legacy-matrices", "Perform benchmark even for legacy TNL matrix formats.", true );
		config.addEntry< String >( "log-file", "Log file name.", "tnl-benchmark-spmv::" + getCurrDateTime() + ".log");
		config.addEntry< String >( "output-mode", "Mode for opening the log file.", "overwrite" );
		config.addEntry< String >( "output-mode", "Mode for opening the log file.", "append" );
		config.addEntryEnum( "append" );
		config.addEntryEnum( "overwrite" );
		config.addEntry< String >( "precision", "Precision of the arithmetics.", "double" );
		@@ -124,16 +124,24 @@ main( int argc, char* argv[] )
		const int verboseMR = parameters.getParameter< int >( "verbose-MReader" );

		// open log file
		bool exist = std::experimental::filesystem::exists(logFileName.getString());
		if( ! exist )
		outputMode = "";
		bool logFileAppend( false );
		if( std::experimental::filesystem::exists(logFileName.getString()) )
		{
		logFileAppend = true;
		std::cout << "Log file " << logFileName << "exists and ";
		if( outputMode == "append" )
		std::cout << "new logs will be appended." << std::endl;
		else
		std::cout << "will be overwritten." << std::endl;
		}

		auto mode = std::ios::out;
		if( outputMode == "append" )
		mode \|= std::ios::app;
		std::ofstream logFile( logFileName.getString(), mode );

		// init benchmark and common metadata
		TNL::Benchmarks::SpMV::BenchmarkType benchmark( loops, verbose, outputMode );
		TNL::Benchmarks::SpMV::BenchmarkType benchmark( loops, verbose, outputMode, logFileAppend );

		// prepare global metadata
		TNL::Benchmarks::SpMV::BenchmarkType::MetadataMap metadata = getHardwareMetadata< Logging >();