Benchmarks: updated script for log->HTML conversion to support the new format (af2daed3) · Commits · TNL / tnl-dev

tests/benchmarks/share/tnl-log-to-html.py

+382 −248

Original line number	Diff line number	Diff line
		#!/usr/bin/env python
		#!/usr/bin/env python3

		import sys;
		import sys
		import collections

		class columnFormating:

		@@ -25,7 +26,7 @@ class columnFormating:
		self.sortingFile = word


		def write( self, htmlFile, line ):
		def get( self, value ):
		color = ""
		if len( self.coloring ) > 0:
		for token in self.coloring:
		@@ -33,23 +34,25 @@ class columnFormating:
		color = token
		else:
		try:
		if float( token ) > float( line ):
		if float( token ) > float( value ):
		break
		except ValueError:
		color = ""
		break

		html = ""
		if color != "":
		htmlFile.write( " bgcolor=\"" )
		htmlFile.write( color )
		htmlFile.write( "\"" )
		html += "bgcolor=\"{}\"".format(color)

		if self.sorting == "+" or self.sorting == "-":
		try:
		number = float( line )
		number = float( value )
		self.sortingData.append( number )
		except ValueError:
		self.sortingNaNs += 1

		return html

		def processSorting( self ):
		if self.sorting == "none":
		return
		@@ -73,29 +76,32 @@ class columnFormating:

		class tableColumn:

		def __init__( self, level, data ):
		self.level = level
		self.maxLevel = 0
		self.height = 0
		def __init__( self, level, data, parentPath=None ):
		self.subcolumns = []
		self.height = 0
		self.numberOfSubcolumns = 0
		self.rowspan = 0
		dataSplit = data.split( ':', 1 );
		label = dataSplit[ 0 ];
		self.label = label.rstrip( ' ' );
		#print self.label

		self.level = level
		dataSplit = data.split( ':', 1 )
		self.label = dataSplit[ 0 ].strip()
		if len(dataSplit) == 2:
		self.formating = columnFormating( dataSplit[ 1 ] )
		self.attributes = dataSplit[1]
		else:
		self.formating = columnFormating( "" )
		self.attributes = ""

		if parentPath is None:
		self.path = []
		else:
		# make a copy!
		self.path = parentPath[:]
		self.path.append(self.label)

		def insertSubcolumn( self, level, label ):
		if level > self.maxLevel:
		self.maxLevel = level
		if level == self.level + 1:
		self.subcolumns.append( tableColumn( level, label ) )
		self.subcolumns.append( tableColumn( level, label, self.path ) )
		if level > self.level + 1:
		self.subcolumns[ len( self.subcolumns ) - 1 ].insertSubcolumn( level, label )
		self.subcolumns[ -1 ].insertSubcolumn( level, label )

		def countSubcolumns( self ):
		if( len( self.subcolumns ) == 0 ):
		@@ -125,138 +131,266 @@ class tableColumn:
		for subcolumn in self.subcolumns:
		subcolumn.recomputeLevel( self.level + self.rowspan )

		def writeToColumnsHeader( self, htmlFile, height, currentLevel ):
		if currentLevel > self.level:
		for subcolumn in self.subcolumns:
		subcolumn.writeToColumnsHeader( htmlFile, self.height , currentLevel )
		def getColumnHeader( self, currentLevel ):
		if currentLevel == self.level:
		#print "Label = ", self.label, " self.height = ", self.height, " height = ", height
		htmlFile.write( " <td rowspan=" + str( self.rowspan ) + " colspan=" + str( self.numberOfSubcolumns) + ">" + self.label + "</td>\n" )
		return " <td rowspan=" + str( self.rowspan ) + " colspan=" + str( self.numberOfSubcolumns) + ">" + self.label + "</td>\n"
		return ""

		def pickLeafColumns( self, leafColumns ):
		if len( self.subcolumns ) == 0:
		#print "Appending leaf column ", self.label
		leafColumns.append( self )
		else:
		for subcolumn in self.subcolumns:
		subcolumn.pickLeafColumns( leafColumns )

		def writeFormating( self, htmlFile, line ):
		self.formating.write( htmlFile, line )
		def getFormating( self, value ):
		formating = columnFormating(self.attributes)
		return formating.get( value )

		def processSorting( self ):
		self.formating.processSorting()

		def __repr__(self):
		return "<tableColumn(label={}, subcolumns={})>".format(self.label, [col.label for col in self.subcolumns])



		class logToHtmlConvertor:

		def __init__(self):
		self.tableColumns = []
		self.html = ""
		self.reset()

		def reset(self):
		self.metadata = {}
		self.maxLevel = 0
		self.leafColumns = []
		self.tableColumns = collections.OrderedDict()
		self.tableRows = []

		def processFile( self, logFileName, htmlFileName ):
		print "Processing file", logFileName
		print "Writing output to", htmlFileName
		# init HTML text
		self.writeHtmlHeader()

		print("Processing file", logFileName)
		logFile = open( logFileName, 'r' )
		self.readFile(logFile)
		logFile.close()

		self.writeHtmlFooter()
		print("Writing output to", htmlFileName)
		htmlFile = open( htmlFileName, 'w' )
		self.writeHtmlHeader( htmlFile, logFile )
		htmlFile.write(self.html)
		htmlFile.close()
		logFile.close()

		def initColumnsStructure( self, logFile ):
		for line in logFile:
		if line[ 0 ] != '#':
		return
		data = line[1:]
		level = len( data ) - len( data.lstrip(' ') )
		level = level + 1
		label = data.lstrip( ' ')
		label = label.rstrip( '\n' )
		self.reset()
		self.html = ""

		def readFile( self, logFile ):
		# read file by lines
		lines = logFile.readlines()

		# drop comments and blank lines
		lines = [line for line in lines if line.strip() and not line.startswith("#")]

		# drop anything before the first metadata block
		while len(lines) > 0 and not lines[0].startswith(":"):
		lines.pop(0)

		while len(lines) > 0:
		self.reset()
		metadata = []
		while len(lines) > 0 and lines[0].startswith(":"):
		metadata.append(lines.pop(0))
		self.parseMetadata(metadata)

		table = []
		while len(lines) > 0 and not lines[0].startswith(":"):
		table.append(lines.pop(0))
		self.parseTable(table)

		self.writeTable()

		def parseMetadata(self, lines):
		for line in lines:
		line = line[1:]
		key, value = line.split("=", 1)
		self.metadata[key.strip()] = value.strip()

		def parseTable(self, lines):
		header = []
		body = []
		while len(lines) > 0:
		while len(lines) > 0 and lines[0].startswith("!"):
		header.append(lines.pop(0))
		while len(lines) > 0 and not lines[0].startswith("!"):
		body.append(lines.pop(0))
		self.parseTableRow(header, body)
		header = []
		body = []

		def parseTableRow(self, header, body):
		columns = []
		for line in header:
		data = line.lstrip("!")
		level = len(line) - len(data)
		label = data.strip()
		#print " Inserting column on level ", level, " and label ", label
		if level > self.maxLevel:
		self.maxLevel = level;
		if level == 1:
		self.tableColumns.append( tableColumn( 1, label ) )
		columns.append( tableColumn( 1, label ) )
		if level > 1:
		self.tableColumns[ len( self.tableColumns ) - 1 ].insertSubcolumn( level, label )
		columns[ -1 ].insertSubcolumn( level, label )

		# merge columns of this block with the previously parsed columns
		self.mergeColumns(columns)

		# pick leaf columns (data will be added here)
		leafColumns = self.pickLeafColumns(columns)

		# elements of the table row corresponding to the header just parsed
		elements = [line.strip() for line in body]

		if len(elements) != len(leafColumns):
		raise Exception("Error in the table format: header has {} leaf columns, but the corresponding row has {} elements.".format(len(leafColumns), len(row)))

		row = collections.OrderedDict()
		for element, column in zip(elements, leafColumns):
		path = tuple(column.path)
		row[path] = element
		self.tableRows.append(row)

		def pickLeafColumns(self, columns):
		leafColumns = []
		for column in columns:
		column.pickLeafColumns(leafColumns)
		return leafColumns

		def mergeColumns(self, columns):
		for col in columns:
		path = tuple(col.path)
		if path in self.tableColumns:
		# merge all column attributes
		self.tableColumns[path].attributes += " " + col.attributes
		# merge new subcolumns
		currentSubPaths = [tuple(col.path) for col in self.tableColumns[path].subcolumns]
		for subcol in col.subcolumns:
		if tuple(subcol.path) not in currentSubPaths:
		self.tableColumns[path].subcolumns.append(subcol)
		else:
		self.tableColumns[path] = col
		self.mergeColumns(col.subcolumns)

		def mergeRows(self):
		# sort table
		self.tableRows.sort(key=lambda row: list(row.values()))

		i = 0
		while i < len(self.tableRows) - 1:
		currentRow = self.tableRows[ i ]
		nextRow = self.tableRows[ i + 1 ]

		can_merge = True
		for key, value in nextRow.items():
		if key in currentRow and currentRow[key] != value:
		can_merge = False
		break
		if can_merge is True:
		currentRow.update(nextRow)
		self.tableRows.pop(i + 1)
		else:
		i += 1

		# TODO: check this
		# sort again (just in case, previous sorting might compare values from
		# different columns)
		self.tableRows.sort(key=lambda row: list(row.values()))

		def countSubcolumns( self ):
		for subcolumn in self.tableColumns:
		subcolumn.countSubcolumns();
		for path, col in self.tableColumns.items():
		if len(path) == 1:
		col.countSubcolumns();

		def countHeight( self ):
		for subcolumn in self.tableColumns:
		subcolumn.countHeight();
		for path, col in self.tableColumns.items():
		if len(path) == 1:
		col.countHeight();

		def countRowspan( self ):
		for subcolumn in self.tableColumns:
		subcolumn.countRowspan( self.maxLevel )
		for path, col in self.tableColumns.items():
		if len(path) == 1:
		col.countRowspan( self.maxLevel )

		def recomputeLevel( self ):
		for subcolumn in self.tableColumns:
		subcolumn.recomputeLevel( 1 )

		def writeColumnsHeader( self, htmlFile ):
		level = 1
		while level <= self.maxLevel:
		#print "Writing columns on level ", level
		htmlFile.write( " <tr>\n")
		for column in self.tableColumns:
		column.writeToColumnsHeader( htmlFile, self.maxLevel, level )
		htmlFile.write( " </tr>\n")
		level = level + 1

		def pickLeafColumns( self ):
		for subcolumn in self.tableColumns:
		subcolumn.pickLeafColumns( self.leafColumns )

		def writeTable( self, logFile, htmlFile ):
		firstLine = "true"
		for line in logFile:
		if len( line ) == 1 or firstLine == "true":
		if firstLine == "true":
		htmlFile.write( " <tr>\n" )
		leafColumnsPointer = 0
		firstLine = "false"
		else:
		htmlFile.write( " </tr>\n" )
		htmlFile.write( " <tr>\n" )
		leafColumnsPointer = 0
		if len( line ) > 1:
		line = line.lstrip( ' ' )
		line = line.rstrip( '\n' )
		leafColumn = self.leafColumns[ leafColumnsPointer ]
		htmlFile.write( " <td" )
		leafColumn.writeFormating( htmlFile, line )
		htmlFile.write( ">")
		htmlFile.write( line )
		htmlFile.write( "</td>\n" )
		leafColumnsPointer = leafColumnsPointer + 1
		htmlFile.write( " </tr>\n" )
		for path, col in self.tableColumns.items():
		if len(path) == 1:
		col.recomputeLevel( 1 )

		def processSorting(self):
		for column in self.leafColumns:
		column.processSorting()

		def writeHtmlHeader( self, htmlFile, logFile ):
		htmlFile.write( "<html>\n" )
		htmlFile.write( " <body>\n" )
		htmlFile.write( " <table border=1>\n")
		self.initColumnsStructure( logFile )
		for path, col in self.tableColumns.items():
		if len(path) == 1:
		col.processSorting()

		def writeTable(self):
		self.mergeRows()
		self.countSubcolumns()
		self.countHeight()
		self.countRowspan()
		self.recomputeLevel()
		self.writeColumnsHeader( htmlFile )
		self.pickLeafColumns();
		self.writeTable( logFile, htmlFile )
		htmlFile.write( " </table>\n")
		htmlFile.write( " </body>\n" )
		htmlFile.write( "</html>\n" )
		self.processSorting()
		# self.processSorting()

		# write metadata
		self.writeMetadata()

		self.html += "<table border=1>\n"

		# write header
		self.writeColumnsHeader()

		# write data
		firstLevelColumns = [column for path, column in self.tableColumns.items() if len(path) == 1]
		leafColumns = self.pickLeafColumns(firstLevelColumns)
		for row in self.tableRows:
		self.html += " <tr>\n"
		# walk through leafColumns to ensure correct order
		for col in leafColumns:
		path = tuple(col.path)
		if path in row:
		value = row[path]
		formating = col.getFormating(value)
		self.html += " <td {}>{}</td>\n".format(formating, value)
		else:
		self.html += " <td></td>\n"
		self.html += " </tr>\n"

		self.html += "</table>\n"

		def writeMetadata(self):
		self.html += "<h2>{}</h2>\n".format(self.metadata.get("title"))
		self.html += "<table border=1>\n"
		self.html += "<tbody>\n"
		for key in sorted(self.metadata.keys()):
		self.html += " <tr><td>{}</td><td>{}</td></tr>\n".format(key, self.metadata[key])
		self.html += "</tbody>\n"
		self.html += "</table>\n"

		def writeColumnsHeader(self):
		level = 1
		while level <= self.maxLevel:
		self.html += " <tr>\n"
		for path, column in self.tableColumns.items():
		self.html += column.getColumnHeader( level )
		self.html += " </tr>\n"
		level += 1

		def writeHtmlHeader(self):
		self.html += "<html>\n"
		self.html += "<body>\n"

		def writeHtmlFooter(self):
		self.html += "</body>\n"
		self.html += "</html>\n"