Put report summary into separate script scripts/create_histogram_summary.py (407fed13) · Commits · IVAS Codec Public Collaboration / IVAS Codec

scripts/create_histogram_summary.py

0 → 100644

+93 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import argparse
		import math
		import numpy as np
		import matplotlib.pyplot as plt
		import csv
		from parse_xml_report import FORMATS, CATEGORIES

		"""
		Parses a CSV report and creates a summary report.
		"""



		# Main routine
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		description="Parses a CSV report and creates a summary report."
		)
		parser.add_argument(
		"csv_report",
		type=str,
		help="CSV report file of test cases, e.g. report.csv",
		)
		parser.add_argument(
		"csv_summary", type=str, help="Output CSV file, e.g. summary.csv"
		)
		parser.add_argument(
		"csv_image",
		type=str,
		nargs='?',
		help="Summary image file, e.g. summary.png",
		default = None
		)
		args = parser.parse_args()
		csv_report = args.csv_report
		csv_summary = args.csv_summary
		csv_image = args.csv_image

		limits = [0, 5, 10, math.inf]

		# Load CSV report
		results_sorted = {}
		with open(csv_report,'r') as fp:
		reader = csv.reader(fp, delimiter=';')
		header = next(reader)
		keys = header[1:-1]
		for row in reader:
		testcase = row[0]
		results_sorted[testcase] = {}
		for k, val in zip(keys, row[1:-1]):
		results_sorted[testcase][k] = val

		# Output CSV file
		with open(csv_summary, "w") as fp:
		limits_labels = ["0"] + [f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"]
		headerline = f"Format;Category;" + ";".join(limits_labels) + "\n"
		fp.write(headerline)

		subplot = 0
		fig, ax = plt.subplots(len(FORMATS)//2, 2)
		for fmt in FORMATS:
		bottom = np.zeros(len(limits)+1)
		for cat in CATEGORIES:
		mld_vals = [x for x in [m["MLD"] for m in results_sorted.values() if m["Format"] == fmt and m["Category"] == cat]]
		mld = [float(x) for x in mld_vals if x != 'None' and x != '0']
		zero = sum([1 for x in mld_vals if x == '0'])
		none = sum([1 for x in mld_vals if x == 'None'])
		hist, _ = np.histogram(mld,limits)
		data = np.array([zero] + list(hist) + [none])

		# CSV output
		line = f"{fmt};{cat};{'; '.join(map(str,data))}\n"
		fp.write(line)

		# Matplotlib histogram
		a = subplot % 4
		b = subplot // 4
		ax[a][b].bar(limits_labels, data, 0.5, label=cat, bottom=bottom)
		bottom += data

		# Histogram layout
		ax[a][b].set_title(fmt)
		ax[a][b].legend(loc="best")
		ax[a][b].set_xlabel("MLD")
		ax[a][b].set_ylabel("Number of test cases")
		subplot += 1

		fig.set_figheight(20)
		fig.set_figwidth(16)
		if csv_image:
		plt.savefig(csv_image)

scripts/parse_xml_report.py

+16 −65

Original line number	Diff line number	Diff line
		@@ -13,41 +13,39 @@ Parse a junit report and create a summary report.

		PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"]

		FORMATS = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel',
		'OSBA':r'OSBA', 'OMASA':r'OMASA','MASA':r'MASA','SBA':r'SBA', 'Renderer':r'renderer'}

		CATEGORIES = {'Normal operation':r'.*', 'DTX':r'DTX', 'PLC':r'%', 'Bitrate switching':r'br sw\|bitrate switching', 'JBM':r'JBM' }

		# Main routine
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		description="Parse a junit report and create a MLD summary report."
		description="Parse a junit report and create an MLD summary report."
		)
		parser.add_argument(
		"xml_report",
		type=str,
		help="XML junit report input file, e.g. report-junit.xml",
		)
		parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. mld.csv")
		parser.add_argument("summary_file", type=str, nargs='?', help="Summary CSV file, e.g. summary.csv", default = None)
		parser.add_argument(
		"csv_file",
		type=str,
		help="Output CSV file, e.g. report.csv"
		)
		args = parser.parse_args()
		xml_report = args.xml_report
		csv_file = args.csv_file
		summary_file = args.summary_file

		tree = ElementTree.parse(xml_report)

		testsuite = tree.find(".//testsuite")


		testcases = tree.findall(".//testcase")

		results_unsorted = {}
		count = {'PASS':0,'FAIL':0,'ERROR':0}

		# Formats, categories and MLD histogram limits
		# formats = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel',
		# 'MASA':r'MASA','SBA':r'SBA', 'OSBA':r'OSBA', 'OMASA':r'OMASA','Renderer':r'renderer'}
		formats = {'Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel',
		'OSBA':r'OSBA', 'OMASA':r'OMASA','MASA':r'MASA','SBA':r'SBA', 'Renderer':r'renderer'}
		categories = {'Normal operation':r'.*', 'DTX':r'DTX', 'PLC':r'%', 'Bitrate switching':r'br sw\|bitrate switching', 'JBM':r'JBM' }
		limits = [0,5,10,20,math.inf]


		for testcase in testcases:
		if testcase.find(".//skipped") is None:
		@@ -76,21 +74,21 @@ if __name__ == "__main__":

		# Identify format and category (mode of operation)
		# For the format, favor the earliest match in the test case name
		fmt = min([(f, re.search(formats[f], fulltestname, re.IGNORECASE).end()) for f in formats if re.search(formats[f], fulltestname, re.IGNORECASE)] , key=lambda x: x[1])[0]
		cat = [c for c in categories if re.search(categories[c], fulltestname, re.IGNORECASE)][-1]
		fmt = min([(f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) for f in FORMATS if re.search(FORMATS[f], fulltestname, re.IGNORECASE)] , key=lambda x: x[1])[0]

		# Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked.
		cat = [c for c in CATEGORIES if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE)][-1]

		# For ERROR cases, both a FAIL and an ERROR result is generated.
		# Here, a FAIL would be overwritten with an ERROR result since it has the same name.
		# Todo: Change to fielded dict for all entries to eliminate confusing literal indices
		#results_unsorted[fulltestname] = (fmt, cat, testresult, properties_values)
		results_unsorted[fulltestname] = {'Format':fmt, 'Category':cat, 'Result':testresult}
		for propertyname, propertyvalue in zip(PROPERTIES, properties_values):
		results_unsorted[fulltestname][propertyname] = propertyvalue

		results_sorted = dict(sorted(results_unsorted.items()))

		keys = list(results_sorted[list(results_unsorted.keys())[0]].keys())

		# Write CSV file
		with open(csv_file, "w") as outfile:
		headerline = ";".join(["testcase"] + keys) + "\n"
		outfile.write(headerline)
		@@ -99,53 +97,6 @@ if __name__ == "__main__":
		line = ";".join([test] + list(results_sorted[test].values())) + "\n"
		outfile.write(line)

		tmp = {}
		for fmt in formats:
		tmp[fmt] = {}
		for cat in categories:
		tmp[fmt][cat] = []

		if summary_file is not None:
		with open(summary_file, "w") as fp:
		for testcase in results_sorted:
		fmt = results_sorted[testcase]["Format"]
		cat = results_sorted[testcase]["Category"]
		tmp[fmt][cat].append(results_sorted[testcase]["MLD"]) # Add MLD score to list

		subplot = 0
		fig, ax = plt.subplots(len(formats)//2, 2)
		for fmt in tmp:
		limits_labels = ["0"] + [f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])] + ["None"]
		headerline = f"{fmt};\nCategory;" + ";".join(limits_labels) + "\n"
		fp.write(headerline)
		bottom = np.zeros(len(limits)+1)
		for cat in categories:
		# CSV output
		# Separate 0 and None as special cases
		mld = [float(x) for x in tmp[fmt][cat] if x != 'None' and x != '0']
		zero = sum([1 for x in tmp[fmt][cat] if x == '0'])
		none = sum([1 for x in tmp[fmt][cat] if x == 'None'])
		hist, _ = np.histogram(mld,limits)
		data = np.array([zero] + list(hist) + [none])
		line = f"{cat}; {'; '.join(map(str,data))}\n"
		fp.write(line)

		# Matplotlib histogram
		a = subplot % 4
		b = subplot // 4
		ax[a][b].bar(limits_labels, data, 0.5, label=cat, bottom=bottom)
		bottom += data
		fp.write("\n")

		ax[a][b].set_title(fmt)
		ax[a][b].legend(loc="best")
		ax[a][b].set_xlabel("MLD")
		ax[a][b].set_ylabel("Number of test cases")
		subplot += 1
		fig.set_figheight(20)
		fig.set_figwidth(16)
		plt.savefig(summary_file +".png")

		print(
		f"Parsed testsuite with {count['PASS']+count['FAIL']+count['ERROR']} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors."
		)