Merge branch 'basop-ci/detailed-report-summary' into 'main' (af1ba8b2) · Commits · IVAS Codec Public Collaboration / IVAS Codec

ci/basop-pages/create_report_pages.py

+55 −5

Original line number	Diff line number	Diff line
		@@ -3,6 +3,7 @@ import pathlib
		import argparse
		from functools import partial

		FORMATS = ["Stereo", "ISM", "Multichannel", "MASA", "SBA", "OSBA", "OMASA", "Renderer"]

		CSV_DELIM = ";"
		SUBPAGE_TMPL_CSS = """
		@@ -25,7 +26,7 @@ SUBPAGE_TMPL_CSS = """

		SUBPAGE_TMPL_HTML = """

		<h1>Report for job {job_name}</h1
		<h1>Report for job {job_name}</h1>

		Comparing:
		<ul>
		@@ -34,6 +35,13 @@ Comparing:
		<li><a href="{job_name}--merged_csv--{id_current}.csv">Merged csv data</a></li>
		</ul>

		{images_mld}

		{images_diff}

		{images_ssnr}


		<br>
		<b>How is the table sorted?</b>
		<ul>
		@@ -92,9 +100,17 @@ ARROW_DOWN = '<span class="arrowdown">⬂</span>'

		# expected columns. actual columns are filtered from the incoming data later, this
		# is mainly for controlling the order in the output table
		COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"]
		COLUMNS = [
		"testcase",
		"Format",
		"Category",
		"Result",
		"MLD",
		"MAXIMUM ABS DIFF",
		"MIN_SSNR",
		]
		COLUMNS_GLOBAL = COLUMNS[:1]
		COLUMNS_DIFFERENTIAL = COLUMNS[1:]
		COLUMNS_DIFFERENTIAL = COLUMNS[3:]
		COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:]


		@@ -106,6 +122,7 @@ def create_subpage(
		id_current: int,
		id_previous: int,
		job_name: str,
		histogram,
		):
		merged_reports = merge_and_cleanup_mld_reports(
		csv_current, csv_previous, id_current, id_previous
		@@ -128,6 +145,33 @@ def create_subpage(
		table_body = "\n".join(
		tr_from_row(row, id_current, id_previous) for row in merged_reports
		)
		if histogram:
		images_mld = (
		f"<h2>MLD summary {job_name}</h2>\n"
		+ " ".join(
		[f"<img src=images/summary_{id_current}_MLD_{x}.png>" for x in FORMATS]
		)
		+ f'\n<br><a href="images/summary_{id_current}_MLD.csv">summary_{id_current}_MLD.csv</a>'
		)
		images_ssnr = (
		f"<h2>MIN_SSNR summary {job_name}</h2>\n"
		+ " ".join(
		[f"<img src=images/summary_{id_current}_SSNR_{x}.png>" for x in FORMATS]
		)
		+ f'\n<br><a href="images/summary_{id_current}_SSNR.csv">summary_{id_current}_SSNR.csv</a>'
		)
		images_diff = (
		f"<h2>MAX ABS DIFFERENCE summary {job_name}</h2>\n"
		+ " ".join(
		[f"<img src=images/summary_{id_current}_DIFF_{x}.png>" for x in FORMATS]
		)
		+ f'\n<br><a href="images/summary_{id_current}_DIFF.csv">summary_{id_current}_DIFF.csv</a>'
		)
		else:
		images_mld = ""
		images_ssnr = ""
		images_diff = ""

		new_subpage = SUBPAGE_TMPL_CSS + SUBPAGE_TMPL_HTML.format(
		id_current=id_current,
		id_previous=id_previous,
		@@ -135,6 +179,9 @@ def create_subpage(
		job_name=job_name,
		table_header_a=table_header_a,
		table_header_b=table_header_b,
		images_mld=images_mld,
		images_ssnr=images_ssnr,
		images_diff=images_diff,
		)
		with open(html_out, "w") as f:
		f.write(new_subpage)
		@@ -226,8 +273,9 @@ def merge_and_cleanup_mld_reports(
		are uninteresting and are put last.
		"""
		try:
		float(x[mld_col_curr])
		float(x[mld_col_prev])
		cols = [mld_col_curr, mld_col_prev] + [p[1] for p in other_col_pairs]
		for c in cols:
		float(x[c])
		except ValueError:
		# Value is no valid floating point value
		return float("inf")
		@@ -293,6 +341,7 @@ if __name__ == "__main__":
		parser.add_argument("id_current", type=int)
		parser.add_argument("id_previous", type=int)
		parser.add_argument("job_name")
		parser.add_argument("--histogram", action="store_true")
		args = parser.parse_args()

		create_subpage(
		@@ -303,4 +352,5 @@ if __name__ == "__main__":
		args.id_current,
		args.id_previous,
		args.job_name,
		args.histogram,
		)

scripts/create_histogram_summary.py

0 → 100644

+127 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import argparse
		import math
		import numpy as np
		# These next three lines are added as a precaution in case the gitlab runner
		# needs DISPLAY to render the plots, even if they are written to file.
		import matplotlib
		matplotlib.use("Agg")
		import matplotlib.pyplot as plt
		import csv
		import os
		from parse_xml_report import FORMATS, CATEGORIES

		"""
		Parses a CSV report and creates a summary report.
		"""


		# Main routine
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		description="Parses a CSV report and creates a summary report."
		)
		parser.add_argument(
		"csv_report",
		type=str,
		help="CSV report file of test cases, e.g. report.csv",
		)
		parser.add_argument(
		"csv_summary", type=str, help="Output CSV file, e.g. summary.csv"
		)
		parser.add_argument(
		"csv_image",
		type=str,
		nargs="?",
		help="Summary image file, e.g. summary.png",
		default=None,
		)
		parser.add_argument(
		"--measure",
		type=str,
		nargs=1,
		help="Measure, any of: MLD, DIFF, SSNR, default: MLD",
		default=["MLD"],
		)
		args = parser.parse_args()
		csv_report = args.csv_report
		csv_summary = args.csv_summary
		csv_image = args.csv_image
		measure = args.measure[0]

		limits_per_measure = {
		"MLD": ("MLD", [0, 5, 10, math.inf]),
		"DIFF": ("MAXIMUM ABS DIFF", [0, 1024, 16384, 32769]),
		"SSNR": ("MIN_SSNR", [-math.inf, 0, 20, 40, 60, 100]),
		}
		(measure_label, limits) = limits_per_measure[measure]

		# Load CSV report
		results_sorted = {}
		with open(csv_report, "r") as fp:
		reader = csv.reader(fp, delimiter=";")
		header = next(reader)
		keys = header[1:]
		for row in reader:
		testcase = row[0]
		results_sorted[testcase] = {}
		for k, val in zip(keys, row[1:]):
		results_sorted[testcase][k] = val

		# Output CSV file
		with open(csv_summary, "w") as fp:
		limits_labels = [
		f"{str(a)} --\n {str(b)}" for (a, b) in zip(limits[0:-1], limits[1:])
		] + ["None"]
		# Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF
		if measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF":
		limits_labels = ["0"] + limits_labels
		headerline = f"Format;Category;" + ";".join(limits_labels) + "\n"
		fp.write(headerline)

		for fmt in FORMATS:
		fig, ax = plt.subplots()
		bottom = np.zeros(len(limits_labels))
		for cat in CATEGORIES:
		values = [
		x
		for x in [
		m[measure_label]
		for m in results_sorted.values()
		if m["Format"] == fmt and m["Category"] == cat
		]
		]
		# Zero difference is treated as a special category for MLD and MAXIMUM ABS DIFF
		if measure_label == "MLD" or measure_label == "MAXIMUM ABS DIFF":
		val = [
		float(x) for x in values if x != "None" and x != "0" and x != ""
		]
		zero = [sum([1 for x in values if x == "0"])]
		none = [sum([1 for x in values if x == "None" or x == ""])]
		else:
		val = [float(x) for x in values if x != "None" and x != ""]
		zero = []
		none = [sum([1 for x in values if x == "None" or x == ""])]
		hist, _ = np.histogram(val, limits)
		data = np.array(zero + list(hist) + none)

		# CSV output
		line = f"{fmt};{cat};{'; '.join(map(str,data))}\n"
		fp.write(line)

		# Matplotlib histogram
		ax.bar(limits_labels, data, 0.5, label=cat, bottom=bottom)
		bottom += data

		# Histogram layout
		ax.set_title(fmt)
		ax.legend(loc="best")
		ax.set_xlabel(measure_label)
		ax.set_ylabel("Number of test cases")

		fig.set_figheight(4)
		fig.set_figwidth(6)
		if csv_image:
		base, ext = os.path.splitext(csv_image)
		plt.savefig(f"{base}_{fmt}{ext}")

scripts/parse_xml_report.py

+72 −48

Original line number	Diff line number	Diff line
		@@ -12,33 +12,52 @@ Parse a junit report and create a summary report.

		PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"]

		FORMATS = {
		"Stereo": r"stereo",
		"ISM": r"ISM",
		"Multichannel": r"Multi-channel",
		"MASA": r"(?<!O)MASA",
		"SBA": r"(?<!O)SBA",
		"OSBA": r"OSBA",
		"OMASA": r"OMASA",
		"Renderer": r"renderer",
		}

		CATEGORIES = {
		"Normal operation": r".*",
		"DTX": r"DTX",
		"PLC": r"%",
		"Bitrate switching": r"br sw\|bitrate switching",
		"JBM": r"JBM",
		}

		# Main routine
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		description="Parse a junit report and create a MLD summary report."
		description="Parse a junit report and create an MLD summary report."
		)
		parser.add_argument(
		"xml_report",
		type=str,
		help="XML junit report input file, e.g. report-junit.xml",
		)
		parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. mld.csv")
		parser.add_argument("summary_file", type=str, nargs='?', help="Summary CSV file, e.g. summary.csv", default = None)
		parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. report.csv")
		args = parser.parse_args()
		xml_report = args.xml_report
		csv_file = args.csv_file
		summary_file = args.summary_file

		tree = ElementTree.parse(xml_report)

		testsuite = tree.find(".//testsuite")


		testcases = tree.findall(".//testcase")

		results_unsorted = {}
		count = {'PASS':0,'FAIL':0,'ERROR':0}
		# Prepare result structure
		results = {}
		for fmt in FORMATS:
		results[fmt] = {}
		for cat in CATEGORIES:
		results[fmt][cat] = {}
		count = {"PASS": 0, "FAIL": 0, "ERROR": 0}

		for testcase in testcases:
		if testcase.find(".//skipped") is None:
		@@ -52,56 +71,61 @@ if __name__ == "__main__":
		fulltestname = testcase.get("file") + "::" + testcase.get("name")

		properties_found = {
		p.get("name"): p.get("value")
		for p in testcase.findall(".//property")
		p.get("name"): p.get("value") for p in testcase.findall(".//property")
		}

		if testcase.find('failure') is not None:
		testresult = 'FAIL'
		elif testcase.find('error') is not None:
		testresult = 'ERROR'
		if testcase.find("failure") is not None:
		testresult = "FAIL"
		elif testcase.find("error") is not None:
		testresult = "ERROR"
		else:
		testresult = 'PASS'
		testresult = "PASS"

		properties_values = [str(properties_found.get(p)) for p in PROPERTIES]

		# Identify format and category (mode of operation)
		# For the format, favor the earliest match in the test case name
		fmt = min(
		[
		(f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end())
		for f in FORMATS
		if re.search(FORMATS[f], fulltestname, re.IGNORECASE)
		],
		key=lambda x: x[1],
		)[0]

		# Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked.
		cat = [
		c
		for c in CATEGORIES
		if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE)
		][-1]

		# For ERROR cases, both a FAIL and an ERROR result is generated.
		# Here, a FAIL would be overwritten with an ERROR result since it has the same name.
		results_unsorted[fulltestname] = (testresult, properties_values)
		results[fmt][cat][fulltestname] = {"Result": testresult}
		for propertyname, propertyvalue in zip(PROPERTIES, properties_values):
		results[fmt][cat][fulltestname][propertyname] = propertyvalue

		results_sorted = dict(sorted(results_unsorted.items()))
		header = ["testcase", "Format", "Category", "Result"] + PROPERTIES

		# Write CSV file
		with open(csv_file, "w") as outfile:
		headerline = ";".join(["testcase","Result"] + PROPERTIES) + "\n"
		headerline = ";".join(header) + "\n"
		outfile.write(headerline)
		for test in results_sorted:
		count[results_sorted[test][0]] += 1
		line = ";".join([test,results_sorted[test][0]] + results_sorted[test][1]) + "\n"
		for fmt in FORMATS:
		for cat in CATEGORIES:
		results[fmt][cat] = dict(sorted(results[fmt][cat].items()))
		for test in results[fmt][cat]:
		count[results[fmt][cat][test]["Result"]] += 1
		line = (
		";".join(
		[test, fmt, cat] + list(results[fmt][cat][test].values())
		)
		+ "\n"
		)
		outfile.write(line)

		categories = {'Normal operation':r'.*', 'JBM':r'JBM', 'PLC':r'%', 'Bitrate switching':r'br sw\|bitrate switching'}
		limits = [0,5,10,20,math.inf]
		tmp = {}

		if summary_file is not None:
		with open(summary_file, "w") as fp:
		for testcase in results_sorted:
		cat = [c for c in categories if re.search(categories[c],testcase)][-1]
		if cat in tmp:
		tmp[cat].append(results_sorted[testcase][1][0]) # Add MLD score to list
		else:
		tmp[cat] = [results_sorted[testcase][1][0]]

		headerline = "Category;0;" + ";".join([f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])]) + ";None\n"
		fp.write(headerline)
		for cat in tmp:
		# Separate 0 and None as special cases
		mld = [float(x) for x in tmp[cat] if x != 'None' and x != '0']
		zero = sum([1 for x in tmp[cat] if x == '0'])
		none = sum([1 for x in tmp[cat] if x == 'None'])
		hist, _ = np.histogram(mld,limits)
		line = f"{cat}; {str(zero)}; {'; '.join(map(str,hist))}; {str(none)}\n"
		fp.write(line)

		print(
		f"Parsed testsuite with {count['PASS']+count['FAIL']+count['ERROR']} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors."
		)