rewrite parse_xml_report.py (5089e568) · Commits · IVAS Codec Public Collaboration / IVAS Codec

scripts/parse_xml_report.py

+53 −185

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import argparse
		import re
		import pandas as pd
		from xml.etree import ElementTree
		from collections import Counter

		"""
		Parse a junit report and create a summary report.
		"""

		PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"]

		IVAS_FORMATS = {
		"Stereo": r"stereo",
		"ISM": r"ISM",
		"Multichannel": r"Multi-channel\|MC",
		"MASA": r"(?<!O)MASA",
		"SBA": r"(?<!O)SBA",
		"OSBA": r"OSBA",
		"OMASA": r"OMASA",
		"Renderer": r"renderer",
		}

		EVS_FORMATS = {
		"AMRWBIO_dec": r"Readme_AMRWB_IO_dec",
		"AMRWBIO_enc": r"Readme_AMRWB_IO_enc",
		"EVS_dec": r"Readme_EVS_dec",
		"EVS_enc": r"Readme_EVS_enc",
		"EVS_JBM_dec": r"Readme_JBM_dec",
		}

		NO_FORMATS = {"Default": r".*"}

		IVAS_CATEGORIES = {
		"Normal operation": r".*",
		"DTX": r"DTX",
		"PLC": r"%",
		"Bitrate switching": r"br sw\|bitrate switching",
		"JBM": r"JBM",
		}

		EVS_CATEGORIES = {
		"Normal operation": r".*",
		"DTX": r"DTX",
		"PLC": r"b10\|f06\|EPF",
		"Bitrate switching": r"sw",
		"JBM": r"JBM",
		}

		NO_CATEGORIES = {"N/A": r".*"}

		def xml_to_dataframe(xml_report: str) -> pd.DataFrame:
		tree = ElementTree.parse(xml_report)
		root = tree.getroot()

		def get_format_from_fulltestname(fulltestname: str) -> str:
		# For the format, favor the earliest match in the test case name
		fmt = min(
		[
		(f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end())
		for f in FORMATS
		if re.search(FORMATS[f], fulltestname, re.IGNORECASE) is not None
		],
		key=lambda x: x[1],
		)[0]
		return fmt
		testcases = root[0].findall("testcase")
		testcases = [tc for tc in testcases if tc.find("skipped") is None]

		parsed_testcases = [parse_testcase(tc) for tc in testcases]
		testcase_df = pd.DataFrame(parsed_testcases)

		def get_category_from_fulltestname(fulltestname: str) -> str:
		cat = [
		c for c in CATEGORIES if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE)
		][-1]
		return cat
		return testcase_df


		def get_testresult(testcase: ElementTree.Element) -> str:
		def get_result_from_testcase(testcase: ElementTree.Element) -> str:
		if testcase.find("failure") is not None:
		testresult = "FAIL"
		elif testcase.find("error") is not None:
		@@ -81,130 +30,49 @@ def get_testresult(testcase: ElementTree.Element) -> str:
		return testresult


		# Main routine
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		description="Parse a junit report and create an MLD summary report."
		)
		parser.add_argument(
		"xml_report",
		type=str,
		help="XML junit report input file, e.g. report-junit.xml",
		)
		parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. report.csv")
		parser.add_argument(
		"--evs",
		action="store_true",
		help="Parse using EVS 26.444 formats",
		)
		parser.add_argument(
		"--clipping",
		action="store_true",
		help="Extract clipping information. Available if encoder has been run with DEBUGGING active.",
		)
		parser.add_argument(
		"--delta_odg",
		action="store_true",
		help="Extract Delta ODG information.",
		)
		parser.add_argument(
		"--skip_formats",
		action="store_true",
		help="Parse without formats and categories. Suitable for general tests which do not match the IVAS categories.",
		)
		args = parser.parse_args()
		xml_report = args.xml_report
		csv_file = args.csv_file
		FORMATS = IVAS_FORMATS
		CATEGORIES = IVAS_CATEGORIES
		if args.evs:
		FORMATS = EVS_FORMATS
		CATEGORIES = EVS_CATEGORIES
		else:
		FORMATS = IVAS_FORMATS
		CATEGORIES = IVAS_CATEGORIES
		if args.clipping:
		PROPERTIES += ["ENC_CORE_OVL", "MAX_OVL", "MIN_OVL"]
		if args.delta_odg:
		PROPERTIES += ["DELTA_ODG"]
		if args.skip_formats:
		FORMATS = NO_FORMATS
		CATEGORIES = NO_CATEGORIES
		def parse_testcase(testcase) -> dict:
		"""
		Get all properties + name for a testcase
		"""
		ret = {}

		tree = ElementTree.parse(xml_report)
		filename = testcase.get("file", testcase.get("classname").replace(".", "/") + ".py")
		fulltestname = filename + "::" + testcase.get("name")
		ret["testcase"] = fulltestname

		result = get_result_from_testcase(testcase)
		ret["result"] = result

		properties = {
		p.get("name"): p.get("value") for p in testcase.findall(".//property")
		}
		ret.update(properties)

		return ret

		testsuite = tree.find(".//testsuite")
		testcases = tree.findall(".//testcase")

		# Prepare result structure
		results = {}
		for fmt in FORMATS:
		results[fmt] = {}
		for cat in CATEGORIES:
		results[fmt][cat] = {}
		count = {"PASS": 0, "FAIL": 0, "ERROR": 0}
		def main(xml_report, csv_file):
		df = xml_to_dataframe(xml_report)
		df.to_csv(csv_file)

		# filter out skipped testcases
		testcases = [tc for tc in testcases if tc.find(".//skipped") is None]
		n_testcases = len(df)
		count = Counter(df["result"])

		for testcase in testcases:
		filename = testcase.get(
		"file", testcase.get("classname").replace(".", "/") + ".py"
		print(
		f"Parsed testsuite with {n_testcases} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors."
		)
		fulltestname = filename + "::" + testcase.get("name")

		# only include the properties listed above
		# we need to find all occurences with any suffixes to also handle the split-comparison
		# runs correctly
		properties_found = {
		p.get("name"): p.get("value")
		for p in testcase.findall(".//property")
		if "CHANNEL" not in p.get("name")
		and any(p_listed in p.get("name") for p_listed in PROPERTIES)
		}

		# Identify format and category (mode of operation)
		# For the format, favor the earliest match in the test case name
		fmt = get_format_from_fulltestname(fulltestname)
		# Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked.
		cat = get_category_from_fulltestname(fulltestname)

		testresult = get_testresult(testcase)

		# get all present suffixes
		pattern = re.compile("\|".join(PROPERTIES))
		suffixes = set(pattern.sub("", p) for p in properties_found)

		# record the result for all suffixes
		# For ERROR cases, both a FAIL and an ERROR result is generated.
		# Here, a FAIL would be overwritten with an ERROR result since it has the same name.
		for s in suffixes:
		fulltestname_suffix = f"{fulltestname}{s}"
		results[fmt][cat][fulltestname_suffix] = {"Result": testresult}
		for propertyname in PROPERTIES:
		results[fmt][cat][fulltestname_suffix][propertyname] = properties_found[
		f"{propertyname}{s}"
		]
		count[testresult] += 1

		header = ["testcase", "Format", "Category", "Result"] + PROPERTIES

		# Write CSV file
		with open(csv_file, "w") as outfile:
		headerline = ";".join(header) + "\n"
		outfile.write(headerline)
		for fmt in FORMATS:
		for cat in CATEGORIES:
		results[fmt][cat] = dict(sorted(results[fmt][cat].items()))
		for test in results[fmt][cat]:
		line = (
		";".join(
		[test, fmt, cat] + list(results[fmt][cat][test].values())
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		description="Parse junit report from IVAS pytest suite and convert to csv file"
		)
		+ "\n"
		parser.add_argument(
		"xml_report",
		type=str,
		help="XML junit report input file, e.g. report-junit.xml",
		)
		outfile.write(line)
		parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. report.csv")

		print(
		f"Parsed testsuite with {count['PASS']+count['FAIL']+count['ERROR']} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors."
		)
		args = parser.parse_args()
		main(args.xml_report, args.csv_file)