Merge branch 'ci/mld-and-abs-diff' into 'main' (544ee16d) · Commits · IVAS Codec Public Collaboration / IVAS Codec

scripts/parse_mld.py

deleted100644 → 0

+0 −32

Original line number	Diff line number	Diff line
		#!/usr/bin/python3

		import argparse
		import re

		# Main routine
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		description="Parse HTML report to extract MLD values"
		)
		parser.add_argument(
		"html_report", type=str, help="HTML report input file, e.g. report.html"
		)
		parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. output.csv")
		args = parser.parse_args()
		html_report = args.html_report
		csv_file = args.csv_file

		mld = {}

		with open(html_report, "r") as infile:
		for line in infile.readlines():
		if "col-name" in line:
		test_name = re.search("\[(.*)\]", line).group(1)
		mld[test_name] = 0.0
		if "MLD" in line:
		mld_val = float(line.split()[1])
		mld[test_name] = mld_val

		with open(csv_file, "w") as outfile:
		for test_name in mld:
		outfile.write(test_name + ";" + str(mld[test_name]) + "\n")

scripts/parse_mld_xml.py

+31 −18

Original line number	Diff line number	Diff line
		@@ -4,9 +4,12 @@ import argparse
		from xml.etree import ElementTree

		"""
		Parse a junit report and create a MLD summary report.
		Parse a junit report and create an MLD summary report.
		"""

		PROPERTIES = ["MLD", "MAXIMUM ABS DIFF"]


		# Main routine
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		@@ -31,10 +34,11 @@ if __name__ == "__main__":

		testcases = tree.findall(".//testcase")

		with open(csv_file, "w") as outfile:
		results_unsorted = {}

		for testcase in testcases:
		if testcase.find(".//skipped") == None:
		if testcase.get("file") == None:
		if testcase.find(".//skipped") is None:
		if testcase.get("file") is None:
		fulltestname = (
		testcase.get("classname").replace(".", "/")
		+ ".py::"
		@@ -42,10 +46,19 @@ if __name__ == "__main__":
		)
		else:
		fulltestname = testcase.get("file") + "::" + testcase.get("name")
		if testcase.find(".//property") == None:
		mld_val = None
		else:
		mld_val = testcase.find(".//property").get(
		"value"
		) # Currently MLD is the only set property. If more are added updates are needed here.
		outfile.write(fulltestname + ";" + str(mld_val) + "\n")

		properties_found = {
		p.get("name"): p.get("value")
		for p in testcase.findall(".//property")
		}
		properties_values = [str(properties_found.get(p)) for p in PROPERTIES]
		outline = ";".join([fulltestname] + properties_values) + "\n"
		results_unsorted[fulltestname] = outline

		results_sorted = dict(sorted(results_unsorted.items()))

		with open(csv_file, "w") as outfile:
		headerline = ";".join(["testcase"] + PROPERTIES) + "\n"
		outfile.write(headerline)
		for test in results_sorted:
		outfile.write(results_sorted[test])

tests/cmp_pcm.py

+20 −11

Original line number	Diff line number	Diff line
		@@ -49,21 +49,30 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str
		s1, s2, fs, per_frame=False, get_mld=get_mld
		)

		if cmp_result["bitexact"]:
		return 0, "SUCCESS: Files are bitexact"
		else:
		output_differs = 0
		reason = "SUCCESS: Files are bitexact"

		if not cmp_result["bitexact"]:
		diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)"
		first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)"
		print(diff_msg)
		print(first_msg)

		reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}"
		output_differs = 1

		if get_mld:
		mld_msg = f"MLD: {cmp_result['MLD']}"
		reason += " - " + mld_msg
		print(mld_msg)

		if cmp_result["MLD"] <= mld_lim:
		return 0, f"MLD: {cmp_result['MLD']} <= {mld_lim}"
		output_differs = 0
		reason += f" <= {mld_lim}"
		else:
		return 1, f"MLD: {cmp_result['MLD']} > {mld_lim}"
		return 1, "Non-BE"
		reason += f" > {mld_lim}"

		return output_differs, reason


		if __name__ == "__main__":

tests/codec_be_on_mr_nonselection/init.py

+2 −0

Original line number	Diff line number	Diff line
		MLD_PATTERN = r"MLD: ([\d\.]*)"
		MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)"

tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py

+18 −6

Original line number	Diff line number	Diff line
		@@ -35,6 +35,7 @@ __doc__ = """

		import errno
		import os
		import re
		from filecmp import cmp
		from typing import Optional

		@@ -43,6 +44,8 @@ import pytest
		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend

		from . import MLD_PATTERN, MAX_DIFF_PATTERN

		# params
		# output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT']
		output_mode_list = ["BINAURAL", "EXT"]
		@@ -216,11 +219,15 @@ def test_masa_enc_dec(
		mld_lim=get_mld_lim,
		)
		if get_mld:
		mld = 0
		if "MLD" in reason:
		mld = float(reason.split(":")[1].split()[0])
		mld = re.search(MLD_PATTERN, reason).groups(1)[0]
		record_property("MLD", mld)

		max_diff = 0
		if pcmcmp_res:
		search_result = re.search(MAX_DIFF_PATTERN, reason)
		max_diff = search_result.groups(1)[0]
		record_property("MAXIMUM ABS DIFF", max_diff)

		if get_mld and get_mld_lim > 0:
		if pcmcmp_res != 0:
		pytest.fail(reason)
		@@ -248,10 +255,15 @@ def test_masa_enc_dec(
		mld_lim=get_mld_lim,
		)
		if get_mld:
		mld = 0
		if "MLD" in reason:
		mld = float(reason.split(":")[1].split()[0])
		mld = re.search(MLD_PATTERN, reason).groups(1)[0]
		record_property("MLD", mld)

		max_diff = 0
		if pcmcmp_res:
		search_result = re.search(MAX_DIFF_PATTERN, reason)
		max_diff = search_result.groups(1)[0]
		record_property("MAXIMUM ABS DIFF", max_diff)

		# Report compare result
		if cmp_result != 0:
		pytest.fail(reason)