diff --git a/scripts/parse_mld.py b/scripts/parse_mld.py deleted file mode 100644 index b999e839032a3204e2b068a38f364779b46def94..0000000000000000000000000000000000000000 --- a/scripts/parse_mld.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python3 - -import argparse -import re - -# Main routine -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Parse HTML report to extract MLD values" - ) - parser.add_argument( - "html_report", type=str, help="HTML report input file, e.g. report.html" - ) - parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. output.csv") - args = parser.parse_args() - html_report = args.html_report - csv_file = args.csv_file - - mld = {} - - with open(html_report, "r") as infile: - for line in infile.readlines(): - if "col-name" in line: - test_name = re.search("\[(.*)\]", line).group(1) - mld[test_name] = 0.0 - if "MLD" in line: - mld_val = float(line.split()[1]) - mld[test_name] = mld_val - - with open(csv_file, "w") as outfile: - for test_name in mld: - outfile.write(test_name + ";" + str(mld[test_name]) + "\n") diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index fb1dc8e73ab38e7a73582aa00d051ca2241eaf3e..84acf4fbba7df736869137058ba935f4dd8e3fba 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -4,9 +4,12 @@ import argparse from xml.etree import ElementTree """ -Parse a junit report and create a MLD summary report. +Parse a junit report and create an MLD summary report. """ +PROPERTIES = ["MLD", "MAXIMUM ABS DIFF"] + + # Main routine if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -30,22 +33,32 @@ if __name__ == "__main__": ) testcases = tree.findall(".//testcase") + + results_unsorted = {} + + for testcase in testcases: + if testcase.find(".//skipped") is None: + if testcase.get("file") is None: + fulltestname = ( + testcase.get("classname").replace(".", "/") + + ".py::" + + testcase.get("name") + ) + else: + fulltestname = testcase.get("file") + "::" + testcase.get("name") + + properties_found = { + p.get("name"): p.get("value") + for p in testcase.findall(".//property") + } + properties_values = [str(properties_found.get(p)) for p in PROPERTIES] + outline = ";".join([fulltestname] + properties_values) + "\n" + results_unsorted[fulltestname] = outline + + results_sorted = dict(sorted(results_unsorted.items())) with open(csv_file, "w") as outfile: - for testcase in testcases: - if testcase.find(".//skipped") == None: - if testcase.get("file") == None: - fulltestname = ( - testcase.get("classname").replace(".", "/") - + ".py::" - + testcase.get("name") - ) - else: - fulltestname = testcase.get("file") + "::" + testcase.get("name") - if testcase.find(".//property") == None: - mld_val = None - else: - mld_val = testcase.find(".//property").get( - "value" - ) # Currently MLD is the only set property. If more are added updates are needed here. - outfile.write(fulltestname + ";" + str(mld_val) + "\n") + headerline = ";".join(["testcase"] + PROPERTIES) + "\n" + outfile.write(headerline) + for test in results_sorted: + outfile.write(results_sorted[test]) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 13a1c1ba5ab9180d05ca5adb2ac2e18fb045feb9..c7df4b800d8f47db19131ef43b216c57adb43180 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -49,21 +49,30 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str s1, s2, fs, per_frame=False, get_mld=get_mld ) - if cmp_result["bitexact"]: - return 0, "SUCCESS: Files are bitexact" - else: + output_differs = 0 + reason = "SUCCESS: Files are bitexact" + + if not cmp_result["bitexact"]: diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" print(diff_msg) print(first_msg) - if get_mld: - mld_msg = f"MLD: {cmp_result['MLD']}" - print(mld_msg) - if cmp_result["MLD"] <= mld_lim: - return 0, f"MLD: {cmp_result['MLD']} <= {mld_lim}" - else: - return 1, f"MLD: {cmp_result['MLD']} > {mld_lim}" - return 1, "Non-BE" + + reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" + output_differs = 1 + + if get_mld: + mld_msg = f"MLD: {cmp_result['MLD']}" + reason += " - " + mld_msg + print(mld_msg) + + if cmp_result["MLD"] <= mld_lim: + output_differs = 0 + reason += f" <= {mld_lim}" + else: + reason += f" > {mld_lim}" + + return output_differs, reason if __name__ == "__main__": diff --git a/tests/codec_be_on_mr_nonselection/__init__.py b/tests/codec_be_on_mr_nonselection/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..aba85788e77b1889a019b49dc761a76b04fa8ac4 100644 --- a/tests/codec_be_on_mr_nonselection/__init__.py +++ b/tests/codec_be_on_mr_nonselection/__init__.py @@ -0,0 +1,2 @@ +MLD_PATTERN = r"MLD: ([\d\.]*)" +MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 75b01db3829d1e1786a2c9bd421f1c8007829713..9dc8ce9e6dbcae08effdb32ddf9f92a1fe4ef514 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -35,6 +35,7 @@ __doc__ = """ import errno import os +import re from filecmp import cmp from typing import Optional @@ -43,6 +44,8 @@ import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend +from . import MLD_PATTERN, MAX_DIFF_PATTERN + # params # output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] output_mode_list = ["BINAURAL", "EXT"] @@ -216,11 +219,15 @@ def test_masa_enc_dec( mld_lim=get_mld_lim, ) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + max_diff = 0 + if pcmcmp_res: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + if get_mld and get_mld_lim > 0: if pcmcmp_res != 0: pytest.fail(reason) @@ -248,10 +255,15 @@ def test_masa_enc_dec( mld_lim=get_mld_lim, ) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + + max_diff = 0 + if pcmcmp_res: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + # Report compare result if cmp_result != 0: pytest.fail(reason) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 9a1405c27e287824c9c2bcc14ef71737bb1b9cf2..890d530f825561e6a9e89e16ee8bdcff427b2ee1 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -35,6 +35,7 @@ Execute tests specified via a parameter file. import errno import filecmp import os +import re import platform from pathlib import Path from subprocess import run @@ -45,6 +46,7 @@ from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend from tests.cut_pcm import cut_samples from tests.testconfig import PARAM_FILE +from . import MLD_PATTERN, MAX_DIFF_PATTERN VALID_DEC_OUTPUT_CONF = [ "MONO", @@ -342,11 +344,15 @@ def test_param_file_tests( md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + max_diff =0 + if output_differs: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + metadata_differs = False for md_file in md_out_files: dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 11a8aeca04baf683bfdc67ee5f25f278e77f2b6b..5b52b9bd2f1c4f42bfd96b49a90b6a415f8ce784 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -34,11 +34,13 @@ __doc__ = """ import errno import os +import re import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend +from . import MLD_PATTERN, MAX_DIFF_PATTERN # params tag_list = ["stvFOA"] @@ -209,11 +211,15 @@ def sba_dec_plc( mld_lim=get_mld_lim, ) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + max_diff = 0 + if cmp_result: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + # report compare result if cmp_result != 0: pytest.fail(reason) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 5782982e9ded4f1ebe96184a0285b836d8bb8dd9..0a07a0d1d7a6348b898111815c8293e3a1303195 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -35,6 +35,7 @@ __doc__ = """ import errno import os +import re import pytest from cut_bs import cut_from_start @@ -42,6 +43,7 @@ from cut_bs import cut_from_start from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend from tests.cut_pcm import cut_samples +from . import MLD_PATTERN, MAX_DIFF_PATTERN # params @@ -727,11 +729,15 @@ def sba_dec( mld_lim=get_mld_lim, ) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + max_diff = 0 + if cmp_result: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + # report compare result if cmp_result != 0: pytest.fail(reason) diff --git a/tests/test_param_file_ltv.py b/tests/test_param_file_ltv.py index 41dfeae30e11f541a6c4bb7c9c41e1d542384840..a2341e93506313ad1403fcd59ab383d1dc49a3e4 100644 --- a/tests/test_param_file_ltv.py +++ b/tests/test_param_file_ltv.py @@ -35,6 +35,7 @@ Execute tests specified via a parameter file. import errno import filecmp import os +import re import platform from pathlib import Path from subprocess import run @@ -44,6 +45,7 @@ import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend from tests.cut_pcm import cut_samples +from .codec_be_on_mr_nonselection import MLD_PATTERN, MAX_DIFF_PATTERN # from tests.testconfig import PARAM_FILE @@ -340,11 +342,15 @@ def test_param_file_tests( md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + max_diff = 0 + if output_differs: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + metadata_differs = False for md_file in md_out_files: dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}")