From 50e695f134149a6bd795e81eca4cae32fc999f70 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 15 Mar 2024 15:41:09 +0100 Subject: [PATCH 1/4] record maximum absolute diff as property in XML report --- tests/cmp_pcm.py | 21 +++++++++++++------ tests/codec_be_on_mr_nonselection/__init__.py | 2 ++ .../test_masa_enc_dec.py | 20 ++++++++++++------ .../test_param_file.py | 10 ++++++--- .../test_sba_bs_dec_plc.py | 10 ++++++--- .../test_sba_bs_enc.py | 10 ++++++--- tests/conftest.py | 3 +-- tests/test_param_file_ltv.py | 5 +++++ 8 files changed, 58 insertions(+), 23 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 13a1c1ba5a..45cdf5b832 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -49,21 +49,30 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str s1, s2, fs, per_frame=False, get_mld=get_mld ) - if cmp_result["bitexact"]: - return 0, "SUCCESS: Files are bitexact" - else: + output_differs = 0 + reason = "SUCCESS: Files are bitexact" + + if not cmp_result["bitexact"]: diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" print(diff_msg) print(first_msg) + + reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" + output_differs = 1 + if get_mld: mld_msg = f"MLD: {cmp_result['MLD']}" + reason += " - " + mld_msg print(mld_msg) + if cmp_result["MLD"] <= mld_lim: - return 0, f"MLD: {cmp_result['MLD']} <= {mld_lim}" + output_differs = 0 + reason += f" <= {mld_lim}" else: - return 1, f"MLD: {cmp_result['MLD']} > {mld_lim}" - return 1, "Non-BE" + reason += f" > {mld_lim}" + + return output_differs, reason if __name__ == "__main__": diff --git a/tests/codec_be_on_mr_nonselection/__init__.py b/tests/codec_be_on_mr_nonselection/__init__.py index e69de29bb2..aba85788e7 100644 --- a/tests/codec_be_on_mr_nonselection/__init__.py +++ b/tests/codec_be_on_mr_nonselection/__init__.py @@ -0,0 +1,2 @@ +MLD_PATTERN = r"MLD: ([\d\.]*)" +MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 75b01db382..470f2618d0 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -35,6 +35,7 @@ __doc__ = """ import errno import os +import re from filecmp import cmp from typing import Optional @@ -43,6 +44,8 @@ import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend +from . import MLD_PATTERN, MAX_DIFF_PATTERN + # params # output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] output_mode_list = ["BINAURAL", "EXT"] @@ -216,11 +219,13 @@ def test_masa_enc_dec( mld_lim=get_mld_lim, ) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + if get_mld and get_mld_lim > 0: if pcmcmp_res != 0: pytest.fail(reason) @@ -248,10 +253,13 @@ def test_masa_enc_dec( mld_lim=get_mld_lim, ) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + # Report compare result if cmp_result != 0: pytest.fail(reason) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 459964a186..e51bd2defa 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -35,6 +35,7 @@ Execute tests specified via a parameter file. import errno import filecmp import os +import re import platform from pathlib import Path from subprocess import run @@ -45,6 +46,7 @@ from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend from tests.cut_pcm import cut_samples from tests.testconfig import PARAM_FILE +from . import MLD_PATTERN, MAX_DIFF_PATTERN VALID_DEC_OUTPUT_CONF = [ "MONO", @@ -342,11 +344,13 @@ def test_param_file_tests( md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + metadata_differs = False for md_file in md_out_files: dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 11a8aeca04..11b0cfc4c0 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -34,11 +34,13 @@ __doc__ = """ import errno import os +import re import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend +from . import MLD_PATTERN, MAX_DIFF_PATTERN # params tag_list = ["stvFOA"] @@ -209,11 +211,13 @@ def sba_dec_plc( mld_lim=get_mld_lim, ) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + # report compare result if cmp_result != 0: pytest.fail(reason) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 4f4c1a3271..3298665eb0 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -35,6 +35,7 @@ __doc__ = """ import errno import os +import re import pytest from cut_bs import cut_from_start @@ -42,6 +43,7 @@ from cut_bs import cut_from_start from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend from tests.cut_pcm import cut_samples +from . import MLD_PATTERN, MAX_DIFF_PATTERN # params @@ -727,11 +729,13 @@ def sba_dec( mld_lim=get_mld_lim, ) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + # report compare result if cmp_result != 0: pytest.fail(reason) diff --git a/tests/conftest.py b/tests/conftest.py index 37b5281403..07004cad09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,10 +37,9 @@ import os from tests import testconfig import pytest import platform -import tempfile import textwrap from pathlib import Path -from subprocess import STDOUT, CalledProcessError, TimeoutExpired, run +from subprocess import TimeoutExpired, run from typing import Optional, Union logger = logging.getLogger(__name__) diff --git a/tests/test_param_file_ltv.py b/tests/test_param_file_ltv.py index 44c6a388f8..15ce69b177 100644 --- a/tests/test_param_file_ltv.py +++ b/tests/test_param_file_ltv.py @@ -35,6 +35,7 @@ Execute tests specified via a parameter file. import errno import filecmp import os +import re import platform from pathlib import Path from subprocess import run @@ -345,6 +346,10 @@ def test_param_file_tests( mld = float(reason.split(":")[1].split()[0]) record_property("MLD", mld) + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] + record_property("MAXIMUM ABS DIFF", max_diff) + metadata_differs = False for md_file in md_out_files: dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") -- GitLab From 8b5f1bee8455cc4b850c7a53e3977c9c0345b6cc Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 18 Mar 2024 14:16:59 +0100 Subject: [PATCH 2/4] fix for no failures --- .../codec_be_on_mr_nonselection/test_masa_enc_dec.py | 12 ++++++++---- tests/codec_be_on_mr_nonselection/test_param_file.py | 6 ++++-- .../test_sba_bs_dec_plc.py | 6 ++++-- tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py | 6 ++++-- tests/test_param_file_ltv.py | 11 ++++++----- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 470f2618d0..9dc8ce9e6d 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -222,8 +222,10 @@ def test_masa_enc_dec( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] + max_diff = 0 + if pcmcmp_res: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] record_property("MAXIMUM ABS DIFF", max_diff) if get_mld and get_mld_lim > 0: @@ -256,8 +258,10 @@ def test_masa_enc_dec( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] + max_diff = 0 + if pcmcmp_res: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] record_property("MAXIMUM ABS DIFF", max_diff) # Report compare result diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index e51bd2defa..f206caed4e 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -347,8 +347,10 @@ def test_param_file_tests( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] + max_diff =0 + if output_differs: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] record_property("MAXIMUM ABS DIFF", max_diff) metadata_differs = False diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 11b0cfc4c0..5b52b9bd2f 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -214,8 +214,10 @@ def sba_dec_plc( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] + max_diff = 0 + if cmp_result: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] record_property("MAXIMUM ABS DIFF", max_diff) # report compare result diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 3298665eb0..47eb8a3d42 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -732,8 +732,10 @@ def sba_dec( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] + max_diff = 0 + if cmp_result: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] record_property("MAXIMUM ABS DIFF", max_diff) # report compare result diff --git a/tests/test_param_file_ltv.py b/tests/test_param_file_ltv.py index 15ce69b177..a32ceb2412 100644 --- a/tests/test_param_file_ltv.py +++ b/tests/test_param_file_ltv.py @@ -45,6 +45,7 @@ import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend from tests.cut_pcm import cut_samples +from .codec_be_on_mr_nonselection import MLD_PATTERN, MAX_DIFF_PATTERN # from tests.testconfig import PARAM_FILE @@ -341,13 +342,13 @@ def test_param_file_tests( md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) if get_mld: - mld = 0 - if "MLD" in reason: - mld = float(reason.split(":")[1].split()[0]) + mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] + max_diff = 0 + if output_differs: + search_result = re.search(MAX_DIFF_PATTERN, reason) + max_diff = search_result.groups(1)[0] record_property("MAXIMUM ABS DIFF", max_diff) metadata_differs = False -- GitLab From fa428a2c31c646f49000508087b68b728023e96e Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 18 Mar 2024 16:01:04 +0100 Subject: [PATCH 3/4] fix mld value from cmp_pcm and add parsing of more than 1 property --- scripts/parse_mld_xml.py | 25 ++++++++++++++++--------- tests/cmp_pcm.py | 20 ++++++++++---------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index fb1dc8e73a..088789903d 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -7,6 +7,9 @@ from xml.etree import ElementTree Parse a junit report and create a MLD summary report. """ +PROPERTIES = ["MLD", "MAXIMUM ABS DIFF"] + + # Main routine if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -32,9 +35,12 @@ if __name__ == "__main__": testcases = tree.findall(".//testcase") with open(csv_file, "w") as outfile: + headerline = ";".join(["testcase"] + PROPERTIES) + "\n" + outfile.write(headerline) + for testcase in testcases: - if testcase.find(".//skipped") == None: - if testcase.get("file") == None: + if testcase.find(".//skipped") is None: + if testcase.get("file") is None: fulltestname = ( testcase.get("classname").replace(".", "/") + ".py::" @@ -42,10 +48,11 @@ if __name__ == "__main__": ) else: fulltestname = testcase.get("file") + "::" + testcase.get("name") - if testcase.find(".//property") == None: - mld_val = None - else: - mld_val = testcase.find(".//property").get( - "value" - ) # Currently MLD is the only set property. If more are added updates are needed here. - outfile.write(fulltestname + ";" + str(mld_val) + "\n") + + properties_found = { + p.get("name"): p.get("value") + for p in testcase.findall(".//property") + } + properties_values = [str(properties_found.get(p)) for p in PROPERTIES] + outline = ";".join([fulltestname] + properties_values) + "\n" + outfile.write(outline) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 45cdf5b832..c7df4b800d 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -61,16 +61,16 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" output_differs = 1 - if get_mld: - mld_msg = f"MLD: {cmp_result['MLD']}" - reason += " - " + mld_msg - print(mld_msg) - - if cmp_result["MLD"] <= mld_lim: - output_differs = 0 - reason += f" <= {mld_lim}" - else: - reason += f" > {mld_lim}" + if get_mld: + mld_msg = f"MLD: {cmp_result['MLD']}" + reason += " - " + mld_msg + print(mld_msg) + + if cmp_result["MLD"] <= mld_lim: + output_differs = 0 + reason += f" <= {mld_lim}" + else: + reason += f" > {mld_lim}" return output_differs, reason -- GitLab From aaf0aa591edc7d6b48af83590cf924f3126cbbf6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 19 Mar 2024 15:44:57 +0100 Subject: [PATCH 4/4] Remove obsolete parse_mld.py and add sorting of results in parse_mld_xml.py --- scripts/parse_mld.py | 32 ---------------------------- scripts/parse_mld_xml.py | 46 +++++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 52 deletions(-) delete mode 100644 scripts/parse_mld.py diff --git a/scripts/parse_mld.py b/scripts/parse_mld.py deleted file mode 100644 index b999e83903..0000000000 --- a/scripts/parse_mld.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python3 - -import argparse -import re - -# Main routine -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Parse HTML report to extract MLD values" - ) - parser.add_argument( - "html_report", type=str, help="HTML report input file, e.g. report.html" - ) - parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. output.csv") - args = parser.parse_args() - html_report = args.html_report - csv_file = args.csv_file - - mld = {} - - with open(html_report, "r") as infile: - for line in infile.readlines(): - if "col-name" in line: - test_name = re.search("\[(.*)\]", line).group(1) - mld[test_name] = 0.0 - if "MLD" in line: - mld_val = float(line.split()[1]) - mld[test_name] = mld_val - - with open(csv_file, "w") as outfile: - for test_name in mld: - outfile.write(test_name + ";" + str(mld[test_name]) + "\n") diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index 088789903d..84acf4fbba 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -4,7 +4,7 @@ import argparse from xml.etree import ElementTree """ -Parse a junit report and create a MLD summary report. +Parse a junit report and create an MLD summary report. """ PROPERTIES = ["MLD", "MAXIMUM ABS DIFF"] @@ -33,26 +33,32 @@ if __name__ == "__main__": ) testcases = tree.findall(".//testcase") + + results_unsorted = {} + + for testcase in testcases: + if testcase.find(".//skipped") is None: + if testcase.get("file") is None: + fulltestname = ( + testcase.get("classname").replace(".", "/") + + ".py::" + + testcase.get("name") + ) + else: + fulltestname = testcase.get("file") + "::" + testcase.get("name") + + properties_found = { + p.get("name"): p.get("value") + for p in testcase.findall(".//property") + } + properties_values = [str(properties_found.get(p)) for p in PROPERTIES] + outline = ";".join([fulltestname] + properties_values) + "\n" + results_unsorted[fulltestname] = outline + + results_sorted = dict(sorted(results_unsorted.items())) with open(csv_file, "w") as outfile: headerline = ";".join(["testcase"] + PROPERTIES) + "\n" outfile.write(headerline) - - for testcase in testcases: - if testcase.find(".//skipped") is None: - if testcase.get("file") is None: - fulltestname = ( - testcase.get("classname").replace(".", "/") - + ".py::" - + testcase.get("name") - ) - else: - fulltestname = testcase.get("file") + "::" + testcase.get("name") - - properties_found = { - p.get("name"): p.get("value") - for p in testcase.findall(".//property") - } - properties_values = [str(properties_found.get(p)) for p in PROPERTIES] - outline = ";".join([fulltestname] + properties_values) + "\n" - outfile.write(outline) + for test in results_sorted: + outfile.write(results_sorted[test]) -- GitLab