diff --git a/tests/cmp_stats_files.py b/tests/cmp_stats_files.py index bc191c87ee5e1c961c90f55c24dc50e588f19a22..2e88f3883fc36bf9d67643ef6f295b8ed1bc0742 100644 --- a/tests/cmp_stats_files.py +++ b/tests/cmp_stats_files.py @@ -18,7 +18,7 @@ def str2num(s): def cmp_stats_files( - ref_stats_file, dut_stats_file, min_enc_file_length_diff=0.1, min_enc_stats_diff=0.1 + ref_stats_file, dut_stats_file, max_enc_file_length_diff, max_enc_stats_diff ) -> (int, str): """ Compare two .stats files containing encoder statistics (extracted from binary files) @@ -41,6 +41,7 @@ def cmp_stats_files( enc_test_result_msg = "" max_total_num_diff = 0 max_total_num_diff_ratio = 0 + max_diff_name = "" for name in ref_stats_names: if name in dut_stats_names: # retrieve the dictionaries @@ -59,7 +60,7 @@ def cmp_stats_files( if ( abs(ref_stats_dict["length"] - dut_stats_dict["length"]) / file_length - > min_enc_file_length_diff + > max_enc_file_length_diff ): result_len_check = 1 @@ -97,7 +98,7 @@ def cmp_stats_files( ) msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)" - if total_num_diff_ratio > min_enc_stats_diff: + if total_num_diff_ratio > max_enc_stats_diff: result_diff_check = 1 msg += "! " else: @@ -108,6 +109,7 @@ def cmp_stats_files( if total_num_diff_ratio > max_total_num_diff_ratio: max_total_num_diff = total_num_diff max_total_num_diff_ratio = total_num_diff_ratio + max_diff_name = name # update test result if result_len_check or result_diff_check: @@ -116,8 +118,8 @@ def cmp_stats_files( print(msg) - if enc_test_result and max_total_num_diff > 0: - msg = f"MAXIMUM ENC DIFF: {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%) " + if enc_test_result or max_total_num_diff > 0: + msg = f"MAXIMUM ENC DIFF: {max_diff_name} {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%)" enc_test_result_msg += msg print(msg) @@ -130,13 +132,13 @@ if __name__ == "__main__": parser.add_argument("ref_stats_file", type=str) parser.add_argument("dut_stats_file", type=str) parser.add_argument( - "--min_enc_file_length_diff", + "--max_enc_file_length_diff", type=float, default=0.1, - dest="min_enc_file_length_diff", + dest="max_enc_file_length_diff", ) parser.add_argument( - "--min_enc_stats_diff", type=float, default=0.1, dest="min_enc_stats_diff" + "--max_enc_stats_diff", type=float, default=0.1, dest="max_enc_stats_diff" ) args = parser.parse_args() diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 9fe43a40b21677cbf2560b39ada3d2d126064ecf..390edcf29b26b987e0727d2a7753d2d32580fdac 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -47,8 +47,8 @@ from tests.cmp_stats_files import cmp_stats_files from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties from tests.testconfig import PARAM_FILE from tests.constants import ( - MIN_ENC_FILE_LENGTH_DIFF, - MIN_ENC_STATS_DIFF, + MAX_ENC_FILE_LENGTH_DIFF, + MAX_ENC_STATS_DIFF, SCRIPTS_DIR, MAX_ENC_DIFF, ) @@ -251,16 +251,18 @@ def test_param_file_tests( enc_test_result, enc_test_result_msg = cmp_stats_files( ref_stats_file, dut_stats_file, - min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF, - min_enc_stats_diff=MIN_ENC_STATS_DIFF, + max_enc_file_length_diff=MAX_ENC_FILE_LENGTH_DIFF, + max_enc_stats_diff=MAX_ENC_STATS_DIFF, ) print("") cmp_result_msg += enc_test_result_msg - props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) - for k, v in props.items(): - dut_encoder_frontend.record_property(k, v) + # avoid double recording of the encoder diff + if encoder_only: + props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) + for k, v in props.items(): + dut_encoder_frontend.record_property(k, v) if encoder_only: if enc_test_result: diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index e8ae17f55b6da65a09d6a00e624792640f728cd5..4bd81cdfd302bec7129b6652616441287bc71d04 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -42,7 +42,7 @@ from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend from ..conftest import parse_properties from ..cmp_stats_files import cmp_stats_files -from ..constants import TESTV_DIR, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF +from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF from tests.testconfig import use_ltv @@ -309,8 +309,8 @@ def test_sba_enc_system( enc_test_result, enc_test_result_msg = cmp_stats_files( ref_stats_file, dut_stats_file, - min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF, - min_enc_stats_diff=MIN_ENC_STATS_DIFF, + max_enc_file_length_diff=MAX_ENC_FILE_LENGTH_DIFF, + max_enc_stats_diff=MAX_ENC_STATS_DIFF, ) print("") diff --git a/tests/conftest.py b/tests/conftest.py index 989f080c906d28e954d252c8aeec4d133242151c..bb9e8ca3871c6e252fcb2bdcdd0c7392fc1df5ee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,6 +46,8 @@ import tempfile from typing import Optional, Union import numpy as np from .constants import ( + # MAX_ENC_DIFF_NAME_PATTERN, + MAX_ENC_DIFF_PARAM_NAME, MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN, @@ -267,11 +269,12 @@ def pytest_addoption(parser): ) parser.addoption( - "--compare_to_input", - action="store_true", - help="Compare output to the input file instead of reference output. N.B. Only applicable to pass-through tests.", - default=False, - ) + "--compare_to_input", + action="store_true", + help="Compare output to the input file instead of reference output. N.B. Only applicable to pass-through tests.", + default=False, + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -332,6 +335,7 @@ def get_odg(request): """ return request.config.option.odg + @pytest.fixture(scope="session") def abs_tol(request) -> int: """ @@ -364,7 +368,7 @@ def dut_encoder_path(request) -> str: if request.config.option.dut_encoder_path: return request.config.option.dut_encoder_path if request.config.option.update_ref == "1": - return None + return None here = Path(__file__).parent.resolve() system = platform.system() @@ -406,9 +410,11 @@ class EncoderFrontend: """ hist_dicts = [] - + if not os.path.exists(dbg_tweak_folder): - print(f"No statistics have been extracted from the res/ folder to the {stats_file} file!") + print( + f"No statistics have been extracted from the res/ folder to the {stats_file} file!" + ) else: for f in ENC_AUX_FILES: filename = f[0] @@ -543,12 +549,16 @@ class EncoderFrontend: self.stdout = result.stdout.decode("ascii") # Record core encoder clipping - for (prop, pattern) in [(ENC_CORE_OVL,ENC_CORE_OVL_PATTERN),(MAX_OVL,MAX_OVL_PATTERN),(MIN_OVL,MIN_OVL_PATTERN)]: + for prop, pattern in [ + (ENC_CORE_OVL, ENC_CORE_OVL_PATTERN), + (MAX_OVL, MAX_OVL_PATTERN), + (MIN_OVL, MIN_OVL_PATTERN), + ]: val = 0 search_result = re.search(pattern, self.stdout) if search_result: val = search_result.group(1) - self.record_property( prop, val ) + self.record_property(prop, val) if self.stdout: stdout_str = textwrap.indent(self.stdout, prefix="\t") @@ -576,7 +586,9 @@ class EncoderFrontend: @pytest.fixture(scope="function") -def dut_encoder_frontend(dut_encoder_path, request, record_property) -> Union[None, EncoderFrontend]: +def dut_encoder_frontend( + dut_encoder_path, request, record_property +) -> Union[None, EncoderFrontend]: """ Return a :class:`conftest.EncoderFrontend` instance as DUT for the test session. """ @@ -584,7 +596,9 @@ def dut_encoder_frontend(dut_encoder_path, request, record_property) -> Union[No if dut_encoder_path: timeout = request.config.getoption("--testcase_timeout") - encoder = EncoderFrontend(dut_encoder_path, "DUT", record_property, timeout=timeout) + encoder = EncoderFrontend( + dut_encoder_path, "DUT", record_property, timeout=timeout + ) yield encoder if encoder is not None: @@ -623,7 +637,9 @@ def ref_encoder_path(request) -> str: @pytest.fixture(scope="function") -def ref_encoder_frontend(ref_encoder_path, request, record_property) -> Union[None, EncoderFrontend]: +def ref_encoder_frontend( + ref_encoder_path, request, record_property +) -> Union[None, EncoderFrontend]: """ Return a :class:`conftest.EncoderFrontend` instance as REF for the test session. """ @@ -631,7 +647,9 @@ def ref_encoder_frontend(ref_encoder_path, request, record_property) -> Union[No if ref_encoder_path: timeout = request.config.getoption("--testcase_timeout") - encoder = EncoderFrontend(ref_encoder_path, "REF", record_property, timeout=timeout) + encoder = EncoderFrontend( + ref_encoder_path, "REF", record_property, timeout=timeout + ) yield encoder @@ -846,7 +864,9 @@ class DecoderFrontend: @pytest.fixture(scope="function") -def dut_decoder_frontend(dut_decoder_path, request, record_property) -> Union[None, DecoderFrontend]: +def dut_decoder_frontend( + dut_decoder_path, request, record_property +) -> Union[None, DecoderFrontend]: """ Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. """ @@ -854,7 +874,13 @@ def dut_decoder_frontend(dut_decoder_path, request, record_property) -> Union[No if dut_decoder_path: timeout = request.config.getoption("--testcase_timeout") - decoder = DecoderFrontend(dut_decoder_path, "DUT", record_property, timeout=timeout, fr=request.config.option.dut_fr) + decoder = DecoderFrontend( + dut_decoder_path, + "DUT", + record_property, + timeout=timeout, + fr=request.config.option.dut_fr, + ) yield decoder @@ -894,7 +920,9 @@ def ref_decoder_path(request) -> str: @pytest.fixture(scope="function") -def ref_decoder_frontend(ref_decoder_path, request, record_property) -> Union[None, DecoderFrontend]: +def ref_decoder_frontend( + ref_decoder_path, request, record_property +) -> Union[None, DecoderFrontend]: """ Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. """ @@ -902,7 +930,9 @@ def ref_decoder_frontend(ref_decoder_path, request, record_property) -> Union[No if ref_decoder_path: timeout = request.config.getoption("--testcase_timeout") - decoder = DecoderFrontend(ref_decoder_path, "REF", record_property, timeout=timeout) + decoder = DecoderFrontend( + ref_decoder_path, "REF", record_property, timeout=timeout + ) yield decoder @@ -959,7 +989,10 @@ def reference_path(request) -> str: path = str(path.resolve()) - if request.config.option.update_ref == "0" and not request.config.option.compare_to_input: + if ( + request.config.option.update_ref == "0" + and not request.config.option.compare_to_input + ): if not os.path.isdir(path): raise FileNotFoundError( f"REF path {path} not found!\nPlease generate the references, first!\n!" @@ -1000,6 +1033,7 @@ def encoder_only(request) -> bool: """ return request.config.getoption("--encoder_only") + @pytest.fixture(scope="session", autouse=True) def compare_to_input(request) -> bool: """ @@ -1007,6 +1041,7 @@ def compare_to_input(request) -> bool: """ return request.config.getoption("--compare_to_input") + def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial") if config.option.param_file: @@ -1079,12 +1114,12 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: props["MIN_ODG_CHANNEL"] = min_odg_channel elif prop == MAX_ENC_DIFF: search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse) - max_enc_diff = 0 + max_enc_diff_ratio = 0.0 + max_enc_diff_param_name = "" if search_result: - max_enc_diff, max_enc_diff_ratio = search_result.groups(0) - if max_enc_diff: - max_enc_diff = float(max_enc_diff) - props[MAX_ENC_DIFF] = max_enc_diff + max_enc_diff_param_name, _, max_enc_diff_ratio = search_result.groups(0) + props[MAX_ENC_DIFF] = float(max_enc_diff_ratio) + props[MAX_ENC_DIFF_PARAM_NAME] = max_enc_diff_param_name return props diff --git a/tests/constants.py b/tests/constants.py index 36dabd0584c78e8f02e39ba90242654654fe38c6..562035f8f484c1497fd07313e2a59933422e95d8 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -12,6 +12,7 @@ MAX_ABS_DIFF = "MAXIMUM ABS DIFF" SSNR = "SSNR" ODG = "ODG" MAX_ENC_DIFF = "MAXIMUM ENC DIFF" +MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM" ENC_CORE_OVL = "ENC_CORE_OVL" MAX_OVL = "MAX_OVL" MIN_OVL = "MIN_OVL" @@ -22,17 +23,16 @@ MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" ODG_PATTERN_PQEVALAUDIO = r"Objective Difference Grade: (-*\d*\.\d*)" ODG_PATTERN = r"ODG: (-*\d*\.\d*)" SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[-*\d\.]*)" -MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)" +MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: ([^ ]*) (\d+) \((\d+\.\d+)%\)" ENC_CORE_OVL_PATTERN = r"Core input overload detected: (\d+)" MAX_OVL_PATTERN = r"Max overload value: (\d+\.\d+)" MIN_OVL_PATTERN = r"Min overload value: (-*\d+\.\d+)" +# maximum "relative" tolerable difference between ref and dut encoder file lengths +MAX_ENC_FILE_LENGTH_DIFF = 0.1 -# minimum difference between ref and dut encoder file lengths -MIN_ENC_FILE_LENGTH_DIFF = 0.1 -# minimum difference between the statistics of ref and dut encoder files -MIN_ENC_STATS_DIFF = 0.1 - +# maximum "relative" tolerable difference between the statistics of ref and dut encoder files +MAX_ENC_STATS_DIFF = 0.5 # list of encoder filename patterns with their data type and number of samples per frame # note: instead of specifying the number of samples per frame, you can use a formula incl. 'fs', e.g. 'fs/50'