From f6d9e5f64a2fb864b70fe0a053f35a0d79f6e996 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Mar 2025 18:34:18 +0100 Subject: [PATCH 01/70] add first infrastructure for splitting still functionally equivalent to before --- tests/cmp_pcm.py | 214 ++++++++++-------- .../test_param_file.py | 29 ++- tests/conftest.py | 31 ++- tests/constants.py | 37 +++ 4 files changed, 196 insertions(+), 115 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 127820525a..e254f1d59a 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -7,7 +7,7 @@ import tempfile import re import subprocess from pathlib import Path -from typing import Optional +from typing import Optional, List THIS_PATH = os.path.join(os.getcwd(), __file__) sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) @@ -41,8 +41,9 @@ def cmp_pcm( ref_jbm_tf: Optional[Path] = None, cut_jbm_tf: Optional[Path] = None, quiet: Optional[bool] = False, - scalefac=1, -) -> tuple[int, str]: + scalefac: int = 1, + split_idx: np.ndarray = np.empty(0), +) -> tuple[List[int], List[str]]: """ Compare 2 PCM files for bitexactness """ @@ -67,13 +68,13 @@ def cmp_pcm( if fs1 != fs2: reason = "FAIL: Sampling rate differs." - return 1, reason + return [1], [reason] # In case number of channels do not match, fail already now. Could happen in case of # comparison to input with for a non-passthrough mode. if s1.shape[1] != s2.shape[1]: reason = "FAIL: Number of channels differ." - return 1, reason + return [1], [reason] handle_differing_lengths = "fail" if allow_differing_lengths: @@ -88,108 +89,115 @@ def cmp_pcm( if get_mld: reason += " - MLD: None" - return 1, reason - - # Apply scalefac if specified. Useful in case scaling has been applied on the input, and the inverse is scaling is supplied in scalefac. - if scalefac != 1: - s1 = np.round(s1*scalefac, 0) # Need rounding for max abs diff search - s2 = np.round(s2*scalefac, 0) - - cmp_result = pyaudio3dtools.audioarray.compare( - s1, - s2, - fs, - per_frame=False, - get_mld=get_mld, - get_ssnr=get_ssnr, - ssnr_thresh_low=-50, - ref_jbm_tf=ref_jbm_tf, - test_jbm_tf=cut_jbm_tf, - handle_differing_lengths=handle_differing_lengths, - ) + return [1], [reason] + + output_differs_parts = [] + reason_parts = [] + + for s1, s2 in zip(np.split(s1, split_idx), np.split(s2, split_idx)): + # Apply scalefac if specified. Useful in case scaling has been applied on the input, and the inverse is scaling is supplied in scalefac. + if scalefac != 1: + s1 = np.round(s1 * scalefac, 0) # Need rounding for max abs diff search + s2 = np.round(s2 * scalefac, 0) + + cmp_result = pyaudio3dtools.audioarray.compare( + s1, + s2, + fs, + per_frame=False, + get_mld=get_mld, + get_ssnr=get_ssnr, + ssnr_thresh_low=-50, + ref_jbm_tf=ref_jbm_tf, + test_jbm_tf=cut_jbm_tf, + handle_differing_lengths=handle_differing_lengths, + ) + + output_differs = 0 + reason = "SUCCESS: Files are bitexact" + + if not cmp_result["bitexact"] and cmp_result["max_abs_diff"] <= abs_tol: + reason = "SUCCESS: Maximum absolute diff below threshold" + elif not cmp_result["bitexact"]: + diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" + first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" + print(diff_msg, file=output_target) + print(first_msg, file=output_target) - output_differs = 0 - reason = "SUCCESS: Files are bitexact" - - if not cmp_result["bitexact"] and cmp_result["max_abs_diff"] <= abs_tol: - reason = "SUCCESS: Maximum absolute diff below threshold" - elif not cmp_result["bitexact"]: - diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" - first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" - print(diff_msg, file=output_target) - print(first_msg, file=output_target) - - reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" - output_differs = 1 - - if get_mld: - mld_msg = f"MLD: {cmp_result['MLD']}" - reason += " - " + mld_msg - print(mld_msg, file=output_target) - - if cmp_result["MLD"] <= mld_lim: - output_differs = 0 - reason += f" <= {mld_lim}" - else: - reason += f" > {mld_lim}" - - if get_ssnr: - reason += " - " - for i, s in enumerate(cmp_result["SSNR"], start=1): - msg = f"Channel {i} SSNR: {s}" - reason += msg + " - " - - if get_odg: - for n in range(nchannels): - pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" + output_differs = 1 + if get_mld: + mld_msg = f"MLD: {cmp_result['MLD']}" + reason += " - " + mld_msg + print(mld_msg, file=output_target) + + if cmp_result["MLD"] <= mld_lim: + output_differs = 0 + reason += f" <= {mld_lim}" + else: + reason += f" > {mld_lim}" + + if get_ssnr: + reason += " - " + for i, s in enumerate(cmp_result["SSNR"], start=1): + msg = f"Channel {i} SSNR: {s}" + reason += msg + " - " + + if get_odg: + for n in range(nchannels): + pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg = float(match_odg.groups()[0]) + msg = f"Channel {n} ODG: {odg}" + reason += " - " + msg + print(msg) + + if get_odg_bin: + odg_files = {} + for f in [odg_input, odg_test, odg_ref]: + # Load PEAQ test files and ensure 48 kHz sampling rate + s, fs = pyaudio3dtools.audiofile.readfile( + f, nchannels, fs, outdtype=np.int16 + ) + odg_files[f] = np.clip( + pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000), + -32768, + 32767, + ).astype(np.int16) + + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_ref], 48000 + ) match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - odg = float(match_odg.groups()[0]) - msg = f"Channel {n} ODG: {odg}" - reason += " - " + msg - print(msg) - - if get_odg_bin: - odg_files = {} - for f in [odg_input, odg_test, odg_ref]: - # Load PEAQ test files and ensure 48 kHz sampling rate - s, fs = pyaudio3dtools.audiofile.readfile( - f, nchannels, fs, outdtype=np.int16 + try: + odg_ref = float(match_odg.groups()[0]) + except AttributeError: + raise OdgParsingFailed("Could not get Odg for ref signal") + + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_test], 48000 ) - odg_files[f] = np.clip( - pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000), - -32768, - 32767, - ).astype(np.int16) - - pqeval_output = pqevalaudio_wrapper( - odg_files[odg_input], odg_files[odg_ref], 48000 - ) - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - try: - odg_ref = float(match_odg.groups()[0]) - except AttributeError: - raise OdgParsingFailed("Could not get Odg for ref signal") - - pqeval_output = pqevalaudio_wrapper( - odg_files[odg_input], odg_files[odg_test], 48000 - ) - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - try: - odg_test = float(match_odg.groups()[0]) - except AttributeError: - raise OdgParsingFailed("Could not get Odg for test signal") + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + try: + odg_test = float(match_odg.groups()[0]) + except AttributeError: + raise OdgParsingFailed("Could not get Odg for test signal") + + odg = odg_test - odg_ref # Todo: store both rather than difference? - odg = odg_test - odg_ref # Todo: store both rather than difference? + msg = f"Delta-ODG: {odg}" + reason += " - " + msg + print(msg, file=output_target) - msg = f"Delta-ODG: {odg}" - reason += " - " + msg - print(msg, file=output_target) + if quiet: + output_target.close() - if quiet: - output_target.close() + output_differs_parts.append(output_differs) + reason_parts.append(reason) - return output_differs, reason + return output_differs_parts, reason_parts class OdgParsingFailed(Exception): @@ -262,7 +270,13 @@ if __name__ == "__main__": parser.add_argument("--get_odg", action="store_true") parser.add_argument("--get_ssnr", action="store_true") parser.add_argument("--allow_differing_lengths", action="store_true") - parser.add_argument("--scalefac", type=float, default=1, dest="scalefac", help="Scale factor to be applied before comparing the output. Useful when input scaling has been applied.") + parser.add_argument( + "--scalefac", + type=float, + default=1, + dest="scalefac", + help="Scale factor to be applied before comparing the output. Useful when input scaling has been applied.", + ) parser.add_argument("--quiet", action="store_true") args = vars(parser.parse_args()) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index ac2d84d1e7..495c583cc3 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -53,6 +53,7 @@ from tests.conftest import ( parse_properties, compare_dmx_signals, log_dbg_msg, + get_split_idx, ) from tests.testconfig import PARAM_FILE from tests.constants import ( @@ -588,7 +589,13 @@ def run_test( ref_file = ref_output_file fs = int(sampling_rate) * 1000 - output_differs, reason = cmp_pcm( + + # split_idx = np.empty(0) + # # TODO:test on --split argument given + # + # split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + + output_differs_parts, reason_parts = cmp_pcm( ref_file, dut_output_file, out_config_2_nchannels(output_config), @@ -608,11 +615,10 @@ def run_test( scalefac=test_info.config.option.scalefac, ) - cmp_result_msg += reason - - props = parse_properties(cmp_result_msg, output_differs, props_to_record) - for k, v in props.items(): - dut_decoder_frontend.record_property(k, v) + for output_differs, reason in zip(output_differs_parts, reason_parts): + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + dut_decoder_frontend.record_property(k, v) metadata_differs = False @@ -647,19 +653,22 @@ def run_test( if enc_test_result: pytest.fail("Too high difference in encoder statistics found.") + at_least_one_output_differs = any(output_differs_parts) + # TODO + reason = reason_parts[0] if tracefile_last_rtp_numbers_differ: pytest.fail( "Last RTP sequence num in tracefiles differ for JBM decoding - Not all frames were decoded in both ref and dut." ) elif get_mld and get_mld_lim > 0: - if output_differs: + if at_least_one_output_differs: pytest.fail(reason) else: - if output_differs or metadata_differs: + if at_least_one_output_differs or metadata_differs: msg = "Difference between ref and dut in " - if output_differs and metadata_differs: + if at_least_one_output_differs and metadata_differs: msg += f"output ({reason}) and metadata" - elif output_differs: + elif at_least_one_output_differs: msg += f"output only ({reason})" elif metadata_differs: msg += "metadata only" diff --git a/tests/conftest.py b/tests/conftest.py index 3b75cbc557..a5eab3ae09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,7 +36,6 @@ import logging import os import re import json -import shutil from tests import testconfig import pytest import platform @@ -49,13 +48,13 @@ import tempfile from typing import Optional, Union import numpy as np from .constants import ( - # MAX_ENC_DIFF_NAME_PATTERN, DMX_DIFF, DMX_MLD, DMX_SSNR, MAX_ENC_DIFF_PARAM_NAME, MLD_PATTERN, MAX_DIFF_PATTERN, + SPLIT_IDX, SSNR_PATTERN, ENC_AUX_FILES, ODG_PATTERN, @@ -306,6 +305,13 @@ def pytest_addoption(parser): default=1, ) + parser.addoption( + "--split_comparison", + action="store_true", + help="If given, split the output files by the provided indices", + ) + + @pytest.fixture(scope="session", autouse=True) def update_ref(request): """ @@ -365,6 +371,7 @@ def get_odg(request): """ return request.config.option.odg + @pytest.fixture(scope="session", autouse=True) def get_odg_bin(request): """ @@ -1217,20 +1224,20 @@ def compare_dmx_signals(ref_dmx_files, dut_dmx_files, fs) -> dict: ref_dmx_files, dmx_file_ref_tmp.name, out_nchans=nchannels, - in_fs=fs*1000, + in_fs=fs * 1000, ) pyaudio3dtools.audiofile.combinefiles( dut_dmx_files, dmx_file_dut_tmp.name, out_nchans=nchannels, - in_fs=fs*1000, + in_fs=fs * 1000, ) dmx_differs, reason = cmp_pcm( dmx_file_ref_tmp.name, dmx_file_dut_tmp.name, nchannels, - fs*1000, + fs * 1000, get_mld=True, get_ssnr=True, quiet=True, @@ -1240,3 +1247,17 @@ def compare_dmx_signals(ref_dmx_files, dut_dmx_files, fs) -> dict: prop_results = parse_properties(reason, dmx_differs, dmx_props) return prop_results + + +def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarray]: + """ + Return array for splitting the output file before doing the comparison. + + If no list of indices is available for the given input file, an empty array is returned. + """ + idx = SPLIT_IDX.get(input_file, np.empty(0)) + + if len(idx) > 0 and sampling_rate_khz != 16: + idx *= sampling_rate_khz // 16 + + return idx diff --git a/tests/constants.py b/tests/constants.py index 9990db37cc..13f46b8553 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -65,3 +65,40 @@ ENC_AUX_FILES = [ ["total_brate", np.float32, "fs/50"], ["vad_flag", np.int16, "fs/50"], ] + +# lists of indices for splitting of output files +# values are in SAMPLES for 16kHz (!). For higher rates, need to multiply +SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( + [ + 302096, + 519280, + 613072, + 690176, + 740992, + 749024, + 782096, + 950064, + 1045408, + 1141408, + 1237424, + 1333392, + 1397712, + 1472912, + 1755840, + 1781680, + 1789712, + 1957664, + 2158112, + 2294704, + 2454688, + 2525872, + 2573888, + 2645504, + 2746096, + 2842096, + 2970080, + 3219792, + ] +) + +SPLIT_IDX = {"ltv16_stereo": SPLIT_IDX_LTV_STEREO_16KHZ} -- GitLab From ddaa327a0961b8893ec078ad49944e84617df66f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 09:42:03 +0100 Subject: [PATCH 02/70] activate split comparison via the cmdl switch recording of the results needs to be added still --- tests/codec_be_on_mr_nonselection/test_param_file.py | 12 ++++++++---- tests/conftest.py | 9 +++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 495c583cc3..f39b22adf2 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -218,6 +218,7 @@ def test_param_file_tests( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -250,6 +251,7 @@ def test_param_file_tests( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ) @@ -282,6 +284,7 @@ def run_test( get_odg_bin, compare_to_input, compare_enc_dmx, + split_comparison, ): # If compare_to_input is set, only run pass-through test cases if compare_to_input: @@ -590,10 +593,10 @@ def run_test( fs = int(sampling_rate) * 1000 - # split_idx = np.empty(0) - # # TODO:test on --split argument given - # - # split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + pytest.set_trace() + split_idx = np.empty(0) + if split_comparison: + split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) output_differs_parts, reason_parts = cmp_pcm( ref_file, @@ -613,6 +616,7 @@ def run_test( ref_jbm_tf=ref_tracefile_dec, cut_jbm_tf=dut_tracefile_dec, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) for output_differs, reason in zip(output_differs_parts, reason_parts): diff --git a/tests/conftest.py b/tests/conftest.py index a5eab3ae09..c5c74a61c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -306,7 +306,7 @@ def pytest_addoption(parser): ) parser.addoption( - "--split_comparison", + "--split-comparison", action="store_true", help="If given, split the output files by the provided indices", ) @@ -447,6 +447,11 @@ def test_info(request): pytest.fail(request.error) +@pytest.fixture(scope="session") +def split_comparison(request): + return request.config.option.split_comparison + + class EncoderFrontend: def __init__(self, path, enc_type, record_property, timeout=None) -> None: self._path = Path(path).absolute() @@ -1255,7 +1260,7 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra If no list of indices is available for the given input file, an empty array is returned. """ - idx = SPLIT_IDX.get(input_file, np.empty(0)) + idx = SPLIT_IDX.get(input_file.lower(), np.empty(0)) if len(idx) > 0 and sampling_rate_khz != 16: idx *= sampling_rate_khz // 16 -- GitLab From 075846353fb0f26c9235d9bd34bba2a0c8ca5acd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 09:59:14 +0100 Subject: [PATCH 03/70] record measurements for individual splits --- .../test_param_file.py | 11 +++++++--- tests/conftest.py | 22 ++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index f39b22adf2..c9854185ac 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -593,7 +593,6 @@ def run_test( fs = int(sampling_rate) * 1000 - pytest.set_trace() split_idx = np.empty(0) if split_comparison: split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) @@ -619,8 +618,14 @@ def run_test( split_idx=split_idx, ) - for output_differs, reason in zip(output_differs_parts, reason_parts): - props = parse_properties(reason, output_differs, props_to_record) + prop_suffix = [""] + if len(split_idx) > 0: + prop_suffix = [f"_split{i}" for i in range(1, len(split_idx) + 1)] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + props = parse_properties(reason, output_differs, props_to_record, suffix) for k, v in props.items(): dut_decoder_frontend.record_property(k, v) diff --git a/tests/conftest.py b/tests/conftest.py index c5c74a61c6..d3717db395 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1169,7 +1169,9 @@ def props_to_record( return props -def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: list): +def parse_properties( + text_to_parse: str, output_differs: bool, props_to_record: list, suffix: str = "" +): """ Record the given properties in the report by parsing their values from the text. """ @@ -1179,7 +1181,7 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: for prop in props_to_record: if prop == MLD or prop == DMX_MLD: mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0]) - props[prop] = mld + props[prop + suffix] = mld elif prop == MAX_ABS_DIFF or prop == DMX_DIFF: max_diff = 0 if output_differs: @@ -1187,33 +1189,33 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: max_diff = match.groups(1)[0] else: raise MaxDiffPatternNotFound() - props[prop] = max_diff + props[prop + suffix] = max_diff elif prop == SSNR or prop == DMX_SSNR: ssnrs = re.findall(SSNR_PATTERN, text_to_parse) min_ssnr = min(ssnrs) min_ssnr_channel = ssnrs.index(min_ssnr) prefix = "MIN" if prop == SSNR else "DMX" - props[f"{prefix}_SSNR"] = min_ssnr - props[f"{prefix}_SSNR_CHANNEL"] = min_ssnr_channel + props[f"{prefix}_SSNR" + suffix] = min_ssnr + props[f"{prefix}_SSNR_CHANNEL" + suffix] = min_ssnr_channel elif prop == ODG: odgs = re.findall(ODG_PATTERN, text_to_parse) min_odg = min(odgs) min_odg_channel = odgs.index(min_odg) - props["MIN_ODG"] = min_odg - props["MIN_ODG_CHANNEL"] = min_odg_channel + props["MIN_ODG" + suffix] = min_odg + props["MIN_ODG_CHANNEL" + suffix] = min_odg_channel elif prop == MAX_ENC_DIFF: search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse) max_enc_diff_ratio = 0.0 max_enc_diff_param_name = "" if search_result: max_enc_diff_param_name, _, max_enc_diff_ratio = search_result.groups(0) - props[MAX_ENC_DIFF] = float(max_enc_diff_ratio) - props[MAX_ENC_DIFF_PARAM_NAME] = max_enc_diff_param_name + props[MAX_ENC_DIFF + suffix] = float(max_enc_diff_ratio) + props[MAX_ENC_DIFF_PARAM_NAME + suffix] = max_enc_diff_param_name elif prop == DELTA_ODG: delta_odg = re.search(DELTA_ODG_PATTERN, text_to_parse) if delta_odg: - props["DELTA_ODG"] = delta_odg.groups(1)[0] + props["DELTA_ODG" + suffix] = delta_odg.groups(1)[0] return props -- GitLab From 024f2a722f937b86133f82829bad8b5c783f4cfe Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 10:40:23 +0100 Subject: [PATCH 04/70] stereo-only config file for testing the split comparison --- ...self_test_ltv_basop_encoder_split_test.prm | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 scripts/config/self_test_ltv_basop_encoder_split_test.prm diff --git a/scripts/config/self_test_ltv_basop_encoder_split_test.prm b/scripts/config/self_test_ltv_basop_encoder_split_test.prm new file mode 100644 index 0000000000..724b8f67bc --- /dev/null +++ b/scripts/config/self_test_ltv_basop_encoder_split_test.prm @@ -0,0 +1,150 @@ +// Self-test parameter file +// +// - each test must have a tag (unique name) which must be entered as a comment (you can use // /* or rem comment) +// - the following line must be the encoder command line +// - the following line must be the decoder command line +// - if the name of the output file are exactly the same as +// the name of the test vector located in ./testv directory, these files will be compared for bit-exactness +// (the easiest way how to achieve this is to use the name of the test vector itself, as shown below) + + +// stereo bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_sw_32-32.tst + +// stereo at 96 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 96000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_96000_32-32.tst + +// stereo at 96 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 96000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_96000_16-16.tst + +// stereo at 64 kbps, 32kHz in, 32kHz out, bandwidth switching +../IVAS_cod -max_band testv/ivas_bws_20fr_start_FB.txt -stereo 64000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_64000_32-32.tst + +// stereo at 64 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 64000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_64000_16-16.tst + +// stereo at 48 kbps, 48 kHz in, 48 kHz out, DTX on, bandwidth switching +../IVAS_cod -max_band testv/ivas_bws_20fr_start_SWB.txt -stereo -dtx 48000 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_48000_48-48_DTX.tst + +// stereo at 48 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 48000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_48000_32-32.tst + +// stereo at 48 kbps, 32 kHz in, 32 kHz out, DTX on +../IVAS_cod -stereo -dtx 48000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_48000_32-32_DTX.tst + +// stereo at 48 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 48000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_48000_16-16.tst + +// stereo at 48 kbps, 16 kHz in, 16 kHz out, DTX on +../IVAS_cod -stereo -dtx 48000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_48000_16-16_DTX.tst + +// stereo at 32 kbps, 48kHz in, 48kHz out, bandwidth switching +../IVAS_cod -stereo -max_band testv/bwidth_cntl.txt 32000 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_32000_48-48_bandwidth_sw.tst + +// stereo at 32 kbps, 48kHz in, 48kHz out, DTX on, bandwidth switching +../IVAS_cod -max_band testv/ivas_bws_20fr_start_WB.txt -stereo -dtx 32000 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_32000_48-48_DTX.tst + +// stereo at 32 kbps, 32kHz in, 48kHz out, STEREO out +../IVAS_cod -stereo 32000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv32_STEREO.wav_stereo_32000_32-48_STEREO.tst + +// stereo at 32 kbps, 32kHz in, 32kHz out, DTX on, STEREO out +../IVAS_cod -stereo -dtx 32000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_32000_32-32_DTX_STEREO.tst + +// stereo at 32 kbps, 16kHz in, 16kHz out, DTX on +../IVAS_cod -dtx -stereo -dtx 32000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_32000_16-16_DTX.tst + +// stereo at 32 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 32000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_32000_16-16.tst + +// stereo at 24.4 kbps, 48kHz in, 48kHz out, DTX on +../IVAS_cod -stereo -dtx 24400 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_24400_48_48_DTX.txt + +// stereo at 24.4 kbps, 32kHz in, 32kHz out, DTX on +../IVAS_cod -stereo -dtx 24400 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_24400_32-32_DTX.tst + +// stereo at 24.4 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 24400 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_24400_32-32.tst + +// stereo at 24.4 kbps, 16kHz in, 16kHz out, DTX on, STEREO out +../IVAS_cod -stereo -dtx 24400 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_24400_16-16_DTX_STEREO.tst + +// stereo at 24.4 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 24400 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_24400_16-16.tst + +// stereo at 16.4 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 16400 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_16400_32-32.tst + +// stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on +../IVAS_cod -stereo -dtx 16400 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv32_STEREO.wav_stereo_16400_32-16_DTX.tst + +// stereo at 16.4 kbps, 16kHz in, 16kHz out, DTX on +../IVAS_cod -stereo -dtx 16400 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_16400_16-16_DTX.tst + +// stereo at 16.4 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 16400 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_16400_16-16.tst + +// stereo at 13.2 kbps, 48kHz in, 48kHz out +../IVAS_cod -stereo 13200 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_13200_48-48.tst + +// stereo at 13.2 kbps, 32kHz in, 32kHz out, DTX on +../IVAS_cod -stereo -dtx 13200 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_13200_32-32_DTX.tst + +// stereo at 13.2 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 13200 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_13200_32-32.tst + +// stereo at 13.2 kbps, 16kHz in, 16kHz out, DTX on +../IVAS_cod -stereo -dtx 13200 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_13200_16-16_DTX.tst + +// stereo at 13.2 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 13200 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_13200_16-16.tst + +// stereo at 128 kbps, 48kHz in, 48kHz out, STEREO out +../IVAS_cod -stereo 128000 48 testv/ltv48_STEREO.wav bit +../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_128000_48-48_STEREO.tst + +// stereo at 128 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo 128000 32 testv/ltv32_STEREO.wav bit +../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_128000_32-32.tst + +// stereo at 128 kbps, 16kHz in, 16kHz out +../IVAS_cod -stereo 128000 16 testv/ltv16_STEREO.wav bit +../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_128000_16-16.tst + +// Stereo downmix to bit-exact EVS at 24400 kbps, 48kHz in, 48kHz out +../IVAS_cod -stereo_dmx_evs 24400 48 testv/ltv48_STEREO.wav bit +../IVAS_dec 48 bit testv/ltv48_STEREO.wav_StereoDmxEVS_24400_48-48.tst + +// Stereo downmix to bit-exact EVS at 13200 kbps, 32kHz in, 32kHz out +../IVAS_cod -stereo_dmx_evs 13200 32 testv/ltv32_STEREO.wav bit +../IVAS_dec 32 bit testv/ltv32_STEREO.wav_StereoDmxEVS_13200_32-32.tst + -- GitLab From 5bf37d9176b713a9d2002674e00c57122ee56797 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Mar 2025 11:12:09 +0100 Subject: [PATCH 05/70] use format as key for split indices --- tests/conftest.py | 3 ++- tests/constants.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d3717db395..238c50432f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1262,7 +1262,8 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra If no list of indices is available for the given input file, an empty array is returned. """ - idx = SPLIT_IDX.get(input_file.lower(), np.empty(0)) + format = input_file.replace(".wav", "").split("_")[-1].lower() + idx = SPLIT_IDX.get(format, np.empty(0)) if len(idx) > 0 and sampling_rate_khz != 16: idx *= sampling_rate_khz // 16 diff --git a/tests/constants.py b/tests/constants.py index 13f46b8553..155d9bdf1d 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -101,4 +101,4 @@ SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( ] ) -SPLIT_IDX = {"ltv16_stereo": SPLIT_IDX_LTV_STEREO_16KHZ} +SPLIT_IDX = {"stereo": SPLIT_IDX_LTV_STEREO_16KHZ} -- GitLab From 16adf89dcb611fd5ae555f588151a325a9c5e1c3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Mar 2025 17:38:24 +0100 Subject: [PATCH 06/70] add split markers for ism files --- tests/constants.py | 60 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/tests/constants.py b/tests/constants.py index 155d9bdf1d..6413b5f539 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -100,5 +100,63 @@ SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( 3219792, ] ) +SPLIT_IDX_LTV_ISM1_16KHZ = np.asarray( + [ + 72000, + 164800, + 233600, + 422400, + 492800, + 592000, + 720000, + 792000, + 873600, + 947200, + 1001600, + 1060805, + 1120000, + 1284809, + 1356800, + 1448000, + 1513600, + 1731200, + 1828480, + 1910400, + 1966409, + 2150400, + 2235088, + 2299552, + 2460800, + ] +) +SPLIT_IDX_LTV_ISM234_16KHZ = np.asarray( + [ + 72000, + 164800, + 233600, + 432000, + 592000, + 720000, + 792000, + 873600, + 937600, + 1216000, + 1448000, + 1513600, + 1731200, + 1828480, + 1931200, + 2156800, + 2235088, + 2299552, + 2460800, + ] +) -SPLIT_IDX = {"stereo": SPLIT_IDX_LTV_STEREO_16KHZ} +SPLIT_IDX = { + "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, + "1ism": SPLIT_IDX_LTV_ISM1_16KHZ, + "2ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "3ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "4ism": SPLIT_IDX_LTV_ISM234_16KHZ, +} -- GitLab From 842de937e300d349580185a5593dc32059f8b9e9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Mar 2025 17:40:30 +0100 Subject: [PATCH 07/70] add markers for mono --- tests/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/constants.py b/tests/constants.py index 6413b5f539..39955e377f 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -155,6 +155,7 @@ SPLIT_IDX_LTV_ISM234_16KHZ = np.asarray( SPLIT_IDX = { "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, + "mono": SPLIT_IDX_LTV_STEREO_16KHZ, "1ism": SPLIT_IDX_LTV_ISM1_16KHZ, "2ism": SPLIT_IDX_LTV_ISM234_16KHZ, "3ism": SPLIT_IDX_LTV_ISM234_16KHZ, -- GitLab From 47ebf75a6ee5f60b1380e600dcb5b40773b9ee08 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 19 Mar 2025 17:41:07 +0100 Subject: [PATCH 08/70] Add handling of split measurements in parse_xml_report.py --- scripts/parse_xml_report.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 1f2157ca8f..fc253bb9d6 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -136,7 +136,17 @@ if __name__ == "__main__": else: testresult = "PASS" - properties_values = [str(properties_found.get(p)) for p in PROPERTIES] + # Extract number of splits, if any + splits = [p.split('_')[-1] for p in properties_found if 'split' in p] + splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order + + if splits: + properties_values = {} + for s in splits: + properties_suffixed = [p + '_' + s for p in PROPERTIES] + properties_values[s] = [str(properties_found.get(p)) for p in properties_suffixed] + else: + properties_values = [str(properties_found.get(p)) for p in PROPERTIES] # Identify format and category (mode of operation) # For the format, favor the earliest match in the test case name @@ -158,9 +168,16 @@ if __name__ == "__main__": # For ERROR cases, both a FAIL and an ERROR result is generated. # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - results[fmt][cat][fulltestname] = {"Result": testresult} - for propertyname, propertyvalue in zip(PROPERTIES, properties_values): - results[fmt][cat][fulltestname][propertyname] = propertyvalue + if splits: + for s in splits: + fulltestname_split = f"{fulltestname}-{s}" + results[fmt][cat][fulltestname_split] = {"Result": testresult} + for propertyname, propertyvalue in zip(PROPERTIES, properties_values[s]): + results[fmt][cat][fulltestname_split][propertyname] = propertyvalue + else: + results[fmt][cat][fulltestname] = {"Result": testresult} + for propertyname, propertyvalue in zip(PROPERTIES, properties_values): + results[fmt][cat][fulltestname][propertyname] = propertyvalue header = ["testcase", "Format", "Category", "Result"] + PROPERTIES @@ -172,7 +189,11 @@ if __name__ == "__main__": for cat in CATEGORIES: results[fmt][cat] = dict(sorted(results[fmt][cat].items())) for test in results[fmt][cat]: - count[results[fmt][cat][test]["Result"]] += 1 + if splits: + if "split1" in test: + count[results[fmt][cat][test]["Result"]] += 1 + else: + count[results[fmt][cat][test]["Result"]] += 1 line = ( ";".join( [test, fmt, cat] + list(results[fmt][cat][test].values()) -- GitLab From fa52449220b84a181135ccd1a2172ab43f0446a2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Mar 2025 16:45:55 +0100 Subject: [PATCH 09/70] add split marker for MC, MASA, SBA --- tests/constants.py | 147 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/tests/constants.py b/tests/constants.py index 39955e377f..939236b08f 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -152,6 +152,142 @@ SPLIT_IDX_LTV_ISM234_16KHZ = np.asarray( 2460800, ] ) +SPLIT_IDX_LTV_FOA_16KHZ = np.asarray( + [ + 111840, + 176320, + 231200, + 327280, + 359280, + 404384, + 452800, + 465600, + 496272, + 565632, + 735712, + 846624, + 1007184, + 1166720, + 1316640, + 1391840, + 1699200, + 1716800, + 1812640, + 1908640, + 2036304, + 2196320, + 2364592, + 2524624, + ] +) +SPLIT_IDX_LTV_HOA3_16KHZ = np.asarray( + [ + 69328, + 169120, + 339200, + 450144, + 610128, + 685120, + 759600, + 845120, + 930144, + 1090144, + 1240128, + 1313600, + 1619200, + 1640000, + 1704416, + 1759360, + 1855360, + 1887360, + 1932480, + 1982400, + 1993600, + 2024640, + 2136128, + 2232400, + 2328480, + 2424640, + ] +) +SPLIT_IDX_LTV_MASA_1TC_16KHZ = np.asarray( + [ + 112000, + 192012, + 288000, + 376008, + 528000, + 659200, + 716803, + 822406, + 892800, + 1024002, + 1120000, + 1220789, + 1316831, + 1444183, + 1588641, + 1719739, + ] +) +SPLIT_IDX_LTV_MASA_2TC_16KHZ = np.asarray( + [ + 119984, + 255588, + 286417, + 480008, + 560002, + 625505, + 716938, + 944138, + 1037348, + 1076332, + 1131937, + 1295806, + 1479906, + 1616003, + 1743895, + 1792355, + 1920075, + 1968066, + 2112000, + 2207637, + 2317218, + 2436529, + 2613994, + ] +) +SPLIT_IDX_LTV_MC_16KHZ = np.asarray( + [ + 69328, + 169120, + 339200, + 450144, + 610128, + 685120, + 759600, + 845120, + 930144, + 1090144, + 1240128, + 1313600, + 1619200, + 1640000, + 1704416, + 1759360, + 1855360, + 1887360, + 1932480, + 1982400, + 1993600, + 2024640, + 2136128, + 2232400, + 2328480, + 2424640, + 2552400, + ] +) SPLIT_IDX = { "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, @@ -160,4 +296,15 @@ SPLIT_IDX = { "2ism": SPLIT_IDX_LTV_ISM234_16KHZ, "3ism": SPLIT_IDX_LTV_ISM234_16KHZ, "4ism": SPLIT_IDX_LTV_ISM234_16KHZ, + "foa": SPLIT_IDX_LTV_FOA_16KHZ, + "hoa2": SPLIT_IDX_LTV_HOA3_16KHZ, + "hoa3": SPLIT_IDX_LTV_HOA3_16KHZ, + "masa1tc": SPLIT_IDX_LTV_MASA_1TC_16KHZ, + "masa2tc": SPLIT_IDX_LTV_MASA_2TC_16KHZ, + "mc51": SPLIT_IDX_LTV_MC_16KHZ, + # there is always one signal that does something different... + "mc512": SPLIT_IDX_LTV_MC_16KHZ[:-1], + "mc514": SPLIT_IDX_LTV_MC_16KHZ, + "mc71": SPLIT_IDX_LTV_MC_16KHZ, + "mc714": SPLIT_IDX_LTV_MC_16KHZ, } -- GitLab From 2d481a897e16acebd36ad1a277ed63f867197e4e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Mar 2025 17:34:45 +0100 Subject: [PATCH 10/70] split markers for OMASA --- tests/conftest.py | 6 ++- tests/constants.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 238c50432f..0ff2df3479 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1262,7 +1262,11 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra If no list of indices is available for the given input file, an empty array is returned. """ - format = input_file.replace(".wav", "").split("_")[-1].lower() + input_file = input_file.lower() + if "omasa" in input_file: + format = "_".join(input_file.replace(".wav", "").split("_")[:-1]) + else: + format = input_file.replace(".wav", "").split("_")[-1].lower() idx = SPLIT_IDX.get(format, np.empty(0)) if len(idx) > 0 and sampling_rate_khz != 16: diff --git a/tests/constants.py b/tests/constants.py index 939236b08f..296c3e7556 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -288,6 +288,97 @@ SPLIT_IDX_LTV_MC_16KHZ = np.asarray( 2552400, ] ) +SPLIT_IDX_LTV_OMASA_1ISM_16KHZ = np.asarray( + [ + 182561, + 250497, + 321192, + 398875, + 462969, + 589104, + 634642, + 704772, + 761550, + 841565, + 905594, + 1077445, + 1147515, + 1204145, + 1284198, + 1348213, + 1474397, + 1519986, + 1590029, + 1686404, + 1726821, + 1790877, + 1872012, + ] +) +SPLIT_IDX_LTV_OMASA_2ISM_16KHZ = np.asarray( + [ + 104025, + 188810, + 248000, + 600000, + 727048, + 832000, + 952951, + 970604, + 1037526, + 1104909, + 1233053, + 1295925, + 1361517, + 1414356, + 1476020, + 1549127, + 1605498, + 1727087, + 1876418, + ] +) +SPLIT_IDX_LTV_OMASA_3ISM_16KHZ = np.asarray( + [ + 155662, + 238144, + 340776, + 415986, + 575167, + 707032, + 832054, + 975997, + 1104017, + 1197434, + 1247993, + 1344330, + 1482134, + 1549575, + 1688766, + 1745715, + ] +) +SPLIT_IDX_LTV_OMASA_4ISM_16KHZ = np.asarray( + [ + 63985, + 192015, + 316768, + 391082, + 500584, + 597477, + 719995, + 838699, + 952531, + 1040041, + 1143121, + 1271008, + 1503986, + 1626407, + 1687966, + 1796807, + 1878304, + ] +) SPLIT_IDX = { "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, @@ -307,4 +398,8 @@ SPLIT_IDX = { "mc514": SPLIT_IDX_LTV_MC_16KHZ, "mc71": SPLIT_IDX_LTV_MC_16KHZ, "mc714": SPLIT_IDX_LTV_MC_16KHZ, + "omasa_1ism": SPLIT_IDX_LTV_OMASA_1ISM_16KHZ, + "omasa_2ism": SPLIT_IDX_LTV_OMASA_2ISM_16KHZ, + "omasa_3ism": SPLIT_IDX_LTV_OMASA_3ISM_16KHZ, + "omasa_4ism": SPLIT_IDX_LTV_OMASA_4ISM_16KHZ, } -- GitLab From c74b0e7575459b4feabc04bc61d787afd6c1daba Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 10:03:54 +0100 Subject: [PATCH 11/70] add osba split markers --- tests/conftest.py | 5 +++++ tests/constants.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 0ff2df3479..a6e3e7596c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1265,6 +1265,11 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra input_file = input_file.lower() if "omasa" in input_file: format = "_".join(input_file.replace(".wav", "").split("_")[:-1]) + elif "osba" in input_file: + if "foa" in input_file: + format = "osba_foa" + else: + format = "osba_hoa" else: format = input_file.replace(".wav", "").split("_")[-1].lower() idx = SPLIT_IDX.get(format, np.empty(0)) diff --git a/tests/constants.py b/tests/constants.py index 296c3e7556..caa2c67247 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -379,6 +379,58 @@ SPLIT_IDX_LTV_OMASA_4ISM_16KHZ = np.asarray( 1878304, ] ) +SPLIT_IDX_LTV_OSBA_FOA_16KHZ = np.asarray( + [ + 110243, + 176353, + 231257, + 327228, + 406430, + 453959, + 465000, + 496215, + 565603, + 735726, + 846637, + 1166658, + 1316644, + 1391826, + 1697613, + 1716813, + 1812764, + 1908681, + 2036312, + 2193583, + 2361556, + ] +) +SPLIT_IDX_LTV_OSBA_HOA_16KHZ = np.asarray( + [ + 69324, + 169120, + 339217, + 450133, + 739726, + 930163, + 1090182, + 1240385, + 1316803, + 1617775, + 1640120, + 1704436, + 1759348, + 1854498, + 1887201, + 1932676, + 1983352, + 1991890, + 2024762, + 2136276, + 2231825, + 2328364, + 2424105, + ] +) SPLIT_IDX = { "stereo": SPLIT_IDX_LTV_STEREO_16KHZ, @@ -402,4 +454,6 @@ SPLIT_IDX = { "omasa_2ism": SPLIT_IDX_LTV_OMASA_2ISM_16KHZ, "omasa_3ism": SPLIT_IDX_LTV_OMASA_3ISM_16KHZ, "omasa_4ism": SPLIT_IDX_LTV_OMASA_4ISM_16KHZ, + "osba_foa": SPLIT_IDX_LTV_OSBA_FOA_16KHZ, + "osba_hoa": SPLIT_IDX_LTV_OSBA_HOA_16KHZ, } -- GitLab From 9344fdf23bc2a41a255a53eaf32eb4fab8b41239 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 11:08:31 +0100 Subject: [PATCH 12/70] run formatter on parse_xml_report.py --- scripts/parse_xml_report.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index fc253bb9d6..6308dbe71f 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -2,8 +2,6 @@ import argparse import re -import math -import numpy as np from xml.etree import ElementTree """ @@ -81,7 +79,7 @@ if __name__ == "__main__": "--skip_formats", action="store_true", help="Parse without formats and categories. Suitable for general tests which do not match the IVAS categories.", - ) + ) args = parser.parse_args() xml_report = args.xml_report csv_file = args.csv_file @@ -94,7 +92,7 @@ if __name__ == "__main__": FORMATS = IVAS_FORMATS CATEGORIES = IVAS_CATEGORIES if args.clipping: - PROPERTIES += ["ENC_CORE_OVL","MAX_OVL","MIN_OVL"] + PROPERTIES += ["ENC_CORE_OVL", "MAX_OVL", "MIN_OVL"] if args.delta_odg: PROPERTIES += ["DELTA_ODG"] if args.skip_formats: @@ -137,14 +135,16 @@ if __name__ == "__main__": testresult = "PASS" # Extract number of splits, if any - splits = [p.split('_')[-1] for p in properties_found if 'split' in p] - splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order + splits = [p.split("_")[-1] for p in properties_found if "split" in p] + splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order if splits: properties_values = {} for s in splits: - properties_suffixed = [p + '_' + s for p in PROPERTIES] - properties_values[s] = [str(properties_found.get(p)) for p in properties_suffixed] + properties_suffixed = [p + "_" + s for p in PROPERTIES] + properties_values[s] = [ + str(properties_found.get(p)) for p in properties_suffixed + ] else: properties_values = [str(properties_found.get(p)) for p in PROPERTIES] @@ -172,8 +172,12 @@ if __name__ == "__main__": for s in splits: fulltestname_split = f"{fulltestname}-{s}" results[fmt][cat][fulltestname_split] = {"Result": testresult} - for propertyname, propertyvalue in zip(PROPERTIES, properties_values[s]): - results[fmt][cat][fulltestname_split][propertyname] = propertyvalue + for propertyname, propertyvalue in zip( + PROPERTIES, properties_values[s] + ): + results[fmt][cat][fulltestname_split][propertyname] = ( + propertyvalue + ) else: results[fmt][cat][fulltestname] = {"Result": testresult} for propertyname, propertyvalue in zip(PROPERTIES, properties_values): @@ -191,8 +195,8 @@ if __name__ == "__main__": for test in results[fmt][cat]: if splits: if "split1" in test: - count[results[fmt][cat][test]["Result"]] += 1 - else: + count[results[fmt][cat][test]["Result"]] += 1 + else: count[results[fmt][cat][test]["Result"]] += 1 line = ( ";".join( -- GitLab From 40ec8adf9cddc459554f193a18021197d913f99f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 11:19:17 +0100 Subject: [PATCH 13/70] remove prm file for testing --- ...self_test_ltv_basop_encoder_split_test.prm | 150 ------------------ 1 file changed, 150 deletions(-) delete mode 100644 scripts/config/self_test_ltv_basop_encoder_split_test.prm diff --git a/scripts/config/self_test_ltv_basop_encoder_split_test.prm b/scripts/config/self_test_ltv_basop_encoder_split_test.prm deleted file mode 100644 index 724b8f67bc..0000000000 --- a/scripts/config/self_test_ltv_basop_encoder_split_test.prm +++ /dev/null @@ -1,150 +0,0 @@ -// Self-test parameter file -// -// - each test must have a tag (unique name) which must be entered as a comment (you can use // /* or rem comment) -// - the following line must be the encoder command line -// - the following line must be the decoder command line -// - if the name of the output file are exactly the same as -// the name of the test vector located in ./testv directory, these files will be compared for bit-exactness -// (the easiest way how to achieve this is to use the name of the test vector itself, as shown below) - - -// stereo bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_sw_32-32.tst - -// stereo at 96 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 96000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_96000_32-32.tst - -// stereo at 96 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 96000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_96000_16-16.tst - -// stereo at 64 kbps, 32kHz in, 32kHz out, bandwidth switching -../IVAS_cod -max_band testv/ivas_bws_20fr_start_FB.txt -stereo 64000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_64000_32-32.tst - -// stereo at 64 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 64000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_64000_16-16.tst - -// stereo at 48 kbps, 48 kHz in, 48 kHz out, DTX on, bandwidth switching -../IVAS_cod -max_band testv/ivas_bws_20fr_start_SWB.txt -stereo -dtx 48000 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_48000_48-48_DTX.tst - -// stereo at 48 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 48000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_48000_32-32.tst - -// stereo at 48 kbps, 32 kHz in, 32 kHz out, DTX on -../IVAS_cod -stereo -dtx 48000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_48000_32-32_DTX.tst - -// stereo at 48 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 48000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_48000_16-16.tst - -// stereo at 48 kbps, 16 kHz in, 16 kHz out, DTX on -../IVAS_cod -stereo -dtx 48000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_48000_16-16_DTX.tst - -// stereo at 32 kbps, 48kHz in, 48kHz out, bandwidth switching -../IVAS_cod -stereo -max_band testv/bwidth_cntl.txt 32000 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_32000_48-48_bandwidth_sw.tst - -// stereo at 32 kbps, 48kHz in, 48kHz out, DTX on, bandwidth switching -../IVAS_cod -max_band testv/ivas_bws_20fr_start_WB.txt -stereo -dtx 32000 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_32000_48-48_DTX.tst - -// stereo at 32 kbps, 32kHz in, 48kHz out, STEREO out -../IVAS_cod -stereo 32000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv32_STEREO.wav_stereo_32000_32-48_STEREO.tst - -// stereo at 32 kbps, 32kHz in, 32kHz out, DTX on, STEREO out -../IVAS_cod -stereo -dtx 32000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_32000_32-32_DTX_STEREO.tst - -// stereo at 32 kbps, 16kHz in, 16kHz out, DTX on -../IVAS_cod -dtx -stereo -dtx 32000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_32000_16-16_DTX.tst - -// stereo at 32 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 32000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_32000_16-16.tst - -// stereo at 24.4 kbps, 48kHz in, 48kHz out, DTX on -../IVAS_cod -stereo -dtx 24400 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_24400_48_48_DTX.txt - -// stereo at 24.4 kbps, 32kHz in, 32kHz out, DTX on -../IVAS_cod -stereo -dtx 24400 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_24400_32-32_DTX.tst - -// stereo at 24.4 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 24400 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_24400_32-32.tst - -// stereo at 24.4 kbps, 16kHz in, 16kHz out, DTX on, STEREO out -../IVAS_cod -stereo -dtx 24400 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_24400_16-16_DTX_STEREO.tst - -// stereo at 24.4 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 24400 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_24400_16-16.tst - -// stereo at 16.4 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 16400 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_16400_32-32.tst - -// stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on -../IVAS_cod -stereo -dtx 16400 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv32_STEREO.wav_stereo_16400_32-16_DTX.tst - -// stereo at 16.4 kbps, 16kHz in, 16kHz out, DTX on -../IVAS_cod -stereo -dtx 16400 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_16400_16-16_DTX.tst - -// stereo at 16.4 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 16400 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_16400_16-16.tst - -// stereo at 13.2 kbps, 48kHz in, 48kHz out -../IVAS_cod -stereo 13200 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_13200_48-48.tst - -// stereo at 13.2 kbps, 32kHz in, 32kHz out, DTX on -../IVAS_cod -stereo -dtx 13200 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_13200_32-32_DTX.tst - -// stereo at 13.2 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 13200 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_13200_32-32.tst - -// stereo at 13.2 kbps, 16kHz in, 16kHz out, DTX on -../IVAS_cod -stereo -dtx 13200 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_13200_16-16_DTX.tst - -// stereo at 13.2 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 13200 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_13200_16-16.tst - -// stereo at 128 kbps, 48kHz in, 48kHz out, STEREO out -../IVAS_cod -stereo 128000 48 testv/ltv48_STEREO.wav bit -../IVAS_dec STEREO 48 bit testv/ltv48_STEREO.wav_stereo_128000_48-48_STEREO.tst - -// stereo at 128 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo 128000 32 testv/ltv32_STEREO.wav bit -../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_128000_32-32.tst - -// stereo at 128 kbps, 16kHz in, 16kHz out -../IVAS_cod -stereo 128000 16 testv/ltv16_STEREO.wav bit -../IVAS_dec STEREO 16 bit testv/ltv16_STEREO.wav_stereo_128000_16-16.tst - -// Stereo downmix to bit-exact EVS at 24400 kbps, 48kHz in, 48kHz out -../IVAS_cod -stereo_dmx_evs 24400 48 testv/ltv48_STEREO.wav bit -../IVAS_dec 48 bit testv/ltv48_STEREO.wav_StereoDmxEVS_24400_48-48.tst - -// Stereo downmix to bit-exact EVS at 13200 kbps, 32kHz in, 32kHz out -../IVAS_cod -stereo_dmx_evs 13200 32 testv/ltv32_STEREO.wav bit -../IVAS_dec 32 bit testv/ltv32_STEREO.wav_StereoDmxEVS_13200_32-32.tst - -- GitLab From e9bb84c3dc60f72d94f0eaba0962e5cc3152ea99 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 12:51:34 +0100 Subject: [PATCH 14/70] add split comp for test_sba testcases also fix running without --split_comparison --- tests/codec_be_on_mr_nonselection/test_sba.py | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index ce6d073942..d97cf9fa90 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -35,6 +35,7 @@ __doc__ = """ import os import pytest +import numpy as np from cut_bs import cut_from_start from pathlib import Path @@ -44,6 +45,7 @@ from tests.conftest import ( EncoderFrontend, compare_dmx_signals, parse_properties, + get_split_idx, ) from ..cmp_stats_files import cmp_stats_files from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF @@ -116,6 +118,7 @@ def test_pca_enc( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): pca = True bitrate = "256000" @@ -198,6 +201,7 @@ def test_pca_enc( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -237,6 +241,7 @@ def test_sba_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): plc_pattern = None pca = False @@ -377,6 +382,7 @@ def test_sba_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -408,6 +414,7 @@ def test_spar_hoa2_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sampling_rate = "48" pca = False @@ -523,6 +530,7 @@ def test_spar_hoa2_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -554,6 +562,7 @@ def test_spar_hoa3_enc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sampling_rate = "48" pca = False @@ -663,6 +672,7 @@ def test_spar_hoa3_enc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -698,6 +708,7 @@ def test_sba_enc_BWforce_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sid = 0 plc_pattern = None @@ -823,6 +834,7 @@ def test_sba_enc_BWforce_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -862,6 +874,7 @@ def test_sba_plc_system( get_enc_stats, compare_to_input, compare_enc_dmx, + split_comparison, ): sid = 0 pca = False @@ -964,6 +977,7 @@ def test_sba_plc_system( get_odg=get_odg, get_odg_bin=get_odg_bin, compare_to_input=compare_to_input, + split_comparison=split_comparison, ) @@ -1140,6 +1154,7 @@ def sba_dec( get_odg=False, get_odg_bin=False, compare_to_input=False, + split_comparison=False, ): dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" ref_pkt_dir = f"{reference_path}/sba_bs/pkt" @@ -1251,7 +1266,13 @@ def sba_dec( allow_differing_lengths = True sampling_rate_Hz = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm( + + split_idx = np.empty(0) + if split_comparison: + input_file = f"{test_vector_path}/{tag}.wav" + split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + + output_differs_parts, reason_parts = cmp_pcm( dut_out_file, ref_out_file, output_config, @@ -1267,13 +1288,19 @@ def sba_dec( odg_test=odg_test, odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - text_to_parse = reason - props = parse_properties(text_to_parse, cmp_result != 0, props_to_record) - for k, v in props.items(): - dut_decoder_frontend.record_property(k, v) + prop_suffix = [""] + if len(split_idx) > 0: + prop_suffix = [f"_split{i}" for i in range(1, len(split_idx) + 1)] + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + props = parse_properties(reason, output_differs, props_to_record, suffix) + for k, v in props.items(): + dut_decoder_frontend.record_property(k, v) # report compare result - if cmp_result != 0: - pytest.fail(text_to_parse) + if output_differs_parts[0] != 0: + pytest.fail(reason_parts[0]) -- GitLab From ef9c6f989a53a88029dc77dcdca056a5873482b0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 13:57:05 +0100 Subject: [PATCH 15/70] add comparison of whole file along with splits --- .../test_param_file.py | 39 +++++++++++++++--- tests/codec_be_on_mr_nonselection/test_sba.py | 40 ++++++++++++++++--- 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index c9854185ac..a9642d3ba0 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -593,9 +593,11 @@ def run_test( fs = int(sampling_rate) * 1000 + ### run the comparison tools split_idx = np.empty(0) - if split_comparison: - split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison output_differs_parts, reason_parts = cmp_pcm( ref_file, @@ -618,9 +620,36 @@ def run_test( split_idx=split_idx, ) - prop_suffix = [""] - if len(split_idx) > 0: - prop_suffix = [f"_split{i}" for i in range(1, len(split_idx) + 1)] + # 2. run comparison on split files if --split_comparison is given + if split_comparison: + split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + + output_differs_splits, reason_splits = cmp_pcm( + ref_file, + dut_output_file, + out_config_2_nchannels(output_config), + fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + ref_jbm_tf=ref_tracefile_dec, + cut_jbm_tf=dut_tracefile_dec, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + prop_suffix = ["_whole"] + [ + f"_split{i}" for i in range(1, len(split_idx) + 1) + ] for output_differs, reason, suffix in zip( output_differs_parts, reason_parts, prop_suffix diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index d97cf9fa90..65c68ae6a6 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1267,10 +1267,11 @@ def sba_dec( sampling_rate_Hz = int(sampling_rate) * 1000 + ### run the comparison tools split_idx = np.empty(0) - if split_comparison: - input_file = f"{test_vector_path}/{tag}.wav" - split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison output_differs_parts, reason_parts = cmp_pcm( dut_out_file, @@ -1291,9 +1292,36 @@ def sba_dec( split_idx=split_idx, ) - prop_suffix = [""] - if len(split_idx) > 0: - prop_suffix = [f"_split{i}" for i in range(1, len(split_idx) + 1)] + # 2. run comparison on split files if --split_comparison is given + if split_comparison: + input_file = f"{test_vector_path}/{tag}.wav" + split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + + output_differs_splits, reason_splits = cmp_pcm( + dut_out_file, + ref_out_file, + output_config, + sampling_rate_Hz, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + prop_suffix = ["_whole"] + [ + f"_split{i}" for i in range(1, len(split_idx) + 1) + ] + for output_differs, reason, suffix in zip( output_differs_parts, reason_parts, prop_suffix ): -- GitLab From 33bb409c158445c41c27a31a19e8c317d0729ae3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 14:48:01 +0100 Subject: [PATCH 16/70] fix other usages of cmp_pcm --- tests/cmp_pcm.py | 4 +- tests/conftest.py | 2 + tests/renderer/utils.py | 81 ++++++++++++++++++++++------------------- tests/test_26444.py | 2 + 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index e254f1d59a..56dc5cd63c 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -283,5 +283,5 @@ if __name__ == "__main__": args["nchannels"] = out_config_2_nchannels(args.pop("out_config")) result, msg = cmp_pcm(**args) - print(msg) - sys.exit(result) + print(msg[0]) + sys.exit(result[0]) diff --git a/tests/conftest.py b/tests/conftest.py index a6e3e7596c..1d5295d599 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1249,6 +1249,8 @@ def compare_dmx_signals(ref_dmx_files, dut_dmx_files, fs) -> dict: get_ssnr=True, quiet=True, ) + dmx_differs = dmx_differs[0] + reason = reason[0] dmx_props = [DMX_DIFF, DMX_MLD, DMX_SSNR] prop_results = parse_properties(reason, dmx_differs, dmx_props) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 2be90224a6..d2246abf40 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -325,37 +325,37 @@ def run_renderer( odg_ref = out_file_ref[0:-4] + ".BINAURAL.wav" if out_fmt not in PEAQ_SUPPORTED_FMT: - if in_fmt in PEAQ_SUPPORTED_FMT: - new_fmt = in_fmt # MONO or STEREO + if in_fmt in PEAQ_SUPPORTED_FMT: + new_fmt = in_fmt # MONO or STEREO + else: + # If input is META which contains stereo, new_fmt needs to be STEREO. + if in_fmt == "META": + with open(in_file, "r") as scene: + if "STEREO" in scene.read(): + new_fmt = "STEREO" + else: + new_fmt = "BINAURAL" else: - # If input is META which contains stereo, new_fmt needs to be STEREO. - if in_fmt == "META": - with open(in_file,"r") as scene: - if "STEREO" in scene.read(): - new_fmt = "STEREO" - else: - new_fmt = "BINAURAL" - else: - new_fmt = "BINAURAL" - - # Render test to PEAQ supported format (MONO, STEREO or BINAURAL) - cmd2 = RENDERER_CMD[:] - cmd2[2] = str(out_file) # in_file - cmd2[4] = str(out_fmt) # in_fmt - cmd2[6] = odg_test # out_file - cmd2[8] = new_fmt # out_fmt - cmd2[10] = str(sr) - cmd2[0] += BIN_SUFFIX_MERGETARGET # Use IVAS_rend_ref for re-rendering - cmd2[0] += binary_suffix - if "MASA" in str(out_fmt): - cmd2.extend(["-im", out_file + ".met"]) - run_cmd(cmd2, test_info, env) - - # Render ref to BINAURAL with same settings as test - cmd2[2] = str(out_file_ref) # in_file - cmd2[6] = odg_ref # out_file - run_cmd(cmd2, test_info, env) - out_fmt_bin = new_fmt + new_fmt = "BINAURAL" + + # Render test to PEAQ supported format (MONO, STEREO or BINAURAL) + cmd2 = RENDERER_CMD[:] + cmd2[2] = str(out_file) # in_file + cmd2[4] = str(out_fmt) # in_fmt + cmd2[6] = odg_test # out_file + cmd2[8] = new_fmt # out_fmt + cmd2[10] = str(sr) + cmd2[0] += BIN_SUFFIX_MERGETARGET # Use IVAS_rend_ref for re-rendering + cmd2[0] += binary_suffix + if "MASA" in str(out_fmt): + cmd2.extend(["-im", out_file + ".met"]) + run_cmd(cmd2, test_info, env) + + # Render ref to BINAURAL with same settings as test + cmd2[2] = str(out_file_ref) # in_file + cmd2[6] = odg_ref # out_file + run_cmd(cmd2, test_info, env) + out_fmt_bin = new_fmt else: out_fmt_bin = out_fmt odg_test = out_file @@ -365,8 +365,8 @@ def run_renderer( # Render input to match out_fmt_bin using same config as input, but with IVAS_rend_ref cmd[0] += BIN_SUFFIX_MERGETARGET cmd[0] += binary_suffix - cmd[6] = odg_input # out_file - cmd[8] = out_fmt_bin # out_fmt + cmd[6] = odg_input # out_file + cmd[8] = out_fmt_bin # out_fmt run_cmd(cmd, test_info, env) else: odg_input = in_file @@ -389,6 +389,8 @@ def run_renderer( odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, ) + output_differs = output_differs[0] + reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) for k, v in props.items(): @@ -455,7 +457,6 @@ def binauralize_input_and_output( in_sr, out_sr, ): - # Use current folder as location for temporary directory, since scene description does not handle spaces in path with tempfile.TemporaryDirectory(dir=".") as tmp_dir: tmp_dir = Path(tmp_dir) @@ -464,7 +465,7 @@ def binauralize_input_and_output( scene_in = str(tmp_dir.joinpath("scene_in.txt")) # File names for binauralized signals, if needed - ref_input_file_binaural = ref_output_file[0:-4] + ".INPUT.BINAURAL.wav" + ref_input_file_binaural = ref_output_file[0:-4] + ".INPUT.BINAURAL.wav" dut_output_file_binaural = dut_output_file[0:-4] + ".BINAURAL.wav" ref_output_file_binaural = ref_output_file[0:-4] + ".BINAURAL.wav" @@ -492,7 +493,7 @@ def binauralize_input_and_output( ",".join(line.split(",")[:2]) + "\n" ) # Keep only first two elements: azim, elev else: - md_out_file = "NULL" # Cannot truncate NULL, just insert it without modification + md_out_file = "NULL" # Cannot truncate NULL, just insert it without modification truncated_meta_files.append(md_out_file) in_meta_files = truncated_meta_files @@ -522,7 +523,9 @@ def binauralize_input_and_output( if "OSBA" in output_config or "OMASA" in output_config: if "OSBA" in output_config: - output_config = output_config[:-1] + '3' # Temporary fix to handle than IVAS_dec produces HOA3 for all OSBA configs. Needs to be removed when this fix is ported to BASOP. + output_config = ( + output_config[:-1] + "3" + ) # Temporary fix to handle than IVAS_dec produces HOA3 for all OSBA configs. Needs to be removed when this fix is ported to BASOP. scene_description_file( output_config, scene_dut, n_obj, dut_output_file, out_meta_files ) @@ -656,7 +659,11 @@ def binauralize_input_and_output( ) else: ref_input_file_binaural = input_file - return (ref_input_file_binaural, dut_output_file_binaural, ref_output_file_binaural) + return ( + ref_input_file_binaural, + dut_output_file_binaural, + ref_output_file_binaural, + ) def findstr(exp, s, one_element=True): diff --git a/tests/test_26444.py b/tests/test_26444.py index 054242c4c9..a1579dfec9 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -176,6 +176,8 @@ def test_evs_26444( get_ssnr=get_ssnr, get_odg=get_odg, ) + output_differs = output_differs[0] + reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) for k, v in props.items(): -- GitLab From 5920a4cdee685a4255d32944ef7d5b12cfd6c93a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 16:57:08 +0100 Subject: [PATCH 17/70] add padding zeros to split index --- tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +- tests/codec_be_on_mr_nonselection/test_sba.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index a9642d3ba0..3c9095db79 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -648,7 +648,7 @@ def run_test( reason_parts += reason_splits prop_suffix = ["_whole"] + [ - f"_split{i}" for i in range(1, len(split_idx) + 1) + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] for output_differs, reason, suffix in zip( diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 65c68ae6a6..76f42471a1 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1319,7 +1319,7 @@ def sba_dec( reason_parts += reason_splits prop_suffix = ["_whole"] + [ - f"_split{i}" for i in range(1, len(split_idx) + 1) + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] for output_differs, reason, suffix in zip( -- GitLab From 6be165b846d8675798835b3b43914e0e5fd304df Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 17:17:59 +0100 Subject: [PATCH 18/70] filter out skipped cases b4 loop for one less indent --- scripts/parse_xml_report.py | 138 ++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 6308dbe71f..f8577a7560 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -112,76 +112,76 @@ if __name__ == "__main__": results[fmt][cat] = {} count = {"PASS": 0, "FAIL": 0, "ERROR": 0} + # filter out skipped testcases + testcases = [tc for tc in testcases if tc.find(".//skipped") is not None] + for testcase in testcases: - if testcase.find(".//skipped") is None: - if testcase.get("file") is None: - fulltestname = ( - testcase.get("classname").replace(".", "/") - + ".py::" - + testcase.get("name") - ) - else: - fulltestname = testcase.get("file") + "::" + testcase.get("name") - - properties_found = { - p.get("name"): p.get("value") for p in testcase.findall(".//property") - } - - if testcase.find("failure") is not None: - testresult = "FAIL" - elif testcase.find("error") is not None: - testresult = "ERROR" - else: - testresult = "PASS" - - # Extract number of splits, if any - splits = [p.split("_")[-1] for p in properties_found if "split" in p] - splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order - - if splits: - properties_values = {} - for s in splits: - properties_suffixed = [p + "_" + s for p in PROPERTIES] - properties_values[s] = [ - str(properties_found.get(p)) for p in properties_suffixed - ] - else: - properties_values = [str(properties_found.get(p)) for p in PROPERTIES] - - # Identify format and category (mode of operation) - # For the format, favor the earliest match in the test case name - fmt = min( - [ - (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) - for f in FORMATS - if re.search(FORMATS[f], fulltestname, re.IGNORECASE) - ], - key=lambda x: x[1], - )[0] - - # Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked. - cat = [ - c - for c in CATEGORIES - if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) - ][-1] - - # For ERROR cases, both a FAIL and an ERROR result is generated. - # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - if splits: - for s in splits: - fulltestname_split = f"{fulltestname}-{s}" - results[fmt][cat][fulltestname_split] = {"Result": testresult} - for propertyname, propertyvalue in zip( - PROPERTIES, properties_values[s] - ): - results[fmt][cat][fulltestname_split][propertyname] = ( - propertyvalue - ) - else: - results[fmt][cat][fulltestname] = {"Result": testresult} - for propertyname, propertyvalue in zip(PROPERTIES, properties_values): - results[fmt][cat][fulltestname][propertyname] = propertyvalue + if testcase.get("file") is None: + fulltestname = ( + testcase.get("classname").replace(".", "/") + + ".py::" + + testcase.get("name") + ) + else: + fulltestname = testcase.get("file") + "::" + testcase.get("name") + + properties_found = { + p.get("name"): p.get("value") for p in testcase.findall(".//property") + } + + if testcase.find("failure") is not None: + testresult = "FAIL" + elif testcase.find("error") is not None: + testresult = "ERROR" + else: + testresult = "PASS" + + # Extract number of splits, if any + splits = [p.split("_")[-1] for p in properties_found if "split" in p] + splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order + + if splits: + properties_values = {} + for s in splits: + properties_suffixed = [p + "_" + s for p in PROPERTIES] + properties_values[s] = [ + str(properties_found.get(p)) for p in properties_suffixed + ] + else: + properties_values = [str(properties_found.get(p)) for p in PROPERTIES] + + # Identify format and category (mode of operation) + # For the format, favor the earliest match in the test case name + fmt = min( + [ + (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) + for f in FORMATS + if re.search(FORMATS[f], fulltestname, re.IGNORECASE) + ], + key=lambda x: x[1], + )[0] + + # Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked. + cat = [ + c + for c in CATEGORIES + if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) + ][-1] + + # For ERROR cases, both a FAIL and an ERROR result is generated. + # Here, a FAIL would be overwritten with an ERROR result since it has the same name. + if splits: + for s in splits: + fulltestname_split = f"{fulltestname}-{s}" + results[fmt][cat][fulltestname_split] = {"Result": testresult} + for propertyname, propertyvalue in zip( + PROPERTIES, properties_values[s] + ): + results[fmt][cat][fulltestname_split][propertyname] = propertyvalue + else: + results[fmt][cat][fulltestname] = {"Result": testresult} + for propertyname, propertyvalue in zip(PROPERTIES, properties_values): + results[fmt][cat][fulltestname][propertyname] = propertyvalue header = ["testcase", "Format", "Category", "Result"] + PROPERTIES -- GitLab From ecc40ef3287528cf7b1fff8dc5bb7a55af0352f9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 17:37:57 +0100 Subject: [PATCH 19/70] move some code into functions for better readability --- scripts/parse_xml_report.py | 69 +++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index f8577a7560..d4bd7a2cab 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -49,6 +49,38 @@ EVS_CATEGORIES = { NO_CATEGORIES = {"N/A": r".*"} + +def get_format_from_fulltestname(fulltestname: str) -> str: + # For the format, favor the earliest match in the test case name + fmt = min( + [ + (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) + for f in FORMATS + if re.search(FORMATS[f], fulltestname, re.IGNORECASE) is not None + ], + key=lambda x: x[1], + )[0] + return fmt + + +def get_category_from_fulltestname(fulltestname: str) -> str: + cat = [ + c for c in CATEGORIES if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) + ][-1] + return cat + + +def get_testresult(testcase: ElementTree.Element) -> str: + if testcase.find("failure") is not None: + testresult = "FAIL" + elif testcase.find("error") is not None: + testresult = "ERROR" + else: + testresult = "PASS" + + return testresult + + # Main routine if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -116,26 +148,15 @@ if __name__ == "__main__": testcases = [tc for tc in testcases if tc.find(".//skipped") is not None] for testcase in testcases: - if testcase.get("file") is None: - fulltestname = ( - testcase.get("classname").replace(".", "/") - + ".py::" - + testcase.get("name") - ) - else: - fulltestname = testcase.get("file") + "::" + testcase.get("name") + filename = testcase.get( + "file", testcase.get("classname").replace(".", "/") + ".py" + ) + fulltestname = filename + "::" + testcase.get("name") properties_found = { p.get("name"): p.get("value") for p in testcase.findall(".//property") } - if testcase.find("failure") is not None: - testresult = "FAIL" - elif testcase.find("error") is not None: - testresult = "ERROR" - else: - testresult = "PASS" - # Extract number of splits, if any splits = [p.split("_")[-1] for p in properties_found if "split" in p] splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order @@ -152,21 +173,11 @@ if __name__ == "__main__": # Identify format and category (mode of operation) # For the format, favor the earliest match in the test case name - fmt = min( - [ - (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) - for f in FORMATS - if re.search(FORMATS[f], fulltestname, re.IGNORECASE) - ], - key=lambda x: x[1], - )[0] - + fmt = get_format_from_fulltestname(fulltestname) # Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked. - cat = [ - c - for c in CATEGORIES - if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) - ][-1] + cat = get_category_from_fulltestname(fulltestname) + + testresult = get_testresult(testcase) # For ERROR cases, both a FAIL and an ERROR result is generated. # Here, a FAIL would be overwritten with an ERROR result since it has the same name. -- GitLab From d79f81cf7e29f627b27cdbf79b8455ebc3ee9455 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Mar 2025 18:18:36 +0100 Subject: [PATCH 20/70] unify split and non-split case and handle "whole" suffix --- scripts/parse_xml_report.py | 54 +++++++++++++++---------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index d4bd7a2cab..313d2682a3 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -145,7 +145,7 @@ if __name__ == "__main__": count = {"PASS": 0, "FAIL": 0, "ERROR": 0} # filter out skipped testcases - testcases = [tc for tc in testcases if tc.find(".//skipped") is not None] + testcases = [tc for tc in testcases if tc.find(".//skipped") is None] for testcase in testcases: filename = testcase.get( @@ -153,24 +153,16 @@ if __name__ == "__main__": ) fulltestname = filename + "::" + testcase.get("name") + # only include the properties listed above + # we need to find all occurences with any suffixes to also handle the split-comparison + # runs correctly properties_found = { - p.get("name"): p.get("value") for p in testcase.findall(".//property") + p.get("name"): p.get("value") + for p in testcase.findall(".//property") + if "CHANNEL" not in p.get("name") + and any(p_listed in p.get("name") for p_listed in PROPERTIES) } - # Extract number of splits, if any - splits = [p.split("_")[-1] for p in properties_found if "split" in p] - splits = list(dict.fromkeys(splits)) # Remove duplicates, keeping order - - if splits: - properties_values = {} - for s in splits: - properties_suffixed = [p + "_" + s for p in PROPERTIES] - properties_values[s] = [ - str(properties_found.get(p)) for p in properties_suffixed - ] - else: - properties_values = [str(properties_found.get(p)) for p in PROPERTIES] - # Identify format and category (mode of operation) # For the format, favor the earliest match in the test case name fmt = get_format_from_fulltestname(fulltestname) @@ -179,20 +171,21 @@ if __name__ == "__main__": testresult = get_testresult(testcase) + # get all present suffixes + pattern = re.compile("|".join(PROPERTIES)) + suffixes = set(pattern.sub("", p) for p in properties_found) + + # record the result for all suffixes # For ERROR cases, both a FAIL and an ERROR result is generated. # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - if splits: - for s in splits: - fulltestname_split = f"{fulltestname}-{s}" - results[fmt][cat][fulltestname_split] = {"Result": testresult} - for propertyname, propertyvalue in zip( - PROPERTIES, properties_values[s] - ): - results[fmt][cat][fulltestname_split][propertyname] = propertyvalue - else: - results[fmt][cat][fulltestname] = {"Result": testresult} - for propertyname, propertyvalue in zip(PROPERTIES, properties_values): - results[fmt][cat][fulltestname][propertyname] = propertyvalue + for s in suffixes: + fulltestname_suffix = f"{fulltestname}{s}" + results[fmt][cat][fulltestname_suffix] = {"Result": testresult} + for propertyname in PROPERTIES: + results[fmt][cat][fulltestname_suffix][propertyname] = properties_found[ + f"{propertyname}{s}" + ] + count[testresult] += 1 header = ["testcase", "Format", "Category", "Result"] + PROPERTIES @@ -204,11 +197,6 @@ if __name__ == "__main__": for cat in CATEGORIES: results[fmt][cat] = dict(sorted(results[fmt][cat].items())) for test in results[fmt][cat]: - if splits: - if "split1" in test: - count[results[fmt][cat][test]["Result"]] += 1 - else: - count[results[fmt][cat][test]["Result"]] += 1 line = ( ";".join( [test, fmt, cat] + list(results[fmt][cat][test].values()) -- GitLab From 99627bd3542acb204da35097134d0c8b8e77e3cc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Mar 2025 11:11:30 +0100 Subject: [PATCH 21/70] adjust histogram bins and run formatter --- scripts/create_histogram_summary.py | 48 +++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py index af9a11de2a..bcb514acd5 100644 --- a/scripts/create_histogram_summary.py +++ b/scripts/create_histogram_summary.py @@ -80,10 +80,19 @@ if __name__ == "__main__": else: limits_per_measure = { "MLD": ("MLD", [0, 1, 2, 3, 4, 5, 10, 20, math.inf]), - "DIFF": ("MAXIMUM ABS DIFF", [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769]), + "DIFF": ( + "MAXIMUM ABS DIFF", + [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], + ), "SSNR": ("MIN_SSNR", [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100]), - "ODG": ("MIN_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), - "DELTA_ODG": ("DELTA_ODG", [-5, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5]), + "ODG": ( + "MIN_ODG", + [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + ), + "DELTA_ODG": ( + "DELTA_ODG", + [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + ), } (measure_label, limits) = limits_per_measure[measure] @@ -100,18 +109,28 @@ if __name__ == "__main__": results_sorted[testcase][k] = val if limits is None: - vals = [float(x) for x in [m[measure_label] for m in results_sorted.values() if m[measure_label] != "None" and m[measure_label] != ""]] + vals = [ + float(x) + for x in [ + m[measure_label] + for m in results_sorted.values() + if m[measure_label] != "None" and m[measure_label] != "" + ] + ] start = min(vals) f = 10 ** (2 - int(np.floor(np.log10(abs(start)))) - 1) - start = np.floor(start*f)/f - step = (max(vals) - start)/10 + start = np.floor(start * f) / f + step = (max(vals) - start) / 10 f = 10 ** (2 - int(np.floor(np.log10(abs(step)))) - 1) - step = np.ceil(step*f)/f - limits = np.arange(start, 10*step, step) + step = np.ceil(step * f) / f + limits = np.arange(start, 10 * step, step) # Output CSV file with open(csv_summary, "w") as fp: - limits_labels = [f"{a:g}" for a in limits] + ["","None"] # Put None cases in separate bin + limits_labels = [f"{a:g}" for a in limits] + [ + "", + "None", + ] # Put None cases in separate bin headerline = f"Format;Category;" + ";".join(limits_labels) + "\n" fp.write(headerline) @@ -138,7 +157,16 @@ if __name__ == "__main__": fp.write(line) # Matplotlib histogram - ax.bar(limits_labels, data, 1, align='edge', edgecolor='black', linewidth=0.5, label=cat, bottom=bottom) + ax.bar( + limits_labels, + data, + 1, + align="edge", + edgecolor="black", + linewidth=0.5, + label=cat, + bottom=bottom, + ) bottom += data # Histogram layout -- GitLab From 96a6242140c60df610508c8c8ef6208bda403f0b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Mar 2025 11:11:50 +0100 Subject: [PATCH 22/70] add image_dir arg this is needed for handling two sets of histograms in basop ci --- ci/basop-pages/create_summary_page.py | 41 +++++++++++---------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py index 1c2b464802..3c19bf1a6c 100644 --- a/ci/basop-pages/create_summary_page.py +++ b/ci/basop-pages/create_summary_page.py @@ -4,7 +4,7 @@ from typing import List from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS -title = { +TITLE = { "MLD": "Maximum MLD across channels", "DIFF": "Maximim absolute difference across channels", "SSNR": "Minimum SSNR across channels", @@ -26,8 +26,15 @@ def create_summary_page( id_current: int, job_name: str, measures: List[str], + image_dir: str, ): - images = histogram_summary(job_name, measures) + images = "
" + for m in measures: + images += ( + f"

{TITLE[m]}

\n" + + " ".join([f"" for x in FORMATS]) + + f'\n
summary_{m}.csv
\n\n' + ) new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format( id_current=id_current, @@ -38,41 +45,25 @@ def create_summary_page( f.write(new_summary_page) -def histogram_summary( - job_name: str, - measures: List[str], -): - images = "
" - for m in measures: - images += ( - f"

{title[m]}

\n" - + " ".join( - [f"" for x in FORMATS] - ) - + f'\n
summary_{m}.csv
\n\n' - ) - return images - - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("html_out") parser.add_argument("id_current", type=int) parser.add_argument("job_name") + parser.add_argument("image_dir") parser.add_argument( "--measures", nargs="+", - help=f"List of measures to include in summary. Allowed values: {' '.join(title.keys())}", + help=f"List of measures to include in summary. Allowed values: {' '.join(TITLE.keys())}", default=["MLD", "DIFF", "SSNR", "ODG"], ) args = parser.parse_args() - if not all([m in title for m in args.measures]): - raise ValueError(f"Invalid list of measures: {args.measures}, expected one of {' '.join(title.keys())}") + if not all([m in TITLE for m in args.measures]): + raise ValueError( + f"Invalid list of measures: {args.measures}, expected one of {' '.join(TITLE.keys())}" + ) create_summary_page( - args.html_out, - args.id_current, - args.job_name, - args.measures, + args.html_out, args.id_current, args.job_name, args.measures, args.image_dir ) -- GitLab From d8c74f73411cffcb4d809ecfa169d983da588534 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 15:09:32 +0100 Subject: [PATCH 23/70] add assert for sampling rate for debugging --- tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 1d5295d599..2a22b90b90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1264,6 +1264,8 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra If no list of indices is available for the given input file, an empty array is returned. """ + assert sampling_rate_khz in [16, 32, 48] + input_file = input_file.lower() if "omasa" in input_file: format = "_".join(input_file.replace(".wav", "").split("_")[:-1]) -- GitLab From 8f4f93c0901fed6cd054faf586c7cb029a184572 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 17:46:54 +0100 Subject: [PATCH 24/70] record format as property --- .../test_param_file.py | 17 ++++++++--- tests/codec_be_on_mr_nonselection/test_sba.py | 9 ++++++ tests/conftest.py | 29 ++++++++++++++++++- tests/renderer/utils.py | 1 + tests/test_26444.py | 1 + 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 3c9095db79..6146cecdfe 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -54,6 +54,7 @@ from tests.conftest import ( compare_dmx_signals, log_dbg_msg, get_split_idx, + get_format_from_enc_opts, ) from tests.testconfig import PARAM_FILE from tests.constants import ( @@ -298,6 +299,10 @@ def run_test( "All non-passthrough modes are skipped when --compare-to-input is set" ) + pytest.set_trace() + testcase_props = {} + testcase_props["format"] = get_format_from_enc_opts(enc_opts) + tag_str = convert_test_string_to_tag(test_tag) # evaluate encoder options @@ -375,8 +380,9 @@ def run_test( # avoid double recording of the encoder diff if encoder_only: - props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) - for k, v in props.items(): + result_props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) + testcase_props.update(result_props) + for k, v in testcase_props.items(): dut_encoder_frontend.record_property(k, v) if encoder_only: @@ -654,8 +660,11 @@ def run_test( for output_differs, reason, suffix in zip( output_differs_parts, reason_parts, prop_suffix ): - props = parse_properties(reason, output_differs, props_to_record, suffix) - for k, v in props.items(): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + testcase_props.update(result_props) + for k, v in testcase_props.items(): dut_decoder_frontend.record_property(k, v) metadata_differs = False diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 76f42471a1..575fefeafe 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -251,6 +251,7 @@ def test_sba_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = True + testcase_props = {"format": "SBA"} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -343,6 +344,7 @@ def test_sba_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -427,6 +429,7 @@ def test_spar_hoa2_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False + testcase_props = {"format": "SBA"} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA2" @@ -491,6 +494,7 @@ def test_spar_hoa2_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -575,6 +579,7 @@ def test_spar_hoa3_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False + testcase_props = {"format": "SBA"} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA3" @@ -633,6 +638,7 @@ def test_spar_hoa3_enc_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -795,6 +801,7 @@ def test_sba_enc_BWforce_system( props = parse_properties( enc_test_result_msg, enc_test_result != 0, props_to_record ) + props.update(testcase_props) for k, v in props.items(): dut_encoder_frontend.record_property(k, v) @@ -1160,6 +1167,7 @@ def sba_dec( ref_pkt_dir = f"{reference_path}/sba_bs/pkt" dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" + testcase_props = {"format": "SBA"} check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -1326,6 +1334,7 @@ def sba_dec( output_differs_parts, reason_parts, prop_suffix ): props = parse_properties(reason, output_differs, props_to_record, suffix) + props.update(testcase_props) for k, v in props.items(): dut_decoder_frontend.record_property(k, v) diff --git a/tests/conftest.py b/tests/conftest.py index 2a22b90b90..4b52c85aa4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,7 @@ from subprocess import TimeoutExpired, run from tempfile import NamedTemporaryFile from shutil import move import tempfile -from typing import Optional, Union +from typing import Optional, Union, List import numpy as np from .constants import ( DMX_DIFF, @@ -1282,3 +1282,30 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra idx *= sampling_rate_khz // 16 return idx + + +IVAS_ENC_FORMATS = { + "stereo": "Stereo", + "ism": "ISM", + "sba": "SBA", + "masa": "Masa", + "ism_sba": "OSBA", + "ism_masa": "OMASA", + "mc": "Multichannel", + "stereo_dmx_evs": "Stereo DMX EVS", +} +PATTERN_IVAS_ENC_FORMAT = re.compile(r"-(" + r"|".join(IVAS_ENC_FORMATS.keys()) + ")") + + +def get_format_from_enc_opts(enc_opts: str) -> str: + """ + Parse the encoder format from the encoder options by searching for any of the + '-' arguments. If none of them is given, encoder will run in EVS mode. + """ + format = "Mono" + m = re.search(PATTERN_IVAS_ENC_FORMAT, enc_opts) + if m is not None: + enc_format_str = m.groups()[0] + format = IVAS_ENC_FORMATS[enc_format_str] + + return format diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index d2246abf40..02f7eaecc6 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -393,6 +393,7 @@ def run_renderer( reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) + props["format"] = "renderer" for k, v in props.items(): record_property(k, v) diff --git a/tests/test_26444.py b/tests/test_26444.py index a1579dfec9..d41d22efc3 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -180,6 +180,7 @@ def test_evs_26444( reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) + props["format"] = "EVS" for k, v in props.items(): ref_decoder_frontend.record_property(k, v) equal &= not output_differs -- GitLab From 89c65f7176aa9a0b3e3d53c4f5832a5538b0ce52 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Mar 2025 18:18:43 +0100 Subject: [PATCH 25/70] record category as property --- .../test_param_file.py | 14 ++++++- tests/codec_be_on_mr_nonselection/test_sba.py | 37 ++++++++++++++++--- tests/constants.py | 6 +++ tests/renderer/utils.py | 2 + 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 6146cecdfe..86248bc3de 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -62,6 +62,11 @@ from tests.constants import ( MAX_ENC_STATS_DIFF, SCRIPTS_DIR, MAX_ENC_DIFF, + CAT_NORMAL, + CAT_DTX, + CAT_BITRATE_SWITCHING, + CAT_JBM, + CAT_PLC, ) from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -299,9 +304,9 @@ def run_test( "All non-passthrough modes are skipped when --compare-to-input is set" ) - pytest.set_trace() testcase_props = {} testcase_props["format"] = get_format_from_enc_opts(enc_opts) + testcase_props["category"] = CAT_NORMAL tag_str = convert_test_string_to_tag(test_tag) @@ -322,6 +327,9 @@ def run_test( bitrate = enc_split.pop() in_sr = sampling_rate + if "-dtx" in enc_opts: + testcase_props["category"] = CAT_DTX + # bitrate can be a filename: change it to an absolute path if not bitrate.isdigit(): if compare_enc_dmx: @@ -329,6 +337,7 @@ def run_test( "Rate switching + --compare_enc_dmx currently skipped due to DEBUGGING code limitations with varying number of transport channels" ) bitrate = Path(bitrate[3:]).absolute() + testcase_props["category"] = CAT_BITRATE_SWITCHING testv_base = testv_file.split("/")[-1] if testv_base.endswith(".pcm"): @@ -421,9 +430,9 @@ def run_test( rootdir, decoder_only, ) + testcase_props["category"] = CAT_JBM # check for eid-xor command line - if eid_opts != "": eid_split = eid_opts.split() assert len(eid_split) >= 3, "eid-xor expects at least 3 parameters" @@ -449,6 +458,7 @@ def run_test( rootdir, decoder_only, ) + testcase_props["category"] = CAT_PLC # evaluate decoder options dec_split = dec_opts.split() diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 575fefeafe..3df1f3a711 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -48,7 +48,7 @@ from tests.conftest import ( get_split_idx, ) from ..cmp_stats_files import cmp_stats_files -from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF +from ..constants import CAT_BITRATE_SWITCHING, CAT_DTX, CAT_NORMAL, CAT_PLC, TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF from tests.testconfig import use_ltv from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -190,6 +190,7 @@ def test_pca_enc( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -251,7 +252,7 @@ def test_sba_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = True - testcase_props = {"format": "SBA"} + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -289,6 +290,12 @@ def test_sba_enc_system( cut_gain = "1.0" input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -373,6 +380,7 @@ def test_sba_enc_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -429,7 +437,7 @@ def test_spar_hoa2_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False - testcase_props = {"format": "SBA"} + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA2" @@ -523,6 +531,7 @@ def test_spar_hoa2_enc_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -579,7 +588,7 @@ def test_spar_hoa3_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False - testcase_props = {"format": "SBA"} + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA3" @@ -667,6 +676,7 @@ def test_spar_hoa3_enc_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -726,6 +736,7 @@ def test_sba_enc_BWforce_system( cut_testv = False sampling_rate = sample_rate_bw_idx[0] max_bw = sample_rate_bw_idx[1] + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -746,6 +757,12 @@ def test_sba_enc_BWforce_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -830,6 +847,7 @@ def test_sba_enc_BWforce_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -889,6 +907,7 @@ def test_sba_plc_system( sba_order = "+1" cut_testv = True output_config = "FOA" + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -925,6 +944,12 @@ def test_sba_plc_system( cut_gain = "1.0" input_config = SBA_FORMAT[abs(int(sba_order))] + if dtx: + testcase_props["category"] = CAT_DTX + try: + int(bitrate) + except ValueError: + testcase_props["category"] = CAT_BITRATE_SWITCHING if not decoder_only: sba_enc( @@ -973,6 +998,7 @@ def test_sba_plc_system( gain_flag, keep_files, decoder_only, + testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -1150,6 +1176,7 @@ def sba_dec( gain_flag, keep_files, decoder_only, + testcase_props cut_gain="1.0", cut_testv=False, plc_pattern=None, @@ -1167,7 +1194,6 @@ def sba_dec( ref_pkt_dir = f"{reference_path}/sba_bs/pkt" dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" - testcase_props = {"format": "SBA"} check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -1194,6 +1220,7 @@ def sba_dec( plc_file = None if plc_pattern is not None: + testcase_props["category"] = CAT_PLC plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" if sid == 1: diff --git a/tests/constants.py b/tests/constants.py index caa2c67247..18e44317ab 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -66,6 +66,12 @@ ENC_AUX_FILES = [ ["vad_flag", np.int16, "fs/50"], ] +CAT_NORMAL = "normal operation" +CAT_DTX = "DTX" +CAT_PLC = "PLC" +CAT_BITRATE_SWITCHING = "bitrate switching" +CAT_JBM = "JBM" + # lists of indices for splitting of output files # values are in SAMPLES for 16kHz (!). For higher rates, need to multiply SPLIT_IDX_LTV_STEREO_16KHZ = np.asarray( diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 02f7eaecc6..6da0a276ab 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -57,6 +57,7 @@ from .constants import ( BIN_SUFFIX_MERGETARGET, PEAQ_SUPPORTED_FMT, ) +from ..constants import CAT_NORMAL sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile @@ -394,6 +395,7 @@ def run_renderer( props = parse_properties(reason, output_differs, props_to_record) props["format"] = "renderer" + props["category"] = CAT_NORMAL for k, v in props.items(): record_property(k, v) -- GitLab From 5089e568a95c451ddfb93da453a22486d93185cd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Mar 2025 08:38:56 +0100 Subject: [PATCH 26/70] rewrite parse_xml_report.py - only parse properties for all testcases and create dataframe - histogram creation will move to other script --- scripts/parse_xml_report.py | 238 ++++++++---------------------------- 1 file changed, 53 insertions(+), 185 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 313d2682a3..5c856a0a90 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -1,76 +1,25 @@ #!/usr/bin/env python3 import argparse -import re +import pandas as pd from xml.etree import ElementTree +from collections import Counter -""" -Parse a junit report and create a summary report. -""" - -PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"] - -IVAS_FORMATS = { - "Stereo": r"stereo", - "ISM": r"ISM", - "Multichannel": r"Multi-channel|MC", - "MASA": r"(? str: - # For the format, favor the earliest match in the test case name - fmt = min( - [ - (f, re.search(FORMATS[f], fulltestname, re.IGNORECASE).end()) - for f in FORMATS - if re.search(FORMATS[f], fulltestname, re.IGNORECASE) is not None - ], - key=lambda x: x[1], - )[0] - return fmt - - -def get_category_from_fulltestname(fulltestname: str) -> str: - cat = [ - c for c in CATEGORIES if re.search(CATEGORIES[c], fulltestname, re.IGNORECASE) - ][-1] - return cat - - -def get_testresult(testcase: ElementTree.Element) -> str: + +def xml_to_dataframe(xml_report: str) -> pd.DataFrame: + tree = ElementTree.parse(xml_report) + root = tree.getroot() + + testcases = root[0].findall("testcase") + testcases = [tc for tc in testcases if tc.find("skipped") is None] + + parsed_testcases = [parse_testcase(tc) for tc in testcases] + testcase_df = pd.DataFrame(parsed_testcases) + + return testcase_df + + +def get_result_from_testcase(testcase: ElementTree.Element) -> str: if testcase.find("failure") is not None: testresult = "FAIL" elif testcase.find("error") is not None: @@ -81,10 +30,42 @@ def get_testresult(testcase: ElementTree.Element) -> str: return testresult -# Main routine +def parse_testcase(testcase) -> dict: + """ + Get all properties + name for a testcase + """ + ret = {} + + filename = testcase.get("file", testcase.get("classname").replace(".", "/") + ".py") + fulltestname = filename + "::" + testcase.get("name") + ret["testcase"] = fulltestname + + result = get_result_from_testcase(testcase) + ret["result"] = result + + properties = { + p.get("name"): p.get("value") for p in testcase.findall(".//property") + } + ret.update(properties) + + return ret + + +def main(xml_report, csv_file): + df = xml_to_dataframe(xml_report) + df.to_csv(csv_file) + + n_testcases = len(df) + count = Counter(df["result"]) + + print( + f"Parsed testsuite with {n_testcases} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors." + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Parse a junit report and create an MLD summary report." + description="Parse junit report from IVAS pytest suite and convert to csv file" ) parser.add_argument( "xml_report", @@ -92,119 +73,6 @@ if __name__ == "__main__": help="XML junit report input file, e.g. report-junit.xml", ) parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. report.csv") - parser.add_argument( - "--evs", - action="store_true", - help="Parse using EVS 26.444 formats", - ) - parser.add_argument( - "--clipping", - action="store_true", - help="Extract clipping information. Available if encoder has been run with DEBUGGING active.", - ) - parser.add_argument( - "--delta_odg", - action="store_true", - help="Extract Delta ODG information.", - ) - parser.add_argument( - "--skip_formats", - action="store_true", - help="Parse without formats and categories. Suitable for general tests which do not match the IVAS categories.", - ) - args = parser.parse_args() - xml_report = args.xml_report - csv_file = args.csv_file - FORMATS = IVAS_FORMATS - CATEGORIES = IVAS_CATEGORIES - if args.evs: - FORMATS = EVS_FORMATS - CATEGORIES = EVS_CATEGORIES - else: - FORMATS = IVAS_FORMATS - CATEGORIES = IVAS_CATEGORIES - if args.clipping: - PROPERTIES += ["ENC_CORE_OVL", "MAX_OVL", "MIN_OVL"] - if args.delta_odg: - PROPERTIES += ["DELTA_ODG"] - if args.skip_formats: - FORMATS = NO_FORMATS - CATEGORIES = NO_CATEGORIES - - tree = ElementTree.parse(xml_report) - testsuite = tree.find(".//testsuite") - testcases = tree.findall(".//testcase") - - # Prepare result structure - results = {} - for fmt in FORMATS: - results[fmt] = {} - for cat in CATEGORIES: - results[fmt][cat] = {} - count = {"PASS": 0, "FAIL": 0, "ERROR": 0} - - # filter out skipped testcases - testcases = [tc for tc in testcases if tc.find(".//skipped") is None] - - for testcase in testcases: - filename = testcase.get( - "file", testcase.get("classname").replace(".", "/") + ".py" - ) - fulltestname = filename + "::" + testcase.get("name") - - # only include the properties listed above - # we need to find all occurences with any suffixes to also handle the split-comparison - # runs correctly - properties_found = { - p.get("name"): p.get("value") - for p in testcase.findall(".//property") - if "CHANNEL" not in p.get("name") - and any(p_listed in p.get("name") for p_listed in PROPERTIES) - } - - # Identify format and category (mode of operation) - # For the format, favor the earliest match in the test case name - fmt = get_format_from_fulltestname(fulltestname) - # Note that only one category is selected, even though several may match, e.g. bitrate switching + JBM. Here the last match is picked. - cat = get_category_from_fulltestname(fulltestname) - - testresult = get_testresult(testcase) - - # get all present suffixes - pattern = re.compile("|".join(PROPERTIES)) - suffixes = set(pattern.sub("", p) for p in properties_found) - - # record the result for all suffixes - # For ERROR cases, both a FAIL and an ERROR result is generated. - # Here, a FAIL would be overwritten with an ERROR result since it has the same name. - for s in suffixes: - fulltestname_suffix = f"{fulltestname}{s}" - results[fmt][cat][fulltestname_suffix] = {"Result": testresult} - for propertyname in PROPERTIES: - results[fmt][cat][fulltestname_suffix][propertyname] = properties_found[ - f"{propertyname}{s}" - ] - count[testresult] += 1 - - header = ["testcase", "Format", "Category", "Result"] + PROPERTIES - - # Write CSV file - with open(csv_file, "w") as outfile: - headerline = ";".join(header) + "\n" - outfile.write(headerline) - for fmt in FORMATS: - for cat in CATEGORIES: - results[fmt][cat] = dict(sorted(results[fmt][cat].items())) - for test in results[fmt][cat]: - line = ( - ";".join( - [test, fmt, cat] + list(results[fmt][cat][test].values()) - ) - + "\n" - ) - outfile.write(line) - - print( - f"Parsed testsuite with {count['PASS']+count['FAIL']+count['ERROR']} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors." - ) + args = parser.parse_args() + main(args.xml_report, args.csv_file) -- GitLab From 32f381ebcd0be8577ff78024d7e7304869ee6659 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Mar 2025 14:30:30 +0100 Subject: [PATCH 27/70] handle duplicate testcase for ERROR results --- scripts/parse_xml_report.py | 68 ++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 5c856a0a90..f326d13e70 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -6,6 +6,47 @@ from xml.etree import ElementTree from collections import Counter +class TestcaseParser(dict): + def __init__(self, testcases: list): + super().__init__() + + for tc in testcases: + self.parse_testcase(tc) + + def parse_testcase(self, testcase): + """ + Get all properties + name for a testcase + """ + + filename = testcase.get( + "file", testcase.get("classname").replace(".", "/") + ".py" + ) + fulltestname = filename + "::" + testcase.get("name") + + result = get_result_from_testcase(testcase) + # for ERRORS, two testcases are recorded, one with FAIL and one with ERROR + # if we already have this testcase, do a sanity check and set result to ERROR + if fulltestname in self: + results = [self[fulltestname]["result"], result] + assert any(r == "ERROR" for r in results) + self[fulltestname]["result"] = "ERROR" + return + + ret = {} + ret["testcase"] = fulltestname + ret["result"] = result + properties = { + p.get("name"): p.get("value") for p in testcase.findall(".//property") + } + ret.update(properties) + self[fulltestname] = ret + + def to_df(self) -> pd.DataFrame: + testcases = list(self.values()) + df = pd.DataFrame(testcases) + return df + + def xml_to_dataframe(xml_report: str) -> pd.DataFrame: tree = ElementTree.parse(xml_report) root = tree.getroot() @@ -13,8 +54,8 @@ def xml_to_dataframe(xml_report: str) -> pd.DataFrame: testcases = root[0].findall("testcase") testcases = [tc for tc in testcases if tc.find("skipped") is None] - parsed_testcases = [parse_testcase(tc) for tc in testcases] - testcase_df = pd.DataFrame(parsed_testcases) + testcase_parser = TestcaseParser(testcases) + testcase_df = testcase_parser.to_df() return testcase_df @@ -30,30 +71,9 @@ def get_result_from_testcase(testcase: ElementTree.Element) -> str: return testresult -def parse_testcase(testcase) -> dict: - """ - Get all properties + name for a testcase - """ - ret = {} - - filename = testcase.get("file", testcase.get("classname").replace(".", "/") + ".py") - fulltestname = filename + "::" + testcase.get("name") - ret["testcase"] = fulltestname - - result = get_result_from_testcase(testcase) - ret["result"] = result - - properties = { - p.get("name"): p.get("value") for p in testcase.findall(".//property") - } - ret.update(properties) - - return ret - - def main(xml_report, csv_file): df = xml_to_dataframe(xml_report) - df.to_csv(csv_file) + df.to_csv(csv_file, index=False) n_testcases = len(df) count = Counter(df["result"]) -- GitLab From 02958130f2480a20a5001823484278f49d24f419 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 08:04:23 +0100 Subject: [PATCH 28/70] add new histogram script with minimal support for now --- scripts/create_histograms.py | 75 ++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 scripts/create_histograms.py diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py new file mode 100644 index 0000000000..5c10c5545e --- /dev/null +++ b/scripts/create_histograms.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import argparse +import math +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt + + +BINS_4_COLS = { + "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], + "MAXIMUM ABS DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], + "SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], + "ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], +} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Parses a CSV report and creates a summary report." + ) + parser.add_argument( + "csv_report", + type=str, + help="CSV report file of test cases, e.g. report.csv", + ) + args = parser.parse_args() + df = pd.read_csv(args.csv_report) + + measures = ["MAXIMUM ABS DIFF"] + formats = df["format"].unique() + categories = df["category"].unique() + + for measure in measures: + bins = BINS_4_COLS[measure] + x = [f"{x}" for x in bins] + ["", "ERROR"] + for fmt in formats: + fig, ax = plt.subplots() + bottom = np.zeros(len(x)) + for cat in categories: + data_mask = np.logical_and(df["format"] == fmt, df["category"] == cat) + df_slice = df[data_mask] + error_mask = df_slice["result"] == "ERROR" + n_errors = np.sum(error_mask) + df_hist = df_slice[np.logical_not(error_mask)] + + counts, _ = np.histogram(df_hist[measure], bins) + + data = np.concat([counts, [0], [n_errors], [0]]) + ax.bar( + x, + data, + 1, + align="edge", + edgecolor="black", + linewidth=0.5, + label=cat, + bottom=bottom, + ) + bottom += data + + # Histogram layout + ax.set_title(fmt) + ax.legend(loc="best") + ax.set_xlabel(measure) + if "DIFF" in measure: + ax.set_xticks(range(len(x)), x, rotation=35) + ax.set_ylabel("Number of test cases") + + fig.set_figheight(4) + fig.set_figwidth(6) + plt.tight_layout() + + plt.show() -- GitLab From 3a3a23dad798c89fe96515212c28460674ea048b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 08:21:24 +0100 Subject: [PATCH 29/70] record "format" and "category" properties before encoder is run this ensures that these are recorded even if one of the binaries fails --- .../test_param_file.py | 16 +++++--- tests/codec_be_on_mr_nonselection/test_sba.py | 37 ++++++++++++++----- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 86248bc3de..8158d3581a 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -343,6 +343,14 @@ def run_test( if testv_base.endswith(".pcm"): testv_base = testv_base[:-4] + if sim_opts != "": + testcase_props["category"] = CAT_JBM + if eid_opts != "": + testcase_props["category"] = CAT_PLC + + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + assert bitstream_file == "bit" # in the parameter file, only "bit" is used as bitstream file name # -> construct bitstream filename @@ -390,8 +398,7 @@ def run_test( # avoid double recording of the encoder diff if encoder_only: result_props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF]) - testcase_props.update(result_props) - for k, v in testcase_props.items(): + for k, v in result_props.items(): dut_encoder_frontend.record_property(k, v) if encoder_only: @@ -430,7 +437,6 @@ def run_test( rootdir, decoder_only, ) - testcase_props["category"] = CAT_JBM # check for eid-xor command line if eid_opts != "": @@ -458,7 +464,6 @@ def run_test( rootdir, decoder_only, ) - testcase_props["category"] = CAT_PLC # evaluate decoder options dec_split = dec_opts.split() @@ -673,8 +678,7 @@ def run_test( result_props = parse_properties( reason, output_differs, props_to_record, suffix ) - testcase_props.update(result_props) - for k, v in testcase_props.items(): + for k, v in result_props.items(): dut_decoder_frontend.record_property(k, v) metadata_differs = False diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 3df1f3a711..39c12e6131 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -48,7 +48,15 @@ from tests.conftest import ( get_split_idx, ) from ..cmp_stats_files import cmp_stats_files -from ..constants import CAT_BITRATE_SWITCHING, CAT_DTX, CAT_NORMAL, CAT_PLC, TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF +from ..constants import ( + CAT_BITRATE_SWITCHING, + CAT_DTX, + CAT_NORMAL, + CAT_PLC, + TESTV_DIR, + MAX_ENC_FILE_LENGTH_DIFF, + MAX_ENC_STATS_DIFF, +) from tests.testconfig import use_ltv from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -131,6 +139,7 @@ def test_pca_enc( cut_testv = True cut_gain = "1.0" plc_pattern = None + testcase_props = {"format": "SBA", "category": CAT_NORMAL} if "ltv" in tag: tag = f"ltv{sampling_rate}_FOA" @@ -142,6 +151,8 @@ def test_pca_enc( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( @@ -190,7 +201,6 @@ def test_pca_enc( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -297,6 +307,9 @@ def test_sba_enc_system( except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, @@ -380,7 +393,6 @@ def test_sba_enc_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -447,6 +459,8 @@ def test_spar_hoa2_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -531,7 +545,6 @@ def test_spar_hoa2_enc_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -598,6 +611,8 @@ def test_spar_hoa3_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -676,7 +691,6 @@ def test_spar_hoa3_enc_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -764,6 +778,9 @@ def test_sba_enc_BWforce_system( except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) + if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, @@ -847,7 +864,6 @@ def test_sba_enc_BWforce_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -950,6 +966,11 @@ def test_sba_plc_system( int(bitrate) except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING + if plc_pattern is not None: + testcase_props["category"] = CAT_PLC + + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( @@ -998,7 +1019,6 @@ def test_sba_plc_system( gain_flag, keep_files, decoder_only, - testcase_props, cut_gain=cut_gain, cut_testv=cut_testv, plc_pattern=plc_pattern, @@ -1176,7 +1196,6 @@ def sba_dec( gain_flag, keep_files, decoder_only, - testcase_props cut_gain="1.0", cut_testv=False, plc_pattern=None, @@ -1220,7 +1239,6 @@ def sba_dec( plc_file = None if plc_pattern is not None: - testcase_props["category"] = CAT_PLC plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" if sid == 1: @@ -1361,7 +1379,6 @@ def sba_dec( output_differs_parts, reason_parts, prop_suffix ): props = parse_properties(reason, output_differs, props_to_record, suffix) - props.update(testcase_props) for k, v in props.items(): dut_decoder_frontend.record_property(k, v) -- GitLab From 15a4ee2059e673b1d9690c7afee226f31acd9019 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 08:35:15 +0100 Subject: [PATCH 30/70] harmonize naming of properties --- tests/constants.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/constants.py b/tests/constants.py index 18e44317ab..60a9f6e846 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -8,17 +8,17 @@ TESTV_DIR = SCRIPTS_DIR.joinpath("testv") # Properties to record MLD = "MLD" -MAX_ABS_DIFF = "MAXIMUM ABS DIFF" +MAX_ABS_DIFF = "MAX_ABS_DIFF" SSNR = "SSNR" ODG = "ODG" -DELTA_ODG = "Delta-ODG" -MAX_ENC_DIFF = "MAXIMUM ENC DIFF" -MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM" +DELTA_ODG = "DELTA_ODG" +MAX_ENC_DIFF = "MAXIMUM_ENC_DIFF" +MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM_ENC_DIFF_PARAM" ENC_CORE_OVL = "ENC_CORE_OVL" MAX_OVL = "MAX_OVL" MIN_OVL = "MIN_OVL" -DMX_DIFF = "DMX MAXIMUM ABS DIFF" -DMX_MLD = "DMX MLD" +DMX_DIFF = "DMX_MAX_ABS_DIFF" +DMX_MLD = "DMX_MLD" DMX_SSNR = "DMX_SSNR" # regex patterns for parsing the output from comparisons -> mainly for BASOP ci -- GitLab From 1eb32d9e4c003ff7eaf9011cfa4cc9f11b8130b3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 08:42:05 +0100 Subject: [PATCH 31/70] add MEASURES_TO_PLOT list --- scripts/create_histograms.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 5c10c5545e..20d67c4828 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -9,12 +9,14 @@ import matplotlib.pyplot as plt BINS_4_COLS = { "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], - "MAXIMUM ABS DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], - "SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], - "ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], + "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], + "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], + "MIN_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], } +MEASURES_TO_PLOT = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] + if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -28,7 +30,7 @@ if __name__ == "__main__": args = parser.parse_args() df = pd.read_csv(args.csv_report) - measures = ["MAXIMUM ABS DIFF"] + measures = MEASURES_TO_PLOT formats = df["format"].unique() categories = df["category"].unique() -- GitLab From 4c855275b3c17624fb2a6fb60627ab82e2c96a2c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 11:23:02 +0100 Subject: [PATCH 32/70] add saving of histograms to file --- scripts/create_histograms.py | 57 +++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 20d67c4828..880be9ac2d 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -2,9 +2,11 @@ import argparse import math +import pathlib import pandas as pd import numpy as np import matplotlib.pyplot as plt +from typing import List BINS_4_COLS = { @@ -18,22 +20,18 @@ BINS_4_COLS = { MEASURES_TO_PLOT = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Parses a CSV report and creates a summary report." - ) - parser.add_argument( - "csv_report", - type=str, - help="CSV report file of test cases, e.g. report.csv", - ) - args = parser.parse_args() - df = pd.read_csv(args.csv_report) - - measures = MEASURES_TO_PLOT +def create_histograms( + df: pd.DataFrame, + measures: List[str], + output_folder: pathlib.Path, + display_only: bool, +): formats = df["format"].unique() categories = df["category"].unique() + if not display_only: + output_folder.mkdir(exist_ok=True, parents=True) + for measure in measures: bins = BINS_4_COLS[measure] x = [f"{x}" for x in bins] + ["", "ERROR"] @@ -74,4 +72,35 @@ if __name__ == "__main__": fig.set_figwidth(6) plt.tight_layout() - plt.show() + if not display_only: + image_file = f"histogram_{measure}_{fmt}.png" + image_path = output_folder.joinpath(image_file) + plt.savefig(image_path) + + if display_only: + plt.show() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Parses a csv file generated by parse_xml_report and creates histograms for the given measures." + ) + parser.add_argument( + "csv_report", + type=str, + help="CSV report file as generated by parse_xml_report.py", + ) + parser.add_argument( + "output_folder", type=pathlib.Path, help="Output folder for writing the " + ) + parser.add_argument( + "--display-only", + action="store_true", + help="Do not write the output files, but display the graphs instead.", + ) + args = parser.parse_args() + df = pd.read_csv(args.csv_report) + + measures = MEASURES_TO_PLOT + + create_histograms(df, measures, args.output_folder, args.display_only) -- GitLab From 00b447a1fd94f9779aa4c5892c3c85a5dfc227a3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 12:11:45 +0100 Subject: [PATCH 33/70] add --no-bins arg --- scripts/create_histograms.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 880be9ac2d..6181142e86 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -9,7 +9,7 @@ import matplotlib.pyplot as plt from typing import List -BINS_4_COLS = { +BINS_FOR_MEASURES = { "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], @@ -20,11 +20,16 @@ BINS_4_COLS = { MEASURES_TO_PLOT = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] +def get_bins_for_diff(data: pd.Series): + return np.linspace(data.min(), data.max(), num=10) + + def create_histograms( df: pd.DataFrame, measures: List[str], output_folder: pathlib.Path, display_only: bool, + bins_for_measures=BINS_FOR_MEASURES, ): formats = df["format"].unique() categories = df["category"].unique() @@ -33,10 +38,11 @@ def create_histograms( output_folder.mkdir(exist_ok=True, parents=True) for measure in measures: - bins = BINS_4_COLS[measure] + bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure])) x = [f"{x}" for x in bins] + ["", "ERROR"] for fmt in formats: fig, ax = plt.subplots() + ax.xaxis.set_major_formatter("{x:.1f}") bottom = np.zeros(len(x)) for cat in categories: data_mask = np.logical_and(df["format"] == fmt, df["category"] == cat) @@ -98,9 +104,21 @@ if __name__ == "__main__": action="store_true", help="Do not write the output files, but display the graphs instead.", ) + parser.add_argument( + "--no-bins", + action="store_true", + help="""Do not use the hardcoded bins for creating the spectrograms. +Use this for visualising diff scores.""", + ) args = parser.parse_args() df = pd.read_csv(args.csv_report) measures = MEASURES_TO_PLOT - create_histograms(df, measures, args.output_folder, args.display_only) + bins_for_measures = BINS_FOR_MEASURES + if args.no_bins: + bins_for_measures = {} + + create_histograms( + df, measures, args.output_folder, args.display_only, bins_for_measures + ) -- GitLab From e6d2a6ac472eea2436bc328322ddc68a5817f2aa Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 12:20:48 +0100 Subject: [PATCH 34/70] add --measures arg --- scripts/create_histograms.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 6181142e86..691ef4f05a 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -17,7 +17,7 @@ BINS_FOR_MEASURES = { "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], } -MEASURES_TO_PLOT = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] +DEFAULT_MEASURES = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] def get_bins_for_diff(data: pd.Series): @@ -110,15 +110,20 @@ if __name__ == "__main__": help="""Do not use the hardcoded bins for creating the spectrograms. Use this for visualising diff scores.""", ) + allowed_measures = " ".join(BINS_FOR_MEASURES.keys()) + parser.add_argument( + "--measures", + nargs="+", + default=DEFAULT_MEASURES, + help=f"Measures to plot from the csv file. One of {allowed_measures}", + ) args = parser.parse_args() df = pd.read_csv(args.csv_report) - measures = MEASURES_TO_PLOT - bins_for_measures = BINS_FOR_MEASURES if args.no_bins: bins_for_measures = {} create_histograms( - df, measures, args.output_folder, args.display_only, bins_for_measures + df, args.measures, args.output_folder, args.display_only, bins_for_measures ) -- GitLab From cab4e0f4d031dc6ab69a5f3f638101bb36623bf0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 12:44:41 +0100 Subject: [PATCH 35/70] add format and category properties for EVS test --- tests/test_26444.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/test_26444.py b/tests/test_26444.py index d41d22efc3..d3305e9984 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -40,11 +40,12 @@ import shutil from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties +from tests.constants import CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_NORMAL, CAT_PLC test_dict = {} TEST_DIR = "evs_be_test" -scripts = [ +SCRIPTS = [ "Readme_AMRWB_IO_dec.txt", "Readme_AMRWB_IO_enc.txt", "Readme_EVS_dec.txt", @@ -52,7 +53,16 @@ scripts = [ "Readme_JBM_dec.txt", ] -for s in scripts: +FORMATS_4_SCRIPTS = dict( + zip( + [s.replace(".txt", "") for s in SCRIPTS], + ["AMRWBIO_dec", "AMRWBIO_enc", "EVS_dec", "EVS_enc", "EVS_JBM_dec"], + ) +) + +PATTERN_EVS_FORMAT = re.compile(r"-(" + r"|".join(FORMATS_4_SCRIPTS.keys()) + ")") + +for s in SCRIPTS: with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp: tag = "" enc_opts = "" @@ -88,11 +98,33 @@ def test_evs_26444( abs_tol, get_ssnr, get_odg, + record_property, ): enc_opts, dec_opts, diff_opts = test_dict[test_tag] + testcase_props = {} + + # get format prop from test_tag + m = re.search(PATTERN_EVS_FORMAT, test_tag) + assert m is not None + testcase_props["format"] = FORMATS_4_SCRIPTS[m.groups()[0]] diff_opts = diff_opts.replace("./", TEST_DIR + "/") + category = CAT_NORMAL + if "JBM" in test_tag: + category = CAT_JBM + elif "br sw" in test_tag or "bitrate switching" in test_tag: + category = CAT_BITRATE_SWITCHING + elif "%" in test_tag: + category = CAT_PLC + elif "DTX" in test_tag: + category = CAT_DTX + + testcase_props["category"] = category + + for k, v in testcase_props.items(): + record_property(k, v) + if enc_opts: args = enc_opts.split()[1:] @@ -180,7 +212,6 @@ def test_evs_26444( reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) - props["format"] = "EVS" for k, v in props.items(): ref_decoder_frontend.record_property(k, v) equal &= not output_differs -- GitLab From 61bf5f1b9727c615553805e8ed3ef903e5bd962b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 12:52:35 +0100 Subject: [PATCH 36/70] remove old histogram script --- scripts/create_histogram_summary.py | 184 ---------------------------- 1 file changed, 184 deletions(-) delete mode 100644 scripts/create_histogram_summary.py diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py deleted file mode 100644 index bcb514acd5..0000000000 --- a/scripts/create_histogram_summary.py +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import math -import numpy as np - -# These next three lines are added as a precaution in case the gitlab runner -# needs DISPLAY to render the plots, even if they are written to file. -import matplotlib - -matplotlib.use("Agg") -import matplotlib.pyplot as plt -import csv -import os -from parse_xml_report import IVAS_FORMATS, EVS_FORMATS, IVAS_CATEGORIES, EVS_CATEGORIES - -""" -Parses a CSV report and creates a summary report. -""" - - -# Main routine -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Parses a CSV report and creates a summary report." - ) - parser.add_argument( - "csv_report", - type=str, - help="CSV report file of test cases, e.g. report.csv", - ) - parser.add_argument( - "csv_summary", type=str, help="Output CSV file, e.g. summary.csv" - ) - parser.add_argument( - "csv_image", - type=str, - nargs="?", - help="Summary image file, e.g. summary.png", - default=None, - ) - parser.add_argument( - "--measure", - type=str, - nargs=1, - help="Measure, any of: MLD, DIFF, SSNR, ODG, default: MLD", - default=["MLD"], - ) - parser.add_argument( - "--evs", - action="store_true", - help="Parse using EVS 26.444 formats", - default=False, - ) - parser.add_argument( - "--diff", - action="store_true", - help="Use limits for diff scores", - default=False, - ) - args = parser.parse_args() - csv_report = args.csv_report - csv_summary = args.csv_summary - csv_image = args.csv_image - measure = args.measure[0] - if args.evs: - FORMATS = EVS_FORMATS - CATEGORIES = EVS_CATEGORIES - else: - FORMATS = IVAS_FORMATS - CATEGORIES = IVAS_CATEGORIES - if args.diff: - limits_per_measure = { - "MLD": ("MLD", None), - "DIFF": ("MAXIMUM ABS DIFF", None), - "SSNR": ("MIN_SSNR", None), - "ODG": ("MIN_ODG", None), - "DELTA_ODG": ("DELTA_ODG", None), - } - else: - limits_per_measure = { - "MLD": ("MLD", [0, 1, 2, 3, 4, 5, 10, 20, math.inf]), - "DIFF": ( - "MAXIMUM ABS DIFF", - [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], - ), - "SSNR": ("MIN_SSNR", [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100]), - "ODG": ( - "MIN_ODG", - [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], - ), - "DELTA_ODG": ( - "DELTA_ODG", - [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], - ), - } - (measure_label, limits) = limits_per_measure[measure] - - # Load CSV report - results_sorted = {} - with open(csv_report, "r") as fp: - reader = csv.reader(fp, delimiter=";") - header = next(reader) - keys = header[1:] - for row in reader: - testcase = row[0] - results_sorted[testcase] = {} - for k, val in zip(keys, row[1:]): - results_sorted[testcase][k] = val - - if limits is None: - vals = [ - float(x) - for x in [ - m[measure_label] - for m in results_sorted.values() - if m[measure_label] != "None" and m[measure_label] != "" - ] - ] - start = min(vals) - f = 10 ** (2 - int(np.floor(np.log10(abs(start)))) - 1) - start = np.floor(start * f) / f - step = (max(vals) - start) / 10 - f = 10 ** (2 - int(np.floor(np.log10(abs(step)))) - 1) - step = np.ceil(step * f) / f - limits = np.arange(start, 10 * step, step) - - # Output CSV file - with open(csv_summary, "w") as fp: - limits_labels = [f"{a:g}" for a in limits] + [ - "", - "None", - ] # Put None cases in separate bin - headerline = f"Format;Category;" + ";".join(limits_labels) + "\n" - fp.write(headerline) - - for fmt in FORMATS: - fig, ax = plt.subplots() - bottom = np.zeros(len(limits_labels)) - for cat in CATEGORIES: - values = [ - x - for x in [ - m[measure_label] - for m in results_sorted.values() - if m["Format"] == fmt and m["Category"] == cat - ] - ] - # Create separate bin for None (errors) - val = [float(x) for x in values if x != "None" and x != ""] - none = [sum([1 for x in values if x == "None" or x == ""])] - hist, _ = np.histogram(val, limits) - data = np.array(list(hist) + [0] + none + [0]) - - # CSV output - line = f"{fmt};{cat};{'; '.join(map(str,data))}\n" - fp.write(line) - - # Matplotlib histogram - ax.bar( - limits_labels, - data, - 1, - align="edge", - edgecolor="black", - linewidth=0.5, - label=cat, - bottom=bottom, - ) - bottom += data - - # Histogram layout - ax.set_title(fmt) - ax.legend(loc="best") - ax.set_xlabel(measure_label) - if "DIFF" in measure_label: - ax.set_xticks(range(len(limits_labels)), limits_labels, rotation=35) - ax.set_ylabel("Number of test cases") - - fig.set_figheight(4) - fig.set_figwidth(6) - if csv_image: - base, ext = os.path.splitext(csv_image) - plt.savefig(f"{base}_{fmt}{ext}") -- GitLab From f227980b1163d2bfaf9eacb15ca4ab483c4d7b06 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 17:27:23 +0100 Subject: [PATCH 37/70] handle DMX measures via prefix in script --- scripts/create_histograms.py | 18 +++++++++++++++--- tests/conftest.py | 8 +++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 691ef4f05a..1aeb7b0a7d 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -30,6 +30,7 @@ def create_histograms( output_folder: pathlib.Path, display_only: bool, bins_for_measures=BINS_FOR_MEASURES, + prefix="", ): formats = df["format"].unique() categories = df["category"].unique() @@ -38,7 +39,8 @@ def create_histograms( output_folder.mkdir(exist_ok=True, parents=True) for measure in measures: - bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure])) + measure_in_df = prefix + measure + bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure_in_df])) x = [f"{x}" for x in bins] + ["", "ERROR"] for fmt in formats: fig, ax = plt.subplots() @@ -51,7 +53,7 @@ def create_histograms( n_errors = np.sum(error_mask) df_hist = df_slice[np.logical_not(error_mask)] - counts, _ = np.histogram(df_hist[measure], bins) + counts, _ = np.histogram(df_hist[measure_in_df], bins) data = np.concat([counts, [0], [n_errors], [0]]) ax.bar( @@ -117,6 +119,11 @@ Use this for visualising diff scores.""", default=DEFAULT_MEASURES, help=f"Measures to plot from the csv file. One of {allowed_measures}", ) + parser.add_argument( + "--prefix", + default="", + help="Common suffix to use when collecting measures from the input csv file", + ) args = parser.parse_args() df = pd.read_csv(args.csv_report) @@ -125,5 +132,10 @@ Use this for visualising diff scores.""", bins_for_measures = {} create_histograms( - df, args.measures, args.output_folder, args.display_only, bins_for_measures + df, + args.measures, + args.output_folder, + args.display_only, + bins_for_measures, + args.prefix, ) diff --git a/tests/conftest.py b/tests/conftest.py index 4b52c85aa4..5fd95996a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1195,9 +1195,11 @@ def parse_properties( min_ssnr = min(ssnrs) min_ssnr_channel = ssnrs.index(min_ssnr) - prefix = "MIN" if prop == SSNR else "DMX" - props[f"{prefix}_SSNR" + suffix] = min_ssnr - props[f"{prefix}_SSNR_CHANNEL" + suffix] = min_ssnr_channel + propname = "MIN_SSNR" + if prop == DMX_SSNR: + propname = "DMX_MIN_SSNR" + props[propname + suffix] = min_ssnr + props[f"{propname}_CHANNEL" + suffix] = min_ssnr_channel elif prop == ODG: odgs = re.findall(ODG_PATTERN, text_to_parse) min_odg = min(odgs) -- GitLab From e96ede2c8b8c6840ac5b3fcb7d73f1baa098bc88 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Mar 2025 19:35:51 +0100 Subject: [PATCH 38/70] handle split comparison results in parse_xml_report.py --- scripts/parse_xml_report.py | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index f326d13e70..77eb857334 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -6,6 +6,10 @@ from xml.etree import ElementTree from collections import Counter +SPLIT_STRING = "_split" +WHOLE_STRING = "_whole" + + class TestcaseParser(dict): def __init__(self, testcases: list): super().__init__() @@ -38,6 +42,50 @@ class TestcaseParser(dict): properties = { p.get("name"): p.get("value") for p in testcase.findall(".//property") } + + ### handle split comparison results + split_props = {k: v for k, v in properties.items() if SPLIT_STRING in k} + whole_props = {k: v for k, v in properties.items() if WHOLE_STRING in k} + + if len(split_props) > 0 and len(whole_props) > 0: + measures_from_split = set( + [m.split(SPLIT_STRING)[0] for m in split_props.keys()] + ) + measures_from_whole = set( + [m.split(WHOLE_STRING)[0] for m in whole_props.keys()] + ) + assert measures_from_split == measures_from_whole + measures = measures_from_whole + + # collect existing split suffixes by evaluating one of the measures only + m_tmp = measures.pop() + splits = sorted( + [ + k.split(SPLIT_STRING)[-1] + for k in split_props.keys() + if k.startswith(m_tmp) + ] + ) + + # record each split under a separate key + # the dict per key has the same fulltestname and an additional key "split" + # this way, the resulting DataFrame in the end can be split by testnames + for s in splits: + split_key = f"{fulltestname} - {s}" + ret_split = {"testcase": fulltestname, "split": s} + for m in measures: + ret_split.update({m: split_props[m + SPLIT_STRING + f"{s}"]}) + self[split_key] = ret_split + + # it can be the case that there are no splits defined in the pytest suite, e.g. for the renderer + # then, there are only "_whole" values recorded where we only need to remove the suffix + # this if also handles the split case - if there are splits, there was also a "_whole" comparison done + if len(whole_props) > 0: + properties = { + k.replace(WHOLE_STRING, ""): v for k, v in whole_props.items() + } + properties["split"] = "whole" + ret.update(properties) self[fulltestname] = ret -- GitLab From 8a842204bcce34bd0a6d6dfe7556f3fcc400cf7e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 11:32:41 +0100 Subject: [PATCH 39/70] fix xticks --- scripts/create_histograms.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 1aeb7b0a7d..b202a7c847 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -74,6 +74,8 @@ def create_histograms( ax.set_xlabel(measure) if "DIFF" in measure: ax.set_xticks(range(len(x)), x, rotation=35) + else: + ax.set_xticks(range(len(x)), x) ax.set_ylabel("Number of test cases") fig.set_figheight(4) -- GitLab From d12b82730eb5550361dba8fdd0c3a8154aaebc73 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 16:07:50 +0100 Subject: [PATCH 40/70] add separate output file for split cases --- scripts/parse_xml_report.py | 45 ++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 77eb857334..c55f4c02d9 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -4,12 +4,20 @@ import argparse import pandas as pd from xml.etree import ElementTree from collections import Counter +from typing import Optional +from enum import Enum SPLIT_STRING = "_split" WHOLE_STRING = "_whole" +class Result(str, Enum): + ERROR = "ERROR" + FAIL = "FAIL" + PASS = "PASS" + + class TestcaseParser(dict): def __init__(self, testcases: list): super().__init__() @@ -32,8 +40,8 @@ class TestcaseParser(dict): # if we already have this testcase, do a sanity check and set result to ERROR if fulltestname in self: results = [self[fulltestname]["result"], result] - assert any(r == "ERROR" for r in results) - self[fulltestname]["result"] = "ERROR" + assert any(r == Result.ERROR for r in results) + self[fulltestname]["result"] = Result.ERROR return ret = {} @@ -110,24 +118,33 @@ def xml_to_dataframe(xml_report: str) -> pd.DataFrame: def get_result_from_testcase(testcase: ElementTree.Element) -> str: if testcase.find("failure") is not None: - testresult = "FAIL" + testresult = Result.FAIL elif testcase.find("error") is not None: - testresult = "ERROR" + testresult = Result.ERROR else: - testresult = "PASS" + testresult = Result.PASS return testresult -def main(xml_report, csv_file): +def main(xml_report: str, csv_file: str, split_csv_file: Optional[str]): df = xml_to_dataframe(xml_report) - df.to_csv(csv_file, index=False) - n_testcases = len(df) count = Counter(df["result"]) + if split_csv_file is not None: + mask_errors = df["result"] == Result.ERROR + mask_whole = df["split"] == "whole" + mask_single = mask_errors | mask_whole + df_split = df[~mask_single] + df_split.to_csv(split_csv_file, index=False) + + df = df[mask_single] + + df.to_csv(csv_file, index=False) + print( - f"Parsed testsuite with {n_testcases} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors." + f"Parsed testsuite with {n_testcases} tests: {count[Result.PASS]} passes, {count[Result.FAIL]} failures and {count[Result.ERROR]} errors." ) @@ -140,7 +157,13 @@ if __name__ == "__main__": type=str, help="XML junit report input file, e.g. report-junit.xml", ) - parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. report.csv") + parser.add_argument("csv_file", help="Output CSV file, e.g. report.csv") + parser.add_argument( + "--split-csv-file", + type=str, + default=None, + help="If given, write the split comparison values to this file separately", + ) args = parser.parse_args() - main(args.xml_report, args.csv_file) + main(args.xml_report, args.csv_file, args.split_csv_file) -- GitLab From a50fb3b82a5b8f9c9378844de919f9fbc6524d93 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 16:08:25 +0100 Subject: [PATCH 41/70] filter out empty format and category to avoid "nan" in output --- scripts/create_histograms.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index b202a7c847..0e1443eb99 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -129,6 +129,11 @@ Use this for visualising diff scores.""", args = parser.parse_args() df = pd.read_csv(args.csv_report) + # filter out missing format/category values + mask_format_missing = df["format"].isna() + mask_category_missing = df["category"].isna() + df = df[~mask_format_missing | ~mask_category_missing] + bins_for_measures = BINS_FOR_MEASURES if args.no_bins: bins_for_measures = {} -- GitLab From ab4eafa7144bd1a9b878209febe98df924bdd5cd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 16:58:57 +0100 Subject: [PATCH 42/70] use copy to prevent modification of object --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2a22b90b90..16992d92d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1276,7 +1276,7 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra format = "osba_hoa" else: format = input_file.replace(".wav", "").split("_")[-1].lower() - idx = SPLIT_IDX.get(format, np.empty(0)) + idx = SPLIT_IDX.get(format, np.empty(0)).copy() if len(idx) > 0 and sampling_rate_khz != 16: idx *= sampling_rate_khz // 16 -- GitLab From d79d26e32132ac35ed287850313959b7ad2afb7d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Mar 2025 17:15:30 +0100 Subject: [PATCH 43/70] fix format and category keys missing in split case --- scripts/create_histograms.py | 2 +- scripts/parse_xml_report.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 0e1443eb99..d5c0bce38f 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -12,7 +12,7 @@ from typing import List BINS_FOR_MEASURES = { "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], - "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100], + "MIN_SSNR": [-math.inf, 0, 10, 20, 30, 40, 40, 50, 60, 100, math.inf], "MIN_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], "DELTA_ODG": [-5, -4, -3, -2, -1, -0.5, -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.5], } diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index c55f4c02d9..aa4baadeb5 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -54,6 +54,11 @@ class TestcaseParser(dict): ### handle split comparison results split_props = {k: v for k, v in properties.items() if SPLIT_STRING in k} whole_props = {k: v for k, v in properties.items() if WHOLE_STRING in k} + other_props = { + k: v + for k, v in properties.items() + if WHOLE_STRING not in k and SPLIT_STRING not in k + } if len(split_props) > 0 and len(whole_props) > 0: measures_from_split = set( @@ -83,6 +88,8 @@ class TestcaseParser(dict): ret_split = {"testcase": fulltestname, "split": s} for m in measures: ret_split.update({m: split_props[m + SPLIT_STRING + f"{s}"]}) + + ret_split.update(other_props) self[split_key] = ret_split # it can be the case that there are no splits defined in the pytest suite, e.g. for the renderer @@ -93,6 +100,7 @@ class TestcaseParser(dict): k.replace(WHOLE_STRING, ""): v for k, v in whole_props.items() } properties["split"] = "whole" + properties.update(other_props) ret.update(properties) self[fulltestname] = ret -- GitLab From 4112ded2ec9e8e5ddba8c94fb3e7b1a3db449acd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 09:46:22 +0100 Subject: [PATCH 44/70] fix bug with measure missing due to removal form set --- scripts/parse_xml_report.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index aa4baadeb5..948bda4308 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -71,7 +71,10 @@ class TestcaseParser(dict): measures = measures_from_whole # collect existing split suffixes by evaluating one of the measures only + # get one measure from set and add it back immediately m_tmp = measures.pop() + measures.add(m_tmp) + splits = sorted( [ k.split(SPLIT_STRING)[-1] -- GitLab From 8a8d18809b75b7658ce3f33a75f50564a7dd2090 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 10:18:40 +0100 Subject: [PATCH 45/70] add the DISPLAY var missing hack --- scripts/create_histograms.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index d5c0bce38f..13b51d4869 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -5,10 +5,16 @@ import math import pathlib import pandas as pd import numpy as np -import matplotlib.pyplot as plt from typing import List +# hack for avoiding missing DISPLAY variable in headless CI runners +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + BINS_FOR_MEASURES = { "MLD": [0, 1, 2, 3, 4, 5, 10, 20, math.inf], "MAX_ABS_DIFF": [0, 16, 256, 1024, 2048, 4096, 8192, 16384, 32769], -- GitLab From 46037ef006220a0173cc9472674f47a9c2f169f9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 10:44:28 +0100 Subject: [PATCH 46/70] fix warning about too many open figures when not displaying --- scripts/create_histograms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 13b51d4869..b0923ddbd8 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -92,6 +92,7 @@ def create_histograms( image_file = f"histogram_{measure}_{fmt}.png" image_path = output_folder.joinpath(image_file) plt.savefig(image_path) + plt.close(fig) if display_only: plt.show() -- GitLab From 7b4d7fe2dcac5a0f2fc0f6710b3b6f16aa051a48 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 11:14:00 +0100 Subject: [PATCH 47/70] fix regex pattern to prevent incorrect partial matches --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index f572ac3bdb..e8cdc229d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1296,7 +1296,8 @@ IVAS_ENC_FORMATS = { "mc": "Multichannel", "stereo_dmx_evs": "Stereo DMX EVS", } -PATTERN_IVAS_ENC_FORMAT = re.compile(r"-(" + r"|".join(IVAS_ENC_FORMATS.keys()) + ")") +# NOTE: the blanks at start and end are important to prevent e.g. "-ism_masa" matching on "-ism" only +PATTERN_IVAS_ENC_FORMAT = re.compile(r"-( " + r"|".join(IVAS_ENC_FORMATS.keys()) + ") ") def get_format_from_enc_opts(enc_opts: str) -> str: -- GitLab From 87691838a0edd0424f2d0b0366ba5eac01655b70 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 11:14:33 +0100 Subject: [PATCH 48/70] adapt summary page creation script to new data format --- ci/basop-pages/create_summary_page.py | 32 +++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py index 3c19bf1a6c..199b35bb51 100644 --- a/ci/basop-pages/create_summary_page.py +++ b/ci/basop-pages/create_summary_page.py @@ -4,11 +4,11 @@ from typing import List from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS -TITLE = { +TITLE_4_MEASURE = { "MLD": "Maximum MLD across channels", - "DIFF": "Maximim absolute difference across channels", - "SSNR": "Minimum SSNR across channels", - "ODG": "Minimum PEAQ ODG across channels", + "MAX_ABS_DIFF": "Maximim absolute difference across channels", + "MIN_SSNR": "Minimum SSNR across channels", + "MIN_ODG": "Minimum PEAQ ODG across channels", "DELTA_ODG": "PEAQ ODG using binauralized input and output", } @@ -28,18 +28,21 @@ def create_summary_page( measures: List[str], image_dir: str, ): - images = "
" + html = "
" for m in measures: - images += ( - f"

{TITLE[m]}

\n" - + " ".join([f"" for x in FORMATS]) - + f'\n
summary_{m}.csv
\n\n' + image_filename_tmpl = "histogram_{measure}_{format}.png" + image_html = " ".join( + [ + f"" + for f in FORMATS + ] ) + html += f"

{TITLE_4_MEASURE[m]}

\n" + image_html new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format( id_current=id_current, job_name=job_name, - images=images, + images=html, ) with open(html_out, "w") as f: f.write(new_summary_page) @@ -54,14 +57,15 @@ if __name__ == "__main__": parser.add_argument( "--measures", nargs="+", - help=f"List of measures to include in summary. Allowed values: {' '.join(TITLE.keys())}", - default=["MLD", "DIFF", "SSNR", "ODG"], + help=f"List of measures to include in summary. Allowed values: {' '.join(TITLE_4_MEASURE.keys())}", + # exclude DELTA_ODG here + default=list(TITLE_4_MEASURE.keys())[:-1], ) args = parser.parse_args() - if not all([m in TITLE for m in args.measures]): + if not all([m in TITLE_4_MEASURE for m in args.measures]): raise ValueError( - f"Invalid list of measures: {args.measures}, expected one of {' '.join(TITLE.keys())}" + f"Invalid list of measures: {args.measures}, expected one of {' '.join(TITLE_4_MEASURE.keys())}" ) create_summary_page( -- GitLab From 08d8466aeb3c9c39890bcbf20505af0cc396fa88 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 11:37:30 +0100 Subject: [PATCH 49/70] small refactoring + change to layout --- ci/basop-pages/create_summary_page.py | 28 ++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py index 199b35bb51..32855fce25 100644 --- a/ci/basop-pages/create_summary_page.py +++ b/ci/basop-pages/create_summary_page.py @@ -6,7 +6,7 @@ from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS TITLE_4_MEASURE = { "MLD": "Maximum MLD across channels", - "MAX_ABS_DIFF": "Maximim absolute difference across channels", + "MAX_ABS_DIFF": "Maximum absolute difference across channels", "MIN_SSNR": "Minimum SSNR across channels", "MIN_ODG": "Minimum PEAQ ODG across channels", "DELTA_ODG": "PEAQ ODG using binauralized input and output", @@ -16,9 +16,13 @@ SUMMARY_PAGE_TMPL_HTML = """

Summary for job {job_name}, ID: {id_current}

+
+ {images} """ +IMAGE_HTML_TMPL = "" +SUBHEADING_HTML_TMP = "

{subtitle}

\n" def create_summary_page( @@ -28,16 +32,18 @@ def create_summary_page( measures: List[str], image_dir: str, ): - html = "
" - for m in measures: - image_filename_tmpl = "histogram_{measure}_{format}.png" - image_html = " ".join( - [ - f"" - for f in FORMATS - ] - ) - html += f"

{TITLE_4_MEASURE[m]}

\n" + image_html + html = "\n
\n".join( + [ + SUBHEADING_HTML_TMP.format(subtitle=TITLE_4_MEASURE[m]) + + " ".join( + [ + IMAGE_HTML_TMPL.format(measure=m, format=f, image_dir=image_dir) + for f in FORMATS + ] + ) + for m in measures + ] + ) new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format( id_current=id_current, -- GitLab From 3438f11d5382c724f5014cac793ce97238b7cc61 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 12:04:26 +0100 Subject: [PATCH 50/70] use concatenate instead of concat --- scripts/create_histograms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index b0923ddbd8..160ea95419 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -61,7 +61,7 @@ def create_histograms( counts, _ = np.histogram(df_hist[measure_in_df], bins) - data = np.concat([counts, [0], [n_errors], [0]]) + data = np.concatenate([counts, [0], [n_errors], [0]]) ax.bar( x, data, -- GitLab From bb5350544752226803f1bb341d39063165236988 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Mar 2025 14:07:18 +0100 Subject: [PATCH 51/70] fix format property recording --- tests/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e8cdc229d0..027b191414 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1287,17 +1287,17 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra IVAS_ENC_FORMATS = { - "stereo": "Stereo", - "ism": "ISM", "sba": "SBA", "masa": "Masa", "ism_sba": "OSBA", "ism_masa": "OMASA", + "ism": "ISM", "mc": "Multichannel", "stereo_dmx_evs": "Stereo DMX EVS", + "stereo": "Stereo", } -# NOTE: the blanks at start and end are important to prevent e.g. "-ism_masa" matching on "-ism" only -PATTERN_IVAS_ENC_FORMAT = re.compile(r"-( " + r"|".join(IVAS_ENC_FORMATS.keys()) + ") ") +# NOTE: the blank at the end is important to prevent e.g. "-ism_masa" matching on "-ism" only +PATTERN_IVAS_ENC_FORMAT = re.compile(r"-(" + r"|".join(IVAS_ENC_FORMATS.keys()) + ") ") def get_format_from_enc_opts(enc_opts: str) -> str: -- GitLab From 741ca23874670a83a79deceb0b0b9e7a12d194aa Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 31 Mar 2025 09:38:47 +0200 Subject: [PATCH 52/70] Correct format names to fit with html page naming --- tests/conftest.py | 2 +- tests/renderer/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 027b191414..0d102b16bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1288,7 +1288,7 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra IVAS_ENC_FORMATS = { "sba": "SBA", - "masa": "Masa", + "masa": "MASA", "ism_sba": "OSBA", "ism_masa": "OMASA", "ism": "ISM", diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 6da0a276ab..ec5fe7f0cb 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -394,7 +394,7 @@ def run_renderer( reason = reason[0] props = parse_properties(reason, output_differs, props_to_record) - props["format"] = "renderer" + props["format"] = "Renderer" props["category"] = CAT_NORMAL for k, v in props.items(): record_property(k, v) -- GitLab From 3d03e4d7357ad847885586e9fd86cf6cd307fd39 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 31 Mar 2025 09:48:18 +0200 Subject: [PATCH 53/70] fix split idx selection for OMASA --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0d102b16bf..69a13e6d7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1270,14 +1270,14 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra input_file = input_file.lower() if "omasa" in input_file: - format = "_".join(input_file.replace(".wav", "").split("_")[:-1]) + format = "_".join(input_file.split("_")[1:-1]) elif "osba" in input_file: if "foa" in input_file: format = "osba_foa" else: format = "osba_hoa" else: - format = input_file.replace(".wav", "").split("_")[-1].lower() + format = input_file.split("_")[-1].lower() idx = SPLIT_IDX.get(format, np.empty(0)).copy() if len(idx) > 0 and sampling_rate_khz != 16: -- GitLab From 1e38c12627f30eddea678a1b5d248903c5468992 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 31 Mar 2025 15:24:18 +0200 Subject: [PATCH 54/70] make renderer tests record props correctly for split-comparison --- tests/renderer/test_renderer.py | 134 +++++++++++++++++++++++--------- tests/renderer/utils.py | 12 ++- 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index b5758d6f06..956f773666 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -62,7 +62,6 @@ from ..conftest import props_to_record """ Ambisonics """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -78,6 +77,7 @@ def test_ambisonics( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -92,10 +92,10 @@ def test_ambisonics( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -111,6 +111,7 @@ def test_ambisonics_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -125,10 +126,10 @@ def test_ambisonics_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @@ -146,6 +147,7 @@ def test_ambisonics_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -161,11 +163,11 @@ def test_ambisonics_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) @pytest.mark.skip(reason="Not supported for BASOP code currently") - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -183,6 +185,7 @@ def test_dynamic_acoustic_environment( get_odg, get_odg_bin, aeid, + split_comparison, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -202,11 +205,11 @@ def test_dynamic_acoustic_environment( get_odg_bin=get_odg_bin, config_file=rend_config_path, aeid=aeid, + split_comparison=split_comparison, ) @pytest.mark.skip(reason="Not supported for BASOP code currently") - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -222,6 +225,7 @@ def test_dynamic_acoustic_environment_file( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -243,13 +247,13 @@ def test_dynamic_acoustic_environment_file( get_odg_bin=get_odg_bin, config_file=rend_config_path, aeid=aeid, + split_comparison=split_comparison, ) """ Multichannel """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -265,6 +269,7 @@ def test_multichannel( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -279,10 +284,10 @@ def test_multichannel( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -298,6 +303,7 @@ def test_multichannel_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -315,10 +321,10 @@ def test_multichannel_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @@ -336,6 +342,7 @@ def test_multichannel_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -354,13 +361,13 @@ def test_multichannel_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ ISM """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -376,6 +383,7 @@ def test_ism( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -391,10 +399,10 @@ def test_ism( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -410,6 +418,7 @@ def test_ism_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -430,10 +439,10 @@ def test_ism_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @@ -451,6 +460,7 @@ def test_ism_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -472,13 +482,13 @@ def test_ism_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ MASA """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -494,6 +504,7 @@ def test_masa( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -509,10 +520,10 @@ def test_masa( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -528,6 +539,7 @@ def test_masa_binaural_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -546,10 +558,10 @@ def test_masa_binaural_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @@ -567,6 +579,7 @@ def test_masa_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -586,10 +599,10 @@ def test_masa_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) def test_masa_prerend( record_property, @@ -601,6 +614,7 @@ def test_masa_prerend( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -615,13 +629,13 @@ def test_masa_prerend( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ Custom loudspeaker layouts """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -637,6 +651,7 @@ def test_custom_ls_input( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -651,10 +666,10 @@ def test_custom_ls_input( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output( @@ -668,6 +683,7 @@ def test_custom_ls_output( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -681,10 +697,10 @@ def test_custom_ls_output( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output( @@ -698,6 +714,7 @@ def test_custom_ls_input_output( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -711,10 +728,10 @@ def test_custom_ls_input_output( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -730,6 +747,7 @@ def test_custom_ls_input_binaural( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -744,10 +762,10 @@ def test_custom_ls_input_binaural( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @@ -765,6 +783,7 @@ def test_custom_ls_input_binaural_headrotation( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -780,13 +799,13 @@ def test_custom_ls_input_binaural_headrotation( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ Metadata / scene description input """ - @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) @@ -802,6 +821,7 @@ def test_metadata( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -817,13 +837,13 @@ def test_metadata( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) """ non diegetic pan """ - @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -839,6 +859,7 @@ def test_non_diegetic_pan_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -853,10 +874,10 @@ def test_non_diegetic_pan_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) - @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) @@ -872,6 +893,7 @@ def test_non_diegetic_pan_ism_static( get_ssnr, get_odg, get_odg_bin, + split_comparison, ): run_renderer( record_property, @@ -886,6 +908,7 @@ def test_non_diegetic_pan_ism_static( get_ssnr=get_ssnr, get_odg=get_odg, get_odg_bin=get_odg_bin, + split_comparison=split_comparison, ) @@ -909,9 +932,7 @@ def test_ambisonics_binaural_headrotation_refrotzero( in_fmt, out_fmt, trj_file, - get_mld, - get_mld_lim, - get_ssnr, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -932,6 +953,7 @@ def test_ambisonics_binaural_headrotation_refrotzero( "refrot_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -941,7 +963,12 @@ def test_ambisonics_binaural_headrotation_refrotzero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotequal( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -965,6 +992,7 @@ def test_ambisonics_binaural_headrotation_refrotequal( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -982,8 +1010,7 @@ def test_ambisonics_binaural_headrotation_refveczero( in_fmt, out_fmt, trj_file, - get_mld, - get_mld_lim, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1004,6 +1031,7 @@ def test_ambisonics_binaural_headrotation_refveczero( "refvec_file": HR_TRAJECTORY_DIR.joinpath("const000-Vector3.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1014,7 +1042,12 @@ def test_ambisonics_binaural_headrotation_refveczero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvecequal( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1042,6 +1075,7 @@ def test_ambisonics_binaural_headrotation_refvecequal( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1052,7 +1086,12 @@ def test_ambisonics_binaural_headrotation_refvecequal( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1081,6 +1120,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1093,7 +1133,12 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1118,6 +1163,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1129,7 +1175,12 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1153,6 +1204,7 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( "refvec_file": HR_TRAJECTORY_DIR.joinpath("full-circle-4s-Vector3.csv"), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1163,7 +1215,12 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) def test_multichannel_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1191,6 +1248,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating( ), "frame_size": "5", }, + split_comparison=split_comparison, ) @@ -1201,7 +1259,12 @@ def test_multichannel_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) def test_ism_binaural_headrotation_refvec_rotating( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -1233,4 +1296,5 @@ def test_ism_binaural_headrotation_refvec_rotating( "in_meta_files": in_meta_files, "frame_size": "5", }, + split_comparison=split_comparison, ) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index ec5fe7f0cb..370da4cf89 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -174,6 +174,7 @@ def run_renderer( out_file=None, sr=48, render_for_peaq=False, + split_comparison=False, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -393,7 +394,13 @@ def run_renderer( output_differs = output_differs[0] reason = reason[0] - props = parse_properties(reason, output_differs, props_to_record) + # splitting is not implemented for renderer tests yet + # if arg is given, need to record as "whole" so that XML parsing works + suffix = "" + if split_comparison: + suffix = "_whole" + + props = parse_properties(reason, output_differs, props_to_record, suffix) props["format"] = "Renderer" props["category"] = CAT_NORMAL for k, v in props.items(): @@ -420,6 +427,7 @@ def compare_renderer_args( out_fmt, ref_kwargs: Dict, cut_kwargs: Dict, + split_comparison=False, ): out_file_ref = run_renderer( record_property, @@ -428,6 +436,7 @@ def compare_renderer_args( in_fmt, out_fmt, **ref_kwargs, + split_comparison=split_comparison, ) ref, ref_fs = readfile(out_file_ref) out_file_cut = run_renderer( @@ -437,6 +446,7 @@ def compare_renderer_args( in_fmt, out_fmt, **cut_kwargs, + split_comparison=split_comparison, ) cut, cut_fs = readfile(out_file_cut) [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) -- GitLab From 13a44da024a6005f9346143b5d1c891fac89af22 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 08:46:27 +0200 Subject: [PATCH 55/70] skip split comparison for JBM cases only record whole-file measurements --- tests/codec_be_on_mr_nonselection/test_param_file.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 8158d3581a..cc7b01ee4a 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -642,7 +642,9 @@ def run_test( ) # 2. run comparison on split files if --split_comparison is given - if split_comparison: + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases + if split_comparison and not sim_opts: split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) output_differs_splits, reason_splits = cmp_pcm( @@ -668,6 +670,8 @@ def run_test( output_differs_parts += output_differs_splits reason_parts += reason_splits + # separate if to also record the whole-file comparison for JBM cases with "_whole" + if split_comparison: prop_suffix = ["_whole"] + [ f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] -- GitLab From 93d77e5f2109a4bcb7e5feb082d6d2248e674bdc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 12:20:20 +0200 Subject: [PATCH 56/70] enable split comparison for renderer tests --- tests/renderer/constants.py | 120 +++++++++++++++++++++++++------- tests/renderer/test_renderer.py | 49 +++++++++++-- tests/renderer/utils.py | 73 ++++++++++++++----- 3 files changed, 194 insertions(+), 48 deletions(-) diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 39f57a6d89..c61800a1a2 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ from pathlib import Path @@ -298,6 +298,77 @@ FORMAT_TO_METADATA_FILES = { ], } +FORMAT_TO_METADATA_FILES_LTV = { + "ISM1": [str(TESTV_DIR.joinpath("ltvISM1.csv"))], + "ISM2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + ], + "ISM3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + ], + "ISM4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + ], + "NDP_ISM4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2_non-diegetic-pan.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + ], + "MASA1": [str(TESTV_DIR.joinpath("ltv48_MASA1TC.met"))], + "MASA2": [str(TESTV_DIR.joinpath("ltv48_MASA2TC.met"))], + "OMASA_1_1": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_1TC.met")), + ], + "OMASA_1_2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met")), + ], + "OMASA_1_3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_1TC.met")), + ], + "OMASA_1_4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_1TC.met")), + ], + "OMASA_2_1": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_2TC.met")), + ], + "OMASA_2_2": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_2TC.met")), + ], + "OMASA_2_3": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_2TC.met")), + ], + "OMASA_2_4": [ + str(TESTV_DIR.joinpath("ltvISM1.csv")), + str(TESTV_DIR.joinpath("ltvISM2.csv")), + str(TESTV_DIR.joinpath("ltvISM3.csv")), + str(TESTV_DIR.joinpath("ltvISM4.csv")), + str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_2TC.met")), + ], +} + """ Input formats """ INPUT_FORMATS_AMBI = ["FOA", "HOA2", "HOA3"] @@ -347,4 +418,5 @@ PEAQ_SUPPORTED_FMT = [ "BINAURAL", "BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB", -] \ No newline at end of file +] + diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 956f773666..60ec37a69c 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -33,6 +33,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pytest from .constants import ( + FORMAT_TO_METADATA_FILES_LTV, OUTPUT_FORMATS, INPUT_FORMATS_AMBI, FRAMING_TO_TEST, @@ -385,13 +386,18 @@ def test_ism( get_odg_bin, split_comparison, ): + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -421,7 +427,11 @@ def test_ism_binaural_static( split_comparison, ): try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -463,7 +473,11 @@ def test_ism_binaural_headrotation( split_comparison, ): try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None @@ -506,13 +520,19 @@ def test_masa( get_odg_bin, split_comparison, ): + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) + run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -544,13 +564,18 @@ def test_masa_binaural_static( if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) run_renderer( record_property, props_to_record, test_info, in_fmt, out_fmt, - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -584,6 +609,12 @@ def test_masa_binaural_headrotation( if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + md_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) + run_renderer( record_property, props_to_record, @@ -591,7 +622,7 @@ def test_masa_binaural_headrotation( in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + in_meta_files=md_files, binary_suffix=EXE_SUFFIX, frame_size=frame_size, get_mld=get_mld, @@ -1270,7 +1301,11 @@ def test_ism_binaural_headrotation_refvec_rotating( pytest.skip("OTR tests only run for smoke test") try: - in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files = ( + FORMAT_TO_METADATA_FILES_LTV[in_fmt] + if test_info.config.option.use_ltv + else FORMAT_TO_METADATA_FILES[in_fmt] + ) except KeyError: in_meta_files = None diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 370da4cf89..e9a8f27345 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -62,7 +62,7 @@ from ..constants import CAT_NORMAL sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile from ..cmp_pcm import cmp_pcm -from ..conftest import parse_properties +from ..conftest import parse_properties, get_split_idx def run_cmd(cmd, test_info, env=None): @@ -233,7 +233,8 @@ def run_renderer( # if in REF or CUT creation mode use the comparetestv if test_info.config.option.create_ref or test_info.config.option.create_cut: FORMAT_TO_FILE = FORMAT_TO_FILE_COMPARETEST - elif test_info.config.option.use_ltv: + + if test_info.config.option.use_ltv: if test_info.config.option.ltv_dir: FORMAT_TO_FILE = dict() for k, v in FORMAT_TO_FILE_LTV.items(): @@ -310,6 +311,14 @@ def run_renderer( env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_info.node.name}" ) + testcase_props = { + "format": "Renderer", + "category": CAT_NORMAL, + } + + for k, v in testcase_props.items(): + record_property(k, v) + # run the renderer run_cmd(cmd, test_info, env) @@ -373,9 +382,13 @@ def run_renderer( else: odg_input = in_file - # see constants.py + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison ref_fs = int(cmd[10]) * 1000 - output_differs, reason = cmp_pcm( + output_differs_parts, reason_parts = cmp_pcm( out_file, out_file_ref, out_fmt, @@ -390,24 +403,50 @@ def run_renderer( odg_test=odg_test, odg_ref=odg_ref, scalefac=test_info.config.option.scalefac, + split_idx=split_idx, ) - output_differs = output_differs[0] - reason = reason[0] - # splitting is not implemented for renderer tests yet - # if arg is given, need to record as "whole" so that XML parsing works - suffix = "" + # 2. run comparison on split files if --split_comparison is given + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases if split_comparison: - suffix = "_whole" + split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) - props = parse_properties(reason, output_differs, props_to_record, suffix) - props["format"] = "Renderer" - props["category"] = CAT_NORMAL - for k, v in props.items(): - record_property(k, v) + output_differs_splits, reason_splits = cmp_pcm( + out_file, + out_file_ref, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + for k, v in result_props.items(): + record_property(k, v) - if output_differs: - pytest.fail(f"Output differs: ({reason})") + if output_differs_parts[0]: + pytest.fail(f"Output differs: ({reason_parts[0]})") # compare metadata files in case of MASA prerendering if "MASA" in str(out_fmt): -- GitLab From 119ef09829f25e325c4752057a9207f329159b59 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 13:51:25 +0200 Subject: [PATCH 57/70] add write out option for histograms to csv --- scripts/create_histograms.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 160ea95419..7b05d93f8f 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -37,17 +37,22 @@ def create_histograms( display_only: bool, bins_for_measures=BINS_FOR_MEASURES, prefix="", + write_out_histograms=False, ): formats = df["format"].unique() categories = df["category"].unique() - if not display_only: + if not display_only or write_out_histograms: output_folder.mkdir(exist_ok=True, parents=True) for measure in measures: measure_in_df = prefix + measure bins = bins_for_measures.get(measure, get_bins_for_diff(df[measure_in_df])) x = [f"{x}" for x in bins] + ["", "ERROR"] + + df_hist = pd.DataFrame(columns=["format", "category"] + x) + hist_row_count = 0 + for fmt in formats: fig, ax = plt.subplots() ax.xaxis.set_major_formatter("{x:.1f}") @@ -57,9 +62,9 @@ def create_histograms( df_slice = df[data_mask] error_mask = df_slice["result"] == "ERROR" n_errors = np.sum(error_mask) - df_hist = df_slice[np.logical_not(error_mask)] + df_slice = df_slice[np.logical_not(error_mask)] - counts, _ = np.histogram(df_hist[measure_in_df], bins) + counts, _ = np.histogram(df_slice[measure_in_df], bins) data = np.concatenate([counts, [0], [n_errors], [0]]) ax.bar( @@ -74,6 +79,10 @@ def create_histograms( ) bottom += data + hist_row = [fmt, cat] + list(counts) + [0] + [0, n_errors] + df_hist.loc[hist_row_count] = hist_row + hist_row_count += 1 + # Histogram layout ax.set_title(fmt) ax.legend(loc="best") @@ -94,6 +103,11 @@ def create_histograms( plt.savefig(image_path) plt.close(fig) + if write_out_histograms: + df_hist.to_csv( + output_folder.joinpath(f"histogram_{measure}.csv"), index=False + ) + if display_only: plt.show() @@ -133,6 +147,11 @@ Use this for visualising diff scores.""", default="", help="Common suffix to use when collecting measures from the input csv file", ) + parser.add_argument( + "--write-out-histograms", + action="store_true", + help="Write out the histogram values to csv", + ) args = parser.parse_args() df = pd.read_csv(args.csv_report) @@ -152,4 +171,5 @@ Use this for visualising diff scores.""", args.display_only, bins_for_measures, args.prefix, + args.write_out_histograms, ) -- GitLab From e69f994532b2d64bdbcbab5b1d3b8304caa5e0f1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 14:06:43 +0200 Subject: [PATCH 58/70] add consistent color assignment this helps keep split and whole page in synch wrt colors, even if a format/category/mode is omitted in one of them (e.g. JBM in split) --- scripts/create_histograms.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 7b05d93f8f..5ec2f0e599 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -3,6 +3,7 @@ import argparse import math import pathlib +import sys import pandas as pd import numpy as np from typing import List @@ -25,6 +26,19 @@ BINS_FOR_MEASURES = { DEFAULT_MEASURES = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] +HERE = pathlib.Path(__file__).parent +ROOT_DIR = HERE.parent +sys.path.append(str(ROOT_DIR)) +from tests.constants import CAT_NORMAL, CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_PLC + +COLORS_FOR_CATEGORIES = { + CAT_DTX: "tab:blue", + CAT_PLC: "tab:orange", + CAT_NORMAL: "tab:green", + CAT_JBM: "tab:red", + CAT_BITRATE_SWITCHING: "tab:purple", +} + def get_bins_for_diff(data: pd.Series): return np.linspace(data.min(), data.max(), num=10) @@ -76,6 +90,7 @@ def create_histograms( linewidth=0.5, label=cat, bottom=bottom, + color=COLORS_FOR_CATEGORIES[cat], ) bottom += data -- GitLab From 2830ade5340c7dfb28d2335bfb072bad32f3bfdf Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 14:16:52 +0200 Subject: [PATCH 59/70] fix wrong ref/cut file order in cmp_pcm call --- tests/renderer/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 6c17370678..e24dc40e2e 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -413,8 +413,8 @@ def run_renderer( split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) output_differs_splits, reason_splits = cmp_pcm( - out_file, out_file_ref, + out_file, out_fmt, ref_fs, get_mld=get_mld, -- GitLab From 883fa28c8675c72e80586ce81542577be56c9cc4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 14:44:32 +0200 Subject: [PATCH 60/70] fix calling record_property on being None --- tests/renderer/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index e24dc40e2e..82622fafe7 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -316,8 +316,9 @@ def run_renderer( "category": CAT_NORMAL, } - for k, v in testcase_props.items(): - record_property(k, v) + if record_property is not None: + for k, v in testcase_props.items(): + record_property(k, v) # run the renderer run_cmd(cmd, test_info, env) -- GitLab From 244927acd7e08a33c190d7fc5350d2713fc8fc72 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 1 Apr 2025 14:49:04 +0200 Subject: [PATCH 61/70] do not record properties when reference is being run --- .../test_param_file.py | 5 +-- tests/codec_be_on_mr_nonselection/test_sba.py | 31 ++++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index cc7b01ee4a..047644e8c0 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -348,8 +348,9 @@ def run_test( if eid_opts != "": testcase_props["category"] = CAT_PLC - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) assert bitstream_file == "bit" # in the parameter file, only "bit" is used as bitstream file name diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 3d2c586158..17d340d484 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -151,8 +151,10 @@ def test_pca_enc( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( @@ -307,8 +309,9 @@ def test_sba_enc_system( except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -459,8 +462,9 @@ def test_spar_hoa2_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -611,8 +615,9 @@ def test_spar_hoa3_enc_system( assert 0 input_config = SBA_FORMAT[abs(int(sba_order))] - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -778,8 +783,9 @@ def test_sba_enc_BWforce_system( except ValueError: testcase_props["category"] = CAT_BITRATE_SWITCHING - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -969,8 +975,9 @@ def test_sba_plc_system( if plc_pattern is not None: testcase_props["category"] = CAT_PLC - for k, v in testcase_props.items(): - dut_encoder_frontend.record_property(k, v) + if update_ref != 1: + for k, v in testcase_props.items(): + dut_encoder_frontend.record_property(k, v) if not decoder_only: sba_enc( -- GitLab From 19fe6270cb5d566ffcb85e6daa223e823752e0c8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 3 Apr 2025 15:37:43 +0200 Subject: [PATCH 62/70] add explicit fail if no split indices found --- tests/conftest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 69a13e6d7f..7f3788e209 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1278,7 +1278,14 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra format = "osba_hoa" else: format = input_file.split("_")[-1].lower() - idx = SPLIT_IDX.get(format, np.empty(0)).copy() + idx = SPLIT_IDX.get(format) + + if idx is None: + pytest.fail( + f"Could not get split indices for file {input_file} with infered format {format}." + ) + + idx = idx.copy() if len(idx) > 0 and sampling_rate_khz != 16: idx *= sampling_rate_khz // 16 -- GitLab From 6dc93d57d61bfed75d1d0e732c9e527cd0c26111 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 7 Apr 2025 11:45:16 +0200 Subject: [PATCH 63/70] back to empty split idx array as default and no fail also avoid double comparison of unsplitted files --- tests/codec_be_on_mr_nonselection/test_param_file.py | 3 +++ tests/codec_be_on_mr_nonselection/test_sba.py | 4 ++++ tests/conftest.py | 9 +++------ tests/renderer/utils.py | 4 ++++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 047644e8c0..d2acacd39e 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -648,6 +648,9 @@ def run_test( if split_comparison and not sim_opts: split_idx = get_split_idx(str(Path(testv_file).stem), int(sampling_rate)) + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: output_differs_splits, reason_splits = cmp_pcm( ref_file, dut_output_file, diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 17d340d484..563f8ac9ad 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1357,6 +1357,9 @@ def sba_dec( input_file = f"{test_vector_path}/{tag}.wav" split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: output_differs_splits, reason_splits = cmp_pcm( ref_out_file, dut_out_file, @@ -1378,6 +1381,7 @@ def sba_dec( output_differs_parts += output_differs_splits reason_parts += reason_splits + if split_comparison: prop_suffix = ["_whole"] + [ f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] diff --git a/tests/conftest.py b/tests/conftest.py index 7f3788e209..5d3632a762 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1278,13 +1278,10 @@ def get_split_idx(input_file: str, sampling_rate_khz: int) -> Optional[np.ndarra format = "osba_hoa" else: format = input_file.split("_")[-1].lower() - idx = SPLIT_IDX.get(format) - - if idx is None: - pytest.fail( - f"Could not get split indices for file {input_file} with infered format {format}." - ) + idx = SPLIT_IDX.get(format, np.empty(0)) + # copy is important because we modify the array below for fs != 16 + # without copy, the constant would be modified and future split values would be wrong idx = idx.copy() if len(idx) > 0 and sampling_rate_khz != 16: diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 82622fafe7..f6f10e6ea7 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -413,6 +413,9 @@ def run_renderer( if split_comparison: split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: output_differs_splits, reason_splits = cmp_pcm( out_file_ref, out_file, @@ -433,6 +436,7 @@ def run_renderer( output_differs_parts += output_differs_splits reason_parts += reason_splits + if split_comparison: prop_suffix = ["_whole"] + [ f"_split{i:03d}" for i in range(1, len(split_idx) + 1) ] -- GitLab From 978f5e327dcce5e77aa16ecf976d33cd858e4870 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 7 Apr 2025 12:21:37 +0200 Subject: [PATCH 64/70] add split summary link --- ci/basop-pages/create_report_pages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 2f01db7bde..41ee81af1b 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -37,6 +37,7 @@ Comparing:

Summary page

+

Split comparison summary page



-- GitLab From 068146d4f72fa7d96ef686fc0a754fbf39c24ee6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 7 Apr 2025 14:17:50 +0200 Subject: [PATCH 65/70] adapt diff_report.py to new csv format --- scripts/diff_report.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/diff_report.py b/scripts/diff_report.py index 5ab64e956f..0aefeaccdc 100644 --- a/scripts/diff_report.py +++ b/scripts/diff_report.py @@ -33,25 +33,29 @@ the United Nations Convention on Contracts on the International Sales of Goods. import pandas as pd import argparse import sys -import os -import pathlib COLUMNS_TO_COMPARE = [ "MLD", - "MAXIMUM ABS DIFF", + "MAX_ABS_DIFF", "MIN_SSNR", "MIN_ODG", ] + def main(args): - df_ref = pd.read_csv(args.csv_ref, sep=";") - df_test = pd.read_csv(args.csv_test, sep=";") + df_ref = pd.read_csv(args.csv_ref).sort_values( + by=["testcase", "format", "category"] + ) + df_test = pd.read_csv(args.csv_test).sort_values( + by=["testcase", "format", "category"] + ) for col in COLUMNS_TO_COMPARE: df_ref[col] = df_test[col] - df_ref[col] - df_ref.to_csv(args.csv_diff, index=False, sep=";") + df_ref.to_csv(args.csv_diff, index=False) return 0 + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_ref") -- GitLab From d898cda02da3bb53e1ec5676a744d32fd9806ad3 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Thu, 17 Apr 2025 11:31:01 +0200 Subject: [PATCH 66/70] add SSNR to batch_comp_audio.py --- scripts/batch_comp_audio.py | 77 ++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/scripts/batch_comp_audio.py b/scripts/batch_comp_audio.py index 7372857edb..32ab7bebb0 100755 --- a/scripts/batch_comp_audio.py +++ b/scripts/batch_comp_audio.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import argparse @@ -95,7 +95,7 @@ def main(args): repeat(fol2), repeat(outputs), repeat(tool), - repeat(test_offset_ms) + repeat(test_offset_ms), ) if args.sort: @@ -164,10 +164,17 @@ def compare_files(f, fol1, fol2, outputs_dict, tool, test_offset_ms): s2, fs2 = readfile(f2, outdtype="int16") cmp_result = compare(s1, s2, fs1, per_frame=False, get_mld=True) tool_output = cmp_result["MLD"] + elif tool == "ssnr": + s1, fs1 = readfile(f1, outdtype="int16") + s2, fs2 = readfile(f2, outdtype="int16") + cmp_result = compare(s1, s2, fs1, per_frame=False, get_ssnr=True) + tool_output = cmp_result["SSNR"] elif tool == "pyaudio3dtools": s1, fs1 = readfile(f1, outdtype="int16") s2, fs2 = readfile(f2, outdtype="int16") - cmp_result = compare(s1, s2, fs1, per_frame=False, test_start_offset_ms=test_offset_ms) + cmp_result = compare( + s1, s2, fs1, per_frame=False, test_start_offset_ms=test_offset_ms + ) tool_output = cmp_result["max_abs_diff"] with threading.Lock(): @@ -298,16 +305,16 @@ if __name__ == "__main__": ) parser.add_argument( "--tool", - choices=["mld", "CompAudio", "pyaudio3dtools"], + choices=["mld", "CompAudio", "pyaudio3dtools", "ssnr"], default="CompAudio", help="Compare tool to run", ) parser.add_argument( - "--test_offset_ms", - type=int, - default=0, - help="Offset in miliseconds that is ignored at the start of the files in folder2 (only used if tool=pyaudio3dtools)" - ) + "--test_offset_ms", + type=int, + default=0, + help="Offset in miliseconds that is ignored at the start of the files in folder2 (only used if tool=pyaudio3dtools)", + ) args = parser.parse_args() sys.exit(main(args)) -- GitLab From 2b476deec147df0a0eecc65843b45db37eb7cc19 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 9 Apr 2025 16:02:38 +0200 Subject: [PATCH 67/70] no split-comp for SID cut cases --- tests/codec_be_on_mr_nonselection/test_sba.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 563f8ac9ad..6b1746f858 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1353,7 +1353,8 @@ def sba_dec( ) # 2. run comparison on split files if --split_comparison is given - if split_comparison: + # in the "cut until bitstream starts with SID" case, we can't do the split comp as the indices would not match anymore + if split_comparison and sid != 1: input_file = f"{test_vector_path}/{tag}.wav" split_idx = get_split_idx(str(Path(input_file).stem), int(sampling_rate)) -- GitLab From 6ef29884512c33152ed1ac0ad93e571b5b4b9b0b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 9 Apr 2025 16:38:56 +0200 Subject: [PATCH 68/70] some tweaking for --no-bins --- scripts/create_histograms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 5ec2f0e599..977af57777 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -41,7 +41,7 @@ COLORS_FOR_CATEGORIES = { def get_bins_for_diff(data: pd.Series): - return np.linspace(data.min(), data.max(), num=10) + return np.round(np.linspace(data.min(), data.max(), num=10), decimals=2) def create_histograms( @@ -102,7 +102,7 @@ def create_histograms( ax.set_title(fmt) ax.legend(loc="best") ax.set_xlabel(measure) - if "DIFF" in measure: + if "DIFF" in measure or len(bins_for_measures) == 0: ax.set_xticks(range(len(x)), x, rotation=35) else: ax.set_xticks(range(len(x)), x) -- GitLab From d290d0f27e4969d3cd349b707b604f28b8492518 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 10 Apr 2025 14:04:35 +0200 Subject: [PATCH 69/70] fix regex for EVS format --- tests/test_26444.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_26444.py b/tests/test_26444.py index d3305e9984..25c72421f5 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -60,7 +60,7 @@ FORMATS_4_SCRIPTS = dict( ) ) -PATTERN_EVS_FORMAT = re.compile(r"-(" + r"|".join(FORMATS_4_SCRIPTS.keys()) + ")") +PATTERN_EVS_FORMAT = re.compile(r"(" + r"|".join(FORMATS_4_SCRIPTS.keys()) + ")") for s in SCRIPTS: with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp: -- GitLab From 1fde08ce7194acfa2d2c8d66f4f7d80f66bf1977 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 10 Apr 2025 14:23:37 +0200 Subject: [PATCH 70/70] add workaround for category import not working sometimes --- scripts/create_histograms.py | 18 ++++++++++++++---- tests/constants.py | 2 ++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/create_histograms.py b/scripts/create_histograms.py index 977af57777..b2a9f0ec1c 100644 --- a/scripts/create_histograms.py +++ b/scripts/create_histograms.py @@ -26,10 +26,20 @@ BINS_FOR_MEASURES = { DEFAULT_MEASURES = ["MAX_ABS_DIFF", "MLD", "MIN_SSNR", "MIN_ODG"] -HERE = pathlib.Path(__file__).parent -ROOT_DIR = HERE.parent -sys.path.append(str(ROOT_DIR)) -from tests.constants import CAT_NORMAL, CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_PLC +### !!! Note: this is duplicated in tests/constatns.py. If you change this here, ALSO ADAPT IT THERE!!! +### (importing from there failed for unknown reasons in some jobs on some runners and I don't have time to properly investigate this...) +### below lines are the original solution, kept here for reference + +# HERE = pathlib.Path(__file__).parent +# ROOT_DIR = HERE.parent +# sys.path.append(str(ROOT_DIR)) +# from tests.constants import CAT_NORMAL, CAT_BITRATE_SWITCHING, CAT_DTX, CAT_JBM, CAT_PLC + +CAT_NORMAL = "normal operation" +CAT_DTX = "DTX" +CAT_PLC = "PLC" +CAT_BITRATE_SWITCHING = "bitrate switching" +CAT_JBM = "JBM" COLORS_FOR_CATEGORIES = { CAT_DTX: "tab:blue", diff --git a/tests/constants.py b/tests/constants.py index 60a9f6e846..d70d125194 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -66,6 +66,8 @@ ENC_AUX_FILES = [ ["vad_flag", np.int16, "fs/50"], ] +### !!! Note: this is duplicated in scripts/create_histogram.py. If you change this here, ALSO ADAPT IT THERE!!! +### (importing from here failed for unknown reasons in some jobs on some runners and I don't have time to properly investigate this...) CAT_NORMAL = "normal operation" CAT_DTX = "DTX" CAT_PLC = "PLC" -- GitLab