From 2c0e6187f41fa818f5c8178650cbf2d7e665f0b1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 10 Jan 2025 16:34:18 +0100 Subject: [PATCH 01/11] add writeout of encoder dmx to pytest --- .../test_param_file.py | 11 ++-- tests/codec_be_on_mr_nonselection/test_sba.py | 21 +++++-- tests/conftest.py | 56 ++++++++++++++----- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 2d90309601..1d50e4ae14 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -205,8 +205,8 @@ def test_param_file_tests( get_enc_stats, get_odg, compare_to_input, + compare_enc_dmx, ): - enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] run_test( @@ -236,6 +236,7 @@ def test_param_file_tests( get_enc_stats, get_odg, compare_to_input, + compare_enc_dmx, ) @@ -266,8 +267,8 @@ def run_test( get_enc_stats, get_odg, compare_to_input, + compare_enc_dmx, ): - # If compare_to_input is set, only run pass-through test cases if compare_to_input: passthrough = [ @@ -328,6 +329,7 @@ def run_test( enc_split, update_ref, get_enc_stats, + compare_enc_dmx, ) # compare binary files extracted from the encoder @@ -489,7 +491,6 @@ def run_test( ) if update_ref in [0, 2]: - # Output file names for comparison dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" ref_output_file = f"{reference_path}/param_file/dec/{output_file}" @@ -499,7 +500,6 @@ def run_test( odg_test = None odg_ref = None if get_odg: - # Find input format in_fmt = [(a, b) for (a, b) in INPUT_FMT if re.search(a, enc_opts)][0][1] @@ -661,6 +661,7 @@ def encode( enc_opts_list, update_ref, get_enc_stats=False, + compare_enc_dmx=False, ): """ Call REF and/or DUT encoder. @@ -691,6 +692,7 @@ def encode( ref_out_file, add_option_list=enc_opts_list, stats_file=ref_stats_file, + compare_enc_dmx=compare_enc_dmx, ) if update_ref in [0, 2]: @@ -704,6 +706,7 @@ def encode( dut_out_file, add_option_list=enc_opts_list, stats_file=dut_stats_file, + compare_enc_dmx=compare_enc_dmx, ) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index da39091567..2c9d69b5c4 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -33,7 +33,6 @@ __doc__ = """ The outputs are compared with C generated references. """ -import errno import os import pytest from cut_bs import cut_from_start @@ -81,7 +80,8 @@ plc_patterns = ["PLperc12mblen5", "PLperc40mblen50", "PLperc42mblen2"] ivas_br_plc = ["13200", "16400", "32000", "64000", "96000", "256000"] # SBA order to IVAS_rend format table -SBA_FORMAT={1: "FOA", 2: "HOA2", 3: "HOA3"} +SBA_FORMAT = {1: "FOA", 2: "HOA2", 3: "HOA3"} + @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("sampling_rate", sample_rate_list) @@ -109,6 +109,7 @@ def test_pca_enc( get_odg, get_enc_stats, compare_to_input, + compare_enc_dmx, ): pca = True bitrate = "256000" @@ -156,6 +157,7 @@ def test_pca_enc( pca=pca, plc_pattern=plc_pattern, get_enc_stats=get_enc_stats, + compare_enc_dmx=compare_enc_dmx, ) if not encoder_only: @@ -226,8 +228,8 @@ def test_sba_enc_system( get_odg, get_enc_stats, compare_to_input, + compare_enc_dmx, ): - plc_pattern = None pca = False max_bw = "FB" @@ -297,6 +299,7 @@ def test_sba_enc_system( pca=pca, plc_pattern=plc_pattern, get_enc_stats=get_enc_stats, + compare_enc_dmx=compare_enc_dmx, ) if update_ref == 0 and get_enc_stats: @@ -394,6 +397,7 @@ def test_spar_hoa2_enc_system( get_odg, get_enc_stats, compare_to_input, + compare_enc_dmx, ): sampling_rate = "48" pca = False @@ -439,6 +443,7 @@ def test_spar_hoa2_enc_system( pca=pca, plc_pattern=plc_pattern, get_enc_stats=get_enc_stats, + compare_enc_dmx=compare_enc_dmx, ) if update_ref == 0 and get_enc_stats: @@ -536,6 +541,7 @@ def test_spar_hoa3_enc_system( get_odg, get_enc_stats, compare_to_input, + compare_enc_dmx, ): sampling_rate = "48" pca = False @@ -581,6 +587,7 @@ def test_spar_hoa3_enc_system( pca=pca, plc_pattern=plc_pattern, get_enc_stats=get_enc_stats, + compare_enc_dmx=compare_enc_dmx, ) if get_enc_stats: @@ -676,6 +683,7 @@ def test_sba_enc_BWforce_system( get_odg, get_enc_stats, compare_to_input, + compare_enc_dmx, ): sid = 0 plc_pattern = None @@ -731,6 +739,7 @@ def test_sba_enc_BWforce_system( pca=pca, plc_pattern=plc_pattern, get_enc_stats=get_enc_stats, + compare_enc_dmx=compare_enc_dmx, ) if update_ref == 0 and get_enc_stats: @@ -836,6 +845,7 @@ def test_sba_plc_system( get_odg, get_enc_stats, compare_to_input, + compare_enc_dmx, ): sid = 0 pca = False @@ -903,6 +913,7 @@ def test_sba_plc_system( pca=pca, plc_pattern=plc_pattern, get_enc_stats=get_enc_stats, + compare_enc_dmx=compare_enc_dmx, ) if not encoder_only: @@ -964,8 +975,8 @@ def sba_enc( pca=False, plc_pattern=None, get_enc_stats=False, + compare_enc_dmx=False, ): - input_path = f"{test_vector_path}/{tag}.wav" dtx_mode = dtx == "1" @@ -1028,6 +1039,7 @@ def sba_enc( pca=pca, dtx_mode=dtx_mode, stats_file=ref_stats_file, + compare_enc_dmx=compare_enc_dmx, ) if update_ref == 0: @@ -1042,6 +1054,7 @@ def sba_enc( pca=pca, dtx_mode=dtx_mode, stats_file=dut_stats_file, + compare_enc_dmx=compare_enc_dmx, ) if sid == 1: diff --git a/tests/conftest.py b/tests/conftest.py index c4795cb863..46b340e2d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -274,7 +274,14 @@ def pytest_addoption(parser): help="Compare output to the input file instead of reference output. N.B. Only applicable to pass-through tests.", default=False, ) - + + parser.addoption( + "--compare_enc_dmx", + action="store_true", + help="Trigger comparison of dmx signals written out from the encoder. If --update_ref is 1, dmx is written without comparison.", + default=False, + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -360,6 +367,11 @@ def compare_bitstream(request) -> bool: return request.config.option.compare_bitstream +@pytest.fixture(scope="session") +def compare_enc_dmx(request) -> bool: + return request.config.option.compare_enc_dmx + + @pytest.fixture(scope="session") def dut_encoder_path(request) -> str: """ @@ -387,11 +399,13 @@ def dut_encoder_path(request) -> str: return path + # fixture returns test information, enabling per-testcase SNR @pytest.fixture def test_info(request): return request + class EncoderFrontend: def __init__(self, path, enc_type, record_property, timeout=None) -> None: self._path = Path(path).absolute() @@ -492,6 +506,7 @@ class EncoderFrontend: add_option_list: Optional[list] = None, run_dir: Optional[Path] = None, stats_file: Optional[Path] = None, + compare_enc_dmx: Optional[bool] = False, ) -> None: command = [str(self._path)] @@ -525,13 +540,13 @@ class EncoderFrontend: cmd_str = textwrap.indent(" ".join(command), prefix="\t") log_dbg_msg(f"{self._type} encoder command:\n{cmd_str}") - try: - with tempfile.TemporaryDirectory() as tmp_dir: - if run_dir is None: - cwd = Path(tmp_dir).absolute() - else: - cwd = Path(run_dir).absolute() + with tempfile.TemporaryDirectory() as tmp_dir: + if run_dir is None: + cwd = Path(tmp_dir).absolute() + else: + cwd = Path(run_dir).absolute() + try: result = run( command, capture_output=True, @@ -539,14 +554,28 @@ class EncoderFrontend: timeout=self.timeout, cwd=cwd, ) + except TimeoutExpired: + pytest.fail( + f"{self._type} encoder run timed out after {self.timeout}s." + ) - if stats_file is not None: - self.extract_enc_stats( - cwd.joinpath("res"), stats_file, input_sampling_rate - ) + if stats_file is not None: + self.extract_enc_stats( + cwd.joinpath("res"), stats_file, input_sampling_rate + ) - except TimeoutExpired: - pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.") + if compare_enc_dmx: + for dmx_file in cwd.glob("res/ivas_input_dmx.id*.pcm"): + id_match = re.search(r"id(\d+).pcm", dmx_file.name) + if id_match is None: + pytest.fail( + "No dmx signal files found - did you build with DEBUG_MODE_INFO?" + ) + assert id_match is not None + id = id_match.group(1) + dmx_file.rename( + Path(output_bitstream_path).with_suffix(f".dmx.ch{id}.pcm") + ) self.returncode = result.returncode self.stderr = result.stderr.decode("ascii") @@ -1052,7 +1081,6 @@ def compare_to_input(request) -> bool: return request.config.getoption("--compare_to_input") - def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial") if config.option.param_file: -- GitLab From 844f99d4ca02fb45fc28ec7b5f2b3b0d129794c2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 10 Jan 2025 17:53:19 +0100 Subject: [PATCH 02/11] add dmx comparison + add to report --- tests/cmp_pcm.py | 76 +++++++++++++------ .../test_param_file.py | 31 +++++++- tests/conftest.py | 3 +- tests/constants.py | 1 + 4 files changed, 83 insertions(+), 28 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 9b837156bb..8263eaccff 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -15,6 +15,7 @@ sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) import numpy as np import pyaudio3dtools import pyivastest + # Hack to resolve import when using from command line or from within scripts. try: from .constants import ODG_PATTERN_PQEVALAUDIO @@ -25,7 +26,7 @@ except ImportError: def cmp_pcm( ref_file, cmp_file, - out_config, + nchannels: int, fs, get_mld=False, allow_differing_lengths=False, @@ -38,23 +39,18 @@ def cmp_pcm( odg_ref=None, ref_jbm_tf: Optional[Path] = None, cut_jbm_tf: Optional[Path] = None, + quiet: Optional[bool] = False, ) -> (int, str): """ Compare 2 PCM files for bitexactness """ - print("Cmp PCM Report") - print("=====================") - out_config = "MONO" if out_config == "" else out_config - # out_config may be a string or a Path. Wrap in str() to avoid error in case it is a Path. - if str(out_config).upper() not in pyivastest.constants.OC_TO_NCHANNELS: - nchannels = ( - pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout( - out_config - ) - ) - else: - nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] + output_target = sys.stdout + if quiet: + output_target = open(os.devnull, "w") + + print("Cmp PCM Report", file=output_target) + print("=====================", file=output_target) s1, fs1 = pyaudio3dtools.audiofile.readfile( ref_file, nchannels, fs, outdtype=np.int16 @@ -80,12 +76,17 @@ def cmp_pcm( if allow_differing_lengths: # to allow for MLD comparison, pad shorter file max_len = max(s1.shape[0], s2.shape[0]) - s1 = np.pad(s1,((0,max_len - s1.shape[0]),(0,0)),mode='constant',constant_values=0) - s2 = np.pad(s2,((0,max_len - s2.shape[0]),(0,0)),mode='constant',constant_values=0) + s1 = np.pad( + s1, ((0, max_len - s1.shape[0]), (0, 0)), mode="constant", constant_values=0 + ) + s2 = np.pad( + s2, ((0, max_len - s2.shape[0]), (0, 0)), mode="constant", constant_values=0 + ) elif s1.shape != s2.shape: print( f"file size in samples: file 1 = {s1.shape[0]},", f"file 2 = {s2.shape[0]}", + file=output_target, ) reason = "FAIL: File lengths differ. MAXIMUM ABS DIFF: None" if get_mld: @@ -96,7 +97,7 @@ def cmp_pcm( cmp_result = pyaudio3dtools.audioarray.compare( s1, s2, - fs, + fs * 1000, per_frame=False, get_mld=get_mld, get_ssnr=get_ssnr, @@ -113,8 +114,8 @@ def cmp_pcm( elif not cmp_result["bitexact"]: diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" - print(diff_msg) - print(first_msg) + print(diff_msg, file=output_target) + print(first_msg, file=output_target) reason = f"Non-BE - MAXIMUM ABS DIFF: {cmp_result['max_abs_diff']}" output_differs = 1 @@ -122,7 +123,7 @@ def cmp_pcm( if get_mld: mld_msg = f"MLD: {cmp_result['MLD']}" reason += " - " + mld_msg - print(mld_msg) + print(mld_msg, file=output_target) if cmp_result["MLD"] <= mld_lim: output_differs = 0 @@ -149,19 +150,26 @@ def cmp_pcm( 32767, ).astype(np.int16) - pqeval_output = pqevalaudio_wrapper(odg_files[odg_input], odg_files[odg_ref], 48000) + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_ref], 48000 + ) match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) odg_ref = float(match_odg.groups()[0]) - pqeval_output = pqevalaudio_wrapper(odg_files[odg_input], odg_files[odg_test], 48000) + pqeval_output = pqevalaudio_wrapper( + odg_files[odg_input], odg_files[odg_test], 48000 + ) match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) odg_test = float(match_odg.groups()[0]) - odg = odg_test - odg_ref # Todo: store both rather than difference? + odg = odg_test - odg_ref # Todo: store both rather than difference? msg = f"ODG: {odg}" reason += " - " + msg - print(msg) + print(msg, file=output_target) + + if quiet: + output_target.close() return output_differs, reason @@ -200,6 +208,21 @@ def pqevalaudio_wrapper( return result.stdout.decode("utf8") +def out_config_2_nchannels(out_config): + out_config = "MONO" if out_config == "" else out_config + # out_config may be a string or a Path. Wrap in str() to avoid error in case it is a Path. + if str(out_config).upper() not in pyivastest.constants.OC_TO_NCHANNELS: + nchannels = ( + pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout( + out_config + ) + ) + else: + nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] + + return nchannels + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("ref_file", type=str) @@ -217,8 +240,11 @@ if __name__ == "__main__": parser.add_argument("--get_odg", action="store_true") parser.add_argument("--get_ssnr", action="store_true") parser.add_argument("--allow_differing_lengths", action="store_true") - args = parser.parse_args() + parser.add_argument("--quiet", action="store_true") + args = vars(parser.parse_args()) + + args["nchannels"] = out_config_2_nchannels(args.pop("out_config")) - result, msg = cmp_pcm(**vars(args)) + result, msg = cmp_pcm(**args) print(msg) sys.exit(result) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 1d50e4ae14..2aadc3c109 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -39,9 +39,15 @@ from pathlib import Path from subprocess import run import pytest import re +import sys import numpy as np -from tests.cmp_pcm import cmp_pcm +THIS_PATH = os.path.join(os.getcwd(), __file__) +sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../../scripts")) + +import pyaudio3dtools + +from tests.cmp_pcm import cmp_pcm, out_config_2_nchannels from tests.cmp_stats_files import cmp_stats_files from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties from tests.testconfig import PARAM_FILE @@ -50,6 +56,7 @@ from tests.constants import ( MAX_ENC_STATS_DIFF, SCRIPTS_DIR, MAX_ENC_DIFF, + DMX_DIFF, ) from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -491,6 +498,26 @@ def run_test( ) if update_ref in [0, 2]: + if compare_enc_dmx: + dut_dmx_files = Path( + f"{dut_base_path}/param_file/enc/{bitstream_file}" + ).parent.glob(f"{Path(bitstream_file).stem}.dmx.ch*.pcm") + + for dut_dmx_file in dut_dmx_files: + ref_dmx_file = str(dut_dmx_file).replace(dut_base_path, reference_path) + testv_sig, _ = pyaudio3dtools.audiofile.readfile(testv_file) + nchannels = testv_sig.shape[1] + dmx_differs, reason = cmp_pcm( + ref_dmx_file, + dut_dmx_file, + nchannels, + in_sr, + quiet=True, + ) + + prop = parse_properties(reason, dmx_differs, [DMX_DIFF]) + dut_decoder_frontend.record_property(DMX_DIFF, prop[DMX_DIFF]) + # Output file names for comparison dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" ref_output_file = f"{reference_path}/param_file/dec/{output_file}" @@ -560,7 +587,7 @@ def run_test( output_differs, reason = cmp_pcm( ref_file, dut_output_file, - output_config, + out_config_2_nchannels(output_config), fs, get_mld=get_mld, mld_lim=get_mld_lim, diff --git a/tests/conftest.py b/tests/conftest.py index 46b340e2d5..a5aad8e71c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,7 @@ from typing import Optional, Union import numpy as np from .constants import ( # MAX_ENC_DIFF_NAME_PATTERN, + DMX_DIFF, MAX_ENC_DIFF_PARAM_NAME, MLD_PATTERN, MAX_DIFF_PATTERN, @@ -1131,7 +1132,7 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: if prop == MLD: mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0]) props[prop] = mld - elif prop == MAX_ABS_DIFF: + elif prop == MAX_ABS_DIFF or prop == DMX_DIFF: max_diff = 0 if output_differs: if (match := re.search(MAX_DIFF_PATTERN, text_to_parse)) is not None: diff --git a/tests/constants.py b/tests/constants.py index 562035f8f4..1b923e391e 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -16,6 +16,7 @@ MAX_ENC_DIFF_PARAM_NAME = "MAXIMUM ENC DIFF PARAM" ENC_CORE_OVL = "ENC_CORE_OVL" MAX_OVL = "MAX_OVL" MIN_OVL = "MIN_OVL" +DMX_DIFF = "DMX MAXIMUM ABS DIFF" # regex patterns for parsing the output from comparisons -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" -- GitLab From ac148bea7d7c6ecad448b186927c1cd4a4d43ed3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 14 Jan 2025 11:30:35 +0100 Subject: [PATCH 03/11] fix dmx comparison + add MLD and SSNR for dmx - combine multi-mono dmx signals into multichannel signal b4 comparison --- .../test_param_file.py | 57 +++++++++++++------ tests/conftest.py | 12 ++-- tests/constants.py | 2 + 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 2aadc3c109..aa3110c3c3 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -37,6 +37,7 @@ import os import platform from pathlib import Path from subprocess import run +from tempfile import NamedTemporaryFile import pytest import re import sys @@ -57,6 +58,8 @@ from tests.constants import ( SCRIPTS_DIR, MAX_ENC_DIFF, DMX_DIFF, + DMX_MLD, + DMX_SSNR, ) from tests.renderer.utils import check_and_makedir, binauralize_input_and_output @@ -499,24 +502,46 @@ def run_test( if update_ref in [0, 2]: if compare_enc_dmx: - dut_dmx_files = Path( - f"{dut_base_path}/param_file/enc/{bitstream_file}" - ).parent.glob(f"{Path(bitstream_file).stem}.dmx.ch*.pcm") - - for dut_dmx_file in dut_dmx_files: - ref_dmx_file = str(dut_dmx_file).replace(dut_base_path, reference_path) - testv_sig, _ = pyaudio3dtools.audiofile.readfile(testv_file) - nchannels = testv_sig.shape[1] - dmx_differs, reason = cmp_pcm( - ref_dmx_file, - dut_dmx_file, - nchannels, - in_sr, - quiet=True, + dut_dmx_files = sorted( + Path(f"{dut_base_path}/param_file/enc/{bitstream_file}").parent.glob( + f"{Path(bitstream_file).stem}.dmx.ch*.pcm" ) + ) + ref_dmx_files = [ + str(f).replace(dut_base_path, reference_path) for f in dut_dmx_files + ] + + nchannels = len(dut_dmx_files) + with NamedTemporaryFile(suffix=".ref_dmx.pcm") as dmx_file_ref_tmp: + with NamedTemporaryFile(suffix=".dut_dmx.pcm") as dmx_file_dut_tmp: + pyaudio3dtools.audiofile.combinefiles( + ref_dmx_files, + dmx_file_ref_tmp.name, + out_nchans=nchannels, + in_fs=in_sr, + ) + pyaudio3dtools.audiofile.combinefiles( + dut_dmx_files, + dmx_file_dut_tmp.name, + out_nchans=nchannels, + in_fs=in_sr, + ) + + dmx_differs, reason = cmp_pcm( + dmx_file_ref_tmp.name, + dmx_file_dut_tmp.name, + nchannels, + in_sr, + get_mld=True, + get_ssnr=True, + quiet=True, + ) - prop = parse_properties(reason, dmx_differs, [DMX_DIFF]) - dut_decoder_frontend.record_property(DMX_DIFF, prop[DMX_DIFF]) + dmx_props = [DMX_DIFF, DMX_MLD, DMX_SSNR] + pytest.set_trace() + prop_results = parse_properties(reason, dmx_differs, dmx_props) + for prop in dmx_props: + dut_decoder_frontend.record_property(prop, prop_results[prop]) # Output file names for comparison dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" diff --git a/tests/conftest.py b/tests/conftest.py index a5aad8e71c..1a8928923b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,6 +48,8 @@ import numpy as np from .constants import ( # MAX_ENC_DIFF_NAME_PATTERN, DMX_DIFF, + DMX_MLD, + DMX_SSNR, MAX_ENC_DIFF_PARAM_NAME, MLD_PATTERN, MAX_DIFF_PATTERN, @@ -1129,7 +1131,7 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: props = dict() for prop in props_to_record: - if prop == MLD: + if prop == MLD or prop == DMX_MLD: mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0]) props[prop] = mld elif prop == MAX_ABS_DIFF or prop == DMX_DIFF: @@ -1140,12 +1142,14 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: else: raise MaxDiffPatternNotFound() props[prop] = max_diff - elif prop == SSNR: + elif prop == SSNR or prop == DMX_SSNR: ssnrs = re.findall(SSNR_PATTERN, text_to_parse) min_ssnr = min(ssnrs) min_ssnr_channel = ssnrs.index(min_ssnr) - props["MIN_SSNR"] = min_ssnr - props["MIN_SSNR_CHANNEL"] = min_ssnr_channel + + prefix = "MIN" if prop == SSNR else "DMX" + props[f"{prefix}_SSNR"] = min_ssnr + props[f"{prefix}_SSNR_CHANNEL"] = min_ssnr_channel elif prop == ODG: odgs = re.findall(ODG_PATTERN, text_to_parse) min_odg = min(odgs) diff --git a/tests/constants.py b/tests/constants.py index 1b923e391e..865a7061a7 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -17,6 +17,8 @@ ENC_CORE_OVL = "ENC_CORE_OVL" MAX_OVL = "MAX_OVL" MIN_OVL = "MIN_OVL" DMX_DIFF = "DMX MAXIMUM ABS DIFF" +DMX_MLD = "DMX MLD" +DMX_SSNR = "DMX_SSNR" # regex patterns for parsing the output from comparisons -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" -- GitLab From c7ad1467366f0f75d1e05cc22eaf2810cbb3c08f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 14 Jan 2025 11:46:50 +0100 Subject: [PATCH 04/11] put dmx comparison in function --- .../test_param_file.py | 44 ++++--------------- tests/conftest.py | 36 +++++++++++++++ 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index aa3110c3c3..5824fcae0a 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -37,7 +37,6 @@ import os import platform from pathlib import Path from subprocess import run -from tempfile import NamedTemporaryFile import pytest import re import sys @@ -46,11 +45,14 @@ import numpy as np THIS_PATH = os.path.join(os.getcwd(), __file__) sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../../scripts")) -import pyaudio3dtools - from tests.cmp_pcm import cmp_pcm, out_config_2_nchannels from tests.cmp_stats_files import cmp_stats_files -from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties +from tests.conftest import ( + DecoderFrontend, + EncoderFrontend, + parse_properties, + compare_dmx_signals, +) from tests.testconfig import PARAM_FILE from tests.constants import ( MAX_ENC_FILE_LENGTH_DIFF, @@ -511,37 +513,9 @@ def run_test( str(f).replace(dut_base_path, reference_path) for f in dut_dmx_files ] - nchannels = len(dut_dmx_files) - with NamedTemporaryFile(suffix=".ref_dmx.pcm") as dmx_file_ref_tmp: - with NamedTemporaryFile(suffix=".dut_dmx.pcm") as dmx_file_dut_tmp: - pyaudio3dtools.audiofile.combinefiles( - ref_dmx_files, - dmx_file_ref_tmp.name, - out_nchans=nchannels, - in_fs=in_sr, - ) - pyaudio3dtools.audiofile.combinefiles( - dut_dmx_files, - dmx_file_dut_tmp.name, - out_nchans=nchannels, - in_fs=in_sr, - ) - - dmx_differs, reason = cmp_pcm( - dmx_file_ref_tmp.name, - dmx_file_dut_tmp.name, - nchannels, - in_sr, - get_mld=True, - get_ssnr=True, - quiet=True, - ) - - dmx_props = [DMX_DIFF, DMX_MLD, DMX_SSNR] - pytest.set_trace() - prop_results = parse_properties(reason, dmx_differs, dmx_props) - for prop in dmx_props: - dut_decoder_frontend.record_property(prop, prop_results[prop]) + prop_results = compare_dmx_signals(ref_dmx_files, dut_dmx_files, in_sr) + for prop, val in prop_results.items(): + dut_decoder_frontend.record_property(prop, val) # Output file names for comparison dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" diff --git a/tests/conftest.py b/tests/conftest.py index 1a8928923b..b991488bcd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,7 @@ import platform import textwrap from pathlib import Path from subprocess import TimeoutExpired, run +from tempfile import NamedTemporaryFile import tempfile from typing import Optional, Union import numpy as np @@ -78,6 +79,8 @@ import sys sys.path.append(str(SCRIPTS_DIR)) import prepare_combined_format_inputs +import pyaudio3dtools +from tests.cmp_pcm import cmp_pcm def log_dbg_msg(message): @@ -1169,3 +1172,36 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: class MaxDiffPatternNotFound(Exception): pass + + +def compare_dmx_signals(ref_dmx_files, dut_dmx_files, fs) -> dict: + nchannels = len(dut_dmx_files) + with NamedTemporaryFile(suffix=".ref_dmx.pcm") as dmx_file_ref_tmp: + with NamedTemporaryFile(suffix=".dut_dmx.pcm") as dmx_file_dut_tmp: + pyaudio3dtools.audiofile.combinefiles( + ref_dmx_files, + dmx_file_ref_tmp.name, + out_nchans=nchannels, + in_fs=fs, + ) + pyaudio3dtools.audiofile.combinefiles( + dut_dmx_files, + dmx_file_dut_tmp.name, + out_nchans=nchannels, + in_fs=fs, + ) + + dmx_differs, reason = cmp_pcm( + dmx_file_ref_tmp.name, + dmx_file_dut_tmp.name, + nchannels, + fs, + get_mld=True, + get_ssnr=True, + quiet=True, + ) + + dmx_props = [DMX_DIFF, DMX_MLD, DMX_SSNR] + prop_results = parse_properties(reason, dmx_differs, dmx_props) + + return prop_results -- GitLab From 32d41f381c26d5a5e72d901474086e25c1b228a7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 14 Jan 2025 14:12:23 +0100 Subject: [PATCH 05/11] port dmx comparison to test_sba.py --- tests/codec_be_on_mr_nonselection/test_sba.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 2c9d69b5c4..a0c72b64f0 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -36,10 +36,15 @@ __doc__ = """ import os import pytest from cut_bs import cut_from_start +from pathlib import Path from tests.cmp_pcm import cmp_pcm -from tests.conftest import DecoderFrontend, EncoderFrontend -from ..conftest import parse_properties +from tests.conftest import ( + DecoderFrontend, + EncoderFrontend, + compare_dmx_signals, + parse_properties, +) from ..cmp_stats_files import cmp_stats_files from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF from tests.testconfig import use_ltv @@ -1057,6 +1062,20 @@ def sba_enc( compare_enc_dmx=compare_enc_dmx, ) + if update_ref in [0, 2] and compare_enc_dmx: + dut_dmx_files = sorted( + Path(dut_pkt_file).parent.glob(f"{Path(dut_pkt_file).stem}.dmx.ch*.pcm") + ) + ref_dmx_files = [ + str(f).replace(dut_pkt_dir, ref_pkt_dir) for f in dut_dmx_files + ] + + prop_results = compare_dmx_signals( + ref_dmx_files, dut_dmx_files, int(sampling_rate) + ) + for prop, val in prop_results.items(): + dut_encoder_frontend.record_property(prop, val) + if sid == 1: # cut .pkt files such that they start with SID frame dut_pkt_file_cut = f"{dut_pkt_dir}/{tag_out}_cut.192" -- GitLab From c78fba62009e691d507c04d5c93898a25f318ec2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 11:15:42 +0100 Subject: [PATCH 06/11] use shutil.move instead of .rename the latter can have problems when copying over filesystem borders, e.g. when using WSL --- tests/conftest.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e9d782f1da..853412cb18 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,7 @@ import logging import os import re import json +import shutil from tests import testconfig import pytest import platform @@ -43,6 +44,7 @@ import textwrap from pathlib import Path from subprocess import TimeoutExpired, run from tempfile import NamedTemporaryFile +from shutil import move import tempfile from typing import Optional, Union import numpy as np @@ -577,11 +579,12 @@ class EncoderFrontend: pytest.fail( "No dmx signal files found - did you build with DEBUG_MODE_INFO?" ) - assert id_match is not None - id = id_match.group(1) - dmx_file.rename( - Path(output_bitstream_path).with_suffix(f".dmx.ch{id}.pcm") - ) + else: + id = id_match.group(1) + move( + dmx_file, + Path(output_bitstream_path).with_suffix(f".dmx.ch{id}.pcm"), + ) self.returncode = result.returncode self.stderr = result.stderr.decode("ascii") -- GitLab From f07d8cecc9c15c88c2d5c21b156f91571f7d6dbc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 21 Jan 2025 09:01:29 +0100 Subject: [PATCH 07/11] add same core dmx dumping mechanism as in BASOP --- lib_dec/lib_dec.c | 1 + lib_enc/ivas_core_pre_proc_front.c | 5 +++++ lib_enc/ivas_cpe_enc.c | 4 ---- lib_enc/ivas_sce_enc.c | 1 - 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index d4e4f91430..3ac7f16b26 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -45,6 +45,7 @@ #include #ifdef DEBUGGING #include "debug.h" +#include #endif #include "wmc_auto.h" diff --git a/lib_enc/ivas_core_pre_proc_front.c b/lib_enc/ivas_core_pre_proc_front.c index 97d6643c08..56d443f383 100644 --- a/lib_enc/ivas_core_pre_proc_front.c +++ b/lib_enc/ivas_core_pre_proc_front.c @@ -34,6 +34,7 @@ #include "options.h" #ifdef DEBUGGING #include "debug.h" +#include #endif #include "cnst.h" #include "ivas_cnst.h" @@ -220,6 +221,10 @@ ivas_error pre_proc_front_ivas( } } +#ifdef DEBUG_MODE_INFO + dbgwrite( &signal_in, sizeof( float ), input_frame, 1, strcat( fname( debug_dir, "ivas_input_dmx", 0, n + 1, ENC ), ".float32" ) ); +#endif + lMemRecalc_12k8 = 0; lMemRecalc = 0; if ( element_mode == IVAS_CPE_TD || element_mode == IVAS_CPE_MDCT ) diff --git a/lib_enc/ivas_cpe_enc.c b/lib_enc/ivas_cpe_enc.c index f2ede33d05..10ea129534 100644 --- a/lib_enc/ivas_cpe_enc.c +++ b/lib_enc/ivas_cpe_enc.c @@ -458,10 +458,6 @@ ivas_error ivas_cpe_enc( } #ifdef DEBUG_MODE_INFO - for ( n = 0; n < n_CoreChannels; n++ ) - { - dbgwrite( sts[0]->input - NS2SA( sts[0]->input_Fs, ACELP_LOOK_NS ), sizeof( float ), input_frame, 1, fname( debug_dir, "input_DMX", n, sts[n]->id_element, ENC ) ); - } dbgwrite( &hCPE->element_mode, sizeof( int16_t ), 1, input_frame, fname( debug_dir, "element_mode", 0, sts[0]->id_element, ENC ) ); #endif diff --git a/lib_enc/ivas_sce_enc.c b/lib_enc/ivas_sce_enc.c index b00af95ed3..80c58d36e8 100644 --- a/lib_enc/ivas_sce_enc.c +++ b/lib_enc/ivas_sce_enc.c @@ -172,7 +172,6 @@ ivas_error ivas_sce_enc( } #ifdef DEBUG_MODE_INFO - dbgwrite( st->input - NS2SA( st->input_Fs, ACELP_LOOK_NS ), sizeof( float ), input_frame, 1, "res/input_DMX" ); dbgwrite( &st->element_mode, sizeof( int16_t ), 1, input_frame, fname( debug_dir, "element_mode", 0, st->id_element, ENC ) ); #endif -- GitLab From 1cb64fb6341aa61887bb737592b85380cc895ee1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 21 Jan 2025 09:07:07 +0100 Subject: [PATCH 08/11] Revert "add same core dmx dumping mechanism as in BASOP" This reverts commit f07d8cecc9c15c88c2d5c21b156f91571f7d6dbc. --- lib_dec/lib_dec.c | 1 - lib_enc/ivas_core_pre_proc_front.c | 5 ----- lib_enc/ivas_cpe_enc.c | 4 ++++ lib_enc/ivas_sce_enc.c | 1 + 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index 3ac7f16b26..d4e4f91430 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -45,7 +45,6 @@ #include #ifdef DEBUGGING #include "debug.h" -#include #endif #include "wmc_auto.h" diff --git a/lib_enc/ivas_core_pre_proc_front.c b/lib_enc/ivas_core_pre_proc_front.c index 56d443f383..97d6643c08 100644 --- a/lib_enc/ivas_core_pre_proc_front.c +++ b/lib_enc/ivas_core_pre_proc_front.c @@ -34,7 +34,6 @@ #include "options.h" #ifdef DEBUGGING #include "debug.h" -#include #endif #include "cnst.h" #include "ivas_cnst.h" @@ -221,10 +220,6 @@ ivas_error pre_proc_front_ivas( } } -#ifdef DEBUG_MODE_INFO - dbgwrite( &signal_in, sizeof( float ), input_frame, 1, strcat( fname( debug_dir, "ivas_input_dmx", 0, n + 1, ENC ), ".float32" ) ); -#endif - lMemRecalc_12k8 = 0; lMemRecalc = 0; if ( element_mode == IVAS_CPE_TD || element_mode == IVAS_CPE_MDCT ) diff --git a/lib_enc/ivas_cpe_enc.c b/lib_enc/ivas_cpe_enc.c index 10ea129534..f2ede33d05 100644 --- a/lib_enc/ivas_cpe_enc.c +++ b/lib_enc/ivas_cpe_enc.c @@ -458,6 +458,10 @@ ivas_error ivas_cpe_enc( } #ifdef DEBUG_MODE_INFO + for ( n = 0; n < n_CoreChannels; n++ ) + { + dbgwrite( sts[0]->input - NS2SA( sts[0]->input_Fs, ACELP_LOOK_NS ), sizeof( float ), input_frame, 1, fname( debug_dir, "input_DMX", n, sts[n]->id_element, ENC ) ); + } dbgwrite( &hCPE->element_mode, sizeof( int16_t ), 1, input_frame, fname( debug_dir, "element_mode", 0, sts[0]->id_element, ENC ) ); #endif diff --git a/lib_enc/ivas_sce_enc.c b/lib_enc/ivas_sce_enc.c index 80c58d36e8..b00af95ed3 100644 --- a/lib_enc/ivas_sce_enc.c +++ b/lib_enc/ivas_sce_enc.c @@ -172,6 +172,7 @@ ivas_error ivas_sce_enc( } #ifdef DEBUG_MODE_INFO + dbgwrite( st->input - NS2SA( st->input_Fs, ACELP_LOOK_NS ), sizeof( float ), input_frame, 1, "res/input_DMX" ); dbgwrite( &st->element_mode, sizeof( int16_t ), 1, input_frame, fname( debug_dir, "element_mode", 0, st->id_element, ENC ) ); #endif -- GitLab From 73317282c0aa50c1a5cd8656d53d70472443b7e0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 21 Jan 2025 09:09:24 +0100 Subject: [PATCH 09/11] add correct dumping mechanism ported from ivas-float-update --- lib_dec/lib_dec.c | 1 + lib_enc/ivas_core_pre_proc_front.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index d4e4f91430..3ac7f16b26 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -45,6 +45,7 @@ #include #ifdef DEBUGGING #include "debug.h" +#include #endif #include "wmc_auto.h" diff --git a/lib_enc/ivas_core_pre_proc_front.c b/lib_enc/ivas_core_pre_proc_front.c index 97d6643c08..f330321758 100644 --- a/lib_enc/ivas_core_pre_proc_front.c +++ b/lib_enc/ivas_core_pre_proc_front.c @@ -34,6 +34,7 @@ #include "options.h" #ifdef DEBUGGING #include "debug.h" +#include #endif #include "cnst.h" #include "ivas_cnst.h" @@ -220,6 +221,14 @@ ivas_error pre_proc_front_ivas( } } +#ifdef DEBUG_MODE_INFO + { + int16_t tmp_dmx_in[L_FRAME48k]; + mvr2s( signal_in - NS2SA( st->input_Fs, ACELP_LOOK_NS ), tmp_dmx_in, input_frame ); + dbgwrite( tmp_dmx_in, sizeof( int16_t ), input_frame, 1, strcat( fname( debug_dir, "ivas_input_dmx", 0, n + 1, ENC ), ".pcm" ) ); + } +#endif + lMemRecalc_12k8 = 0; lMemRecalc = 0; if ( element_mode == IVAS_CPE_TD || element_mode == IVAS_CPE_MDCT ) -- GitLab From 877862b6ace27f6b5642d2eef5266d1e8c78942f Mon Sep 17 00:00:00 2001 From: malenov Date: Thu, 23 Jan 2025 12:27:24 +0100 Subject: [PATCH 10/11] avoid the dumping of the secondary channel in TD stereo --- lib_enc/ivas_core_pre_proc_front.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib_enc/ivas_core_pre_proc_front.c b/lib_enc/ivas_core_pre_proc_front.c index f330321758..36da36c128 100644 --- a/lib_enc/ivas_core_pre_proc_front.c +++ b/lib_enc/ivas_core_pre_proc_front.c @@ -222,6 +222,7 @@ ivas_error pre_proc_front_ivas( } #ifdef DEBUG_MODE_INFO + if ( !( hCPE->hStereoTD != NULL && n > 0 ) ) { int16_t tmp_dmx_in[L_FRAME48k]; mvr2s( signal_in - NS2SA( st->input_Fs, ACELP_LOOK_NS ), tmp_dmx_in, input_frame ); -- GitLab From 82ab5823e0eea9adab71e944123ba5ad6f642823 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Fri, 24 Jan 2025 13:20:00 +0100 Subject: [PATCH 11/11] use with_suffix() to append .exe for Windows platforms --- scripts/pyaudio3dtools/audioarray.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 5741c1ada4..320241f4ed 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -383,8 +383,9 @@ def compare( raise NotImplementedError( f"wav-diff tool not available for {curr_platform}" ) + search_path = toolsdir.joinpath(curr_platform.replace("Windows", "Win32")) - wdiff = search_path.joinpath("wav-diff") + wdiff = search_path.joinpath("wav-diff").with_suffix(".exe" if curr_platform == "Windows" else "") if not wdiff.exists(): wdiff = shutil.which("wav-diff") -- GitLab