From 1a1717f1f3b636541d9f2a6b89d63a424e0afc1c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 8 Jul 2024 16:13:59 +0200 Subject: [PATCH 01/28] execute codec binaries always in temporary dir --- .../test_param_file.py | 10 ++- .../test_sba_bs_dec_plc.py | 2 +- .../test_sba_bs_enc.py | 4 +- tests/conftest.py | 72 +++++++++++++------ 4 files changed, 62 insertions(+), 26 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 814a6441df..dc10a17b57 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -176,7 +176,7 @@ def test_param_file_tests( # bitrate can be a filename: remove leading "../" if bitrate.startswith("../"): - bitrate = bitrate[3:] + bitrate = Path(bitrate[3:]).absolute() testv_base = testv_file.split("/")[-1] if testv_base.endswith(".pcm"): @@ -331,8 +331,12 @@ def test_param_file_tests( ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}" # check for same RTP sequence number in last line of tracefile - dut_rtp_num_last = np.genfromtxt(dut_tracefile_dec, delimiter=";", usecols=[0])[-1] - ref_rtp_num_last = np.genfromtxt(ref_tracefile_dec, delimiter=";", usecols=[0])[-1] + dut_rtp_num_last = np.genfromtxt( + dut_tracefile_dec, delimiter=";", usecols=[0] + )[-1] + ref_rtp_num_last = np.genfromtxt( + ref_tracefile_dec, delimiter=";", usecols=[0] + )[-1] tracefile_last_rtp_numbers_differ = dut_rtp_num_last != ref_rtp_num_last # same sequence number -> likely no crash, assume length difference is due to difference in TSM diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index e188d4a114..a18ebd2a93 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -221,7 +221,7 @@ def sba_dec_plc( get_ssnr=get_ssnr, ) - props = parse_properties(reason, cmp_result!=0, props_to_record) + props = parse_properties(reason, cmp_result != 0, props_to_record) for k, v in props.items(): record_property(k, v) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index ce99c55a27..b6c24c3ce2 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -35,6 +35,8 @@ __doc__ = """ import errno import os +import tempfile +from pathlib import Path import pytest from cut_bs import cut_from_start @@ -753,7 +755,7 @@ def sba_dec( get_ssnr=get_ssnr, ) - props = parse_properties(reason, cmp_result!=0, props_to_record) + props = parse_properties(reason, cmp_result != 0, props_to_record) for k, v in props.items(): record_property(k, v) diff --git a/tests/conftest.py b/tests/conftest.py index 24446c9678..8b465645ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,6 +41,7 @@ import platform import textwrap from pathlib import Path from subprocess import TimeoutExpired, run +import tempfile from typing import Optional, Union from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN @@ -311,7 +312,7 @@ def dut_encoder_path(request) -> str: class EncoderFrontend: def __init__(self, path, enc_type, timeout=None) -> None: - self._path = path + self._path = str(Path(path).absolute()) self._type = enc_type self.returncode = None self.stdout = None @@ -320,7 +321,7 @@ class EncoderFrontend: def run( self, - bitrate: int, + bitrate: Union[int, Path], input_sampling_rate: int, input_path: Path, output_bitstream_path: Path, @@ -365,13 +366,18 @@ class EncoderFrontend: log_dbg_msg(f"{self._type} encoder command:\n{cmd_str}") try: - result = run( - command, - capture_output=True, - check=False, - timeout=self.timeout, - cwd=run_dir, - ) + with tempfile.TemporaryDirectory() as tmp_dir: + if run_dir is None: + cwd = tmp_dir + else: + cwd = run_dir + result = run( + command, + capture_output=True, + check=False, + timeout=self.timeout, + cwd=cwd, + ) except TimeoutExpired: pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.") @@ -492,7 +498,7 @@ def dut_decoder_path(request) -> str: class DecoderFrontend: def __init__(self, path, dec_type, timeout=None, fr=20) -> None: - self._path = path + self._path = str(Path(path).absolute()) self._type = dec_type self.returncode = None self.stdout = None @@ -546,7 +552,12 @@ class DecoderFrontend: try: if not os.path.exists(str(input_bitstream_path) + eid_output_suffix): - result = run(eid_command, check=True, cwd=run_dir) + with tempfile.TemporaryDirectory() as tmp_dir: + if run_dir is None: + cwd = tmp_dir + else: + cwd = run_dir + result = run(eid_command, check=True, cwd=cwd) except Exception as e: pytest.fail(f"eid-xor operation failed! - {e}") @@ -566,14 +577,28 @@ class DecoderFrontend: raise ValueError(f'Wrong system "{system}"!') if not os.path.isfile(netsim_path): - raise FileNotFoundError(f"network simulator binary {netsim_path} not found!\n") + raise FileNotFoundError( + f"network simulator binary {netsim_path} not found!\n" + ) netsim_bitstream_path = input_bitstream_path.with_suffix(".netsimout") tracefile_path = input_bitstream_path.with_suffix(".netsimtrace") # TODO: need to check if the "1" works with every profile - netsim_command = [netsim_path, netsim_profile, input_bitstream_path, netsim_bitstream_path, tracefile_path, "1"] + netsim_command = [ + netsim_path, + netsim_profile, + input_bitstream_path, + netsim_bitstream_path, + tracefile_path, + "1", + ] print(netsim_command) try: - run(netsim_command, check=True, cwd=run_dir) + with tempfile.TemporaryDirectory() as tmp_dir: + if run_dir is None: + cwd = tmp_dir + else: + cwd = run_dir + run(netsim_command, check=True, cwd=cwd) except Exception as e: pytest.fail(f"netsim operation failed! - {e}") @@ -601,13 +626,18 @@ class DecoderFrontend: log_dbg_msg(f"{self._type} decoder command:\n{cmd_str}") try: - result = run( - command, - capture_output=True, - check=False, - timeout=self.timeout, - cwd=run_dir, - ) + with tempfile.TemporaryDirectory() as tmp_dir: + if run_dir is None: + cwd = tmp_dir + else: + cwd = run_dir + result = run( + command, + capture_output=True, + check=False, + timeout=self.timeout, + cwd=cwd, + ) except TimeoutExpired: pytest.fail(f"{self._type} decoder run timed out after {self.timeout}s.") -- GitLab From 4bb8605f9bb5ffa8f4894cbf99dfc930edda8abd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 8 Jul 2024 17:04:23 +0200 Subject: [PATCH 02/28] port encoder logging from basop repo --- .../test_param_file.py | 140 +++++++++++++++++- tests/conftest.py | 117 +++++++++++++-- tests/constants.py | 36 ++++- 3 files changed, 279 insertions(+), 14 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index dc10a17b57..f58c8828c7 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -35,6 +35,7 @@ Execute tests specified via a parameter file. import errno import filecmp import os +import json import platform from pathlib import Path from subprocess import run @@ -44,6 +45,7 @@ import numpy as np from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties from tests.testconfig import PARAM_FILE +from tests.constants import MIN_ENC_AUX_FILE_LENGTH_DIFF, MIN_ENC_AUX_FILE_DIFF_THR VALID_DEC_OUTPUT_CONF = [ @@ -129,6 +131,16 @@ def convert_test_string_to_tag(test_string): return tag_str +def num(s): + """ + Convert string either to integer or float + """ + try: + return int(s) + except ValueError: + return float(s) + + @pytest.mark.create_ref @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) # hack to have stv/ltv/evs in the test name @@ -136,6 +148,7 @@ def convert_test_string_to_tag(test_string): def test_param_file_tests( record_property, props_to_record, + encoder_only, decoder_only, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -199,8 +212,118 @@ def test_param_file_tests( bitstream_file, enc_split, update_ref, + encoder_only, ) + # compare binary files extracted from the encoder + if encoder_only: + print("Comparing encoder auxiliary files") + print("=================================\n") + + stats_file = bitstream_file.replace(".192", ".stats") + ref_stats_file = f"{reference_path}/param_file/enc/{stats_file}" + dut_stats_file = f"{dut_base_path}/param_file/enc/{stats_file}" + + # open and read the .stats file + with open(ref_stats_file, "r") as f_aux: + ref_stats = json.load(f_aux) + # create dictionary to map "name" to the corresponding dictionaries + ref_stats_names = {d["name"]: d for d in ref_stats} + + with open(dut_stats_file, "r") as f_aux: + dut_stats = json.load(f_aux) + # create dictionary to map "name" to the corresponding dictionaries + dut_stats_names = {d["name"]: d for d in dut_stats} + + # loop over all common aux files + enc_test_result = 0 + enc_test_result_msg = "" + max_enc_diff = 0 + for name in ref_stats_names: + if name in dut_stats_names: + # retrieve the dictionaries + ref_stats_dict = ref_stats_names[name] + dut_stats_dict = dut_stats_names[name] + + msg = f"File {name}" + + # compare the file lengths + result_len_check = 0 + file_length = max( + ref_stats_dict["length"], dut_stats_dict["length"] + ) + if ref_stats_dict["length"] != dut_stats_dict["length"]: + msg += f" has different length between Ref {ref_stats_dict['length']} and DuT {dut_stats_dict['length']}" + + # check if threshold has been exceeded + if ( + abs(ref_stats_dict["length"] - dut_stats_dict["length"]) + / file_length + > MIN_ENC_AUX_FILE_LENGTH_DIFF + ): + result_len_check = 1 + + msg += ", " + + # remove the "name" and "length" keys for further processing + del ref_stats_dict["name"] + del dut_stats_dict["name"] + del ref_stats_dict["length"] + del dut_stats_dict["length"] + + # convert keys and values from string to float + ref_hist = {num(i): num(j) for i, j in ref_stats_dict.items()} + cut_hist = {num(i): num(j) for i, j in dut_stats_dict.items()} + delta_ref = set(cut_hist) - set(ref_hist) + delta_cut = set(ref_hist) - set(cut_hist) + + # append missing keys + for item in delta_cut: + cut_hist[item] = 0 + + for item in delta_ref: + ref_hist[item] = 0 + + ref_hist = dict(sorted(ref_hist.items())) + cut_hist = dict(sorted(cut_hist.items())) + + # caculate difference of statistics + diff_hist = {k: cut_hist[k] - ref_hist[k] for k in ref_hist.keys()} + + # calculate the total number of differences + total_num_diff = sum(np.abs(list(diff_hist.values()))) + total_num_diff_ratio = total_num_diff / ( + sum(ref_hist.values()) + sum(cut_hist.values()) + ) + + msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)" + if total_num_diff_ratio > MIN_ENC_AUX_FILE_DIFF_THR: + result_diff_check = 1 + msg += "! " + else: + result_diff_check = 0 + msg += ". " + + # check if the maximum difference has been exceeded + if total_num_diff_ratio > max_enc_diff: + max_enc_diff = total_num_diff_ratio + + # update test result + if result_len_check or result_diff_check: + enc_test_result = 1 + enc_test_result_msg += msg + + print(msg) + + print("") + + if enc_test_result: + record_property("MAXIMUM ENC DIFF", max_enc_diff) + pytest.fail(enc_test_result_msg) + + if encoder_only: + return + # check for networkSimulator_g192 command line if sim_opts != "": sim_split = sim_opts.split() @@ -423,6 +546,7 @@ def encode( bitstream_file, enc_opts_list, update_ref, + encoder_only, ): """ Call REF and/or DUT encoder. @@ -434,8 +558,17 @@ def encode( ref_out_file = f"{ref_out_dir}/{bitstream_file}" dut_out_file = f"{dut_out_dir}/{bitstream_file}" - if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file): + if encoder_only: + stats_file = bitstream_file.replace(".192", ".stats") + ref_stats_file = f"{ref_out_dir}/{stats_file}" + dut_stats_file = f"{dut_out_dir}/{stats_file}" + else: + ref_stats_file = None + dut_stats_file = None + + if (update_ref in [1, 2] and not os.path.exists(ref_out_file)) or encoder_only: check_and_makedir(ref_out_dir) + # call REF encoder ref_encoder_frontend.run( bitrate, @@ -443,10 +576,12 @@ def encode( testv_file, ref_out_file, add_option_list=enc_opts_list, + stats_file=ref_stats_file, ) - if update_ref in [0, 2]: + if update_ref in [0, 2] or encoder_only: check_and_makedir(dut_out_dir) + # call DUT encoder dut_encoder_frontend.run( bitrate, @@ -454,6 +589,7 @@ def encode( testv_file, dut_out_file, add_option_list=enc_opts_list, + stats_file=dut_stats_file, ) diff --git a/tests/conftest.py b/tests/conftest.py index 8b465645ff..0b854ba717 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,6 +35,7 @@ Pytest customization (configuration and fixtures) for the IVAS codec test suite. import logging import os import re +import json from tests import testconfig import pytest import platform @@ -43,7 +44,8 @@ from pathlib import Path from subprocess import TimeoutExpired, run import tempfile from typing import Optional, Union -from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN +import numpy as np +from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN, ENC_AUX_FILES logger = logging.getLogger(__name__) USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature @@ -201,6 +203,13 @@ def pytest_addoption(parser): default=20, ) + parser.addoption( + "--encoder_only", + help="Only run encoder parts of tests in 'codec_be_on_mr_nonselection'.", + action="store_true", + default=False, + ) + parser.addoption( "--decoder_only", help="Only run decoder parts of tests in 'codec_be_on_mr_nonselection'. Use ref encoder output bitstreams.", @@ -312,13 +321,83 @@ def dut_encoder_path(request) -> str: class EncoderFrontend: def __init__(self, path, enc_type, timeout=None) -> None: - self._path = str(Path(path).absolute()) + self._path = Path(path).absolute() self._type = enc_type self.returncode = None self.stdout = None self.stderr = None self.timeout = timeout + def extract_enc_stats( + self, + dbg_tweak_folder, + stats_file, + sampling_rate, + ): + """ + Extract statistics from auxiliary encoder files generated by running the encoder with DEBUG_MODE_INFO. + Write the statistics to a text file + """ + + hist_dicts = [] + for f in ENC_AUX_FILES: + filename = f[0] + dtype = f[1] + fs = int(sampling_rate) * 1000 + if isinstance(f[2], str): + nsamples_per_frame = np.int16(eval(f[2])) + else: + nsamples_per_frame = np.int16(f[2]) + + # aux_files = glob.glob(os.path.join(dbg_tweak_folder, filename + '\.*')) + aux_files = [ + f + for f in os.listdir(dbg_tweak_folder) + if re.search(rf"^{filename}(\..*)?$", f) + ] + # aux_files = [os.path.basename(f) for f in aux_files] + + for aux_file in aux_files: + # extract statistics from the aux file based on histogram of values + print( + f"Extracting statistics from {os.path.basename(aux_file)} ... ", + end="", + ) + + # read the aux file + with open(os.path.join(dbg_tweak_folder, aux_file), "r") as f_aux: + data = np.fromfile(f_aux, dtype=dtype) + + # get file length + data_len = data.shape[0] + + # remove the duplicates of each value per frame + if nsamples_per_frame > 1: + data = data[::nsamples_per_frame] + + # calculate histogram from data + unique_values = np.sort(np.unique(data)) + hist, _ = np.histogram( + data, bins=np.append(unique_values, unique_values[-1] + 10) + ) + + # convert to dict and sort by absolute value of difference + hist_dict = {"name": os.path.basename(aux_file), "length": data_len} + dict_values = { + str(unique_values[i]): str(hist[i]) + for i in range(len(unique_values)) + } + hist_dict.update(dict_values) + hist_dicts.append(hist_dict) + + print("DONE") + + print("") + + with open(stats_file, "w") as f_stats: + # append the statistics to the output file in text format + f_stats.write(json.dumps(hist_dicts, indent=2)) + def run( self, bitrate: Union[int, Path], @@ -332,8 +411,9 @@ class EncoderFrontend: quiet_mode: Optional[bool] = True, add_option_list: Optional[list] = None, run_dir: Optional[Path] = None, + stats_file: Optional[Path] = None, ) -> None: - command = [self._path] + command = [str(self._path)] # add optional parameters if sba_order is not None: @@ -368,9 +448,10 @@ class EncoderFrontend: try: with tempfile.TemporaryDirectory() as tmp_dir: if run_dir is None: - cwd = tmp_dir + cwd = Path(tmp_dir).absolute() else: - cwd = run_dir + cwd = Path(run_dir).absolute() + result = run( command, capture_output=True, @@ -378,6 +459,12 @@ class EncoderFrontend: timeout=self.timeout, cwd=cwd, ) + + if stats_file is not None: + self.extract_enc_stats( + cwd.joinpath("res"), stats_file, input_sampling_rate + ) + except TimeoutExpired: pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.") @@ -554,9 +641,9 @@ class DecoderFrontend: if not os.path.exists(str(input_bitstream_path) + eid_output_suffix): with tempfile.TemporaryDirectory() as tmp_dir: if run_dir is None: - cwd = tmp_dir + cwd = Path(tmp_dir).absolute() else: - cwd = run_dir + cwd = Path(run_dir).absolute() result = run(eid_command, check=True, cwd=cwd) except Exception as e: pytest.fail(f"eid-xor operation failed! - {e}") @@ -595,9 +682,9 @@ class DecoderFrontend: try: with tempfile.TemporaryDirectory() as tmp_dir: if run_dir is None: - cwd = tmp_dir + cwd = Path(tmp_dir).absolute() else: - cwd = run_dir + cwd = Path(run_dir).absolute() run(netsim_command, check=True, cwd=cwd) except Exception as e: pytest.fail(f"netsim operation failed! - {e}") @@ -628,9 +715,9 @@ class DecoderFrontend: try: with tempfile.TemporaryDirectory() as tmp_dir: if run_dir is None: - cwd = tmp_dir + cwd = Path(tmp_dir).absolute() else: - cwd = run_dir + cwd = Path(run_dir).absolute() result = run( command, capture_output=True, @@ -816,6 +903,14 @@ def decoder_only(request) -> bool: return request.config.getoption("--decoder_only") +@pytest.fixture(scope="session", autouse=True) +def encoder_only(request) -> bool: + """ + Return value of cmdl param --encoder_only + """ + return request.config.getoption("--encoder_only") + + def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial") config.addinivalue_line( diff --git a/tests/constants.py b/tests/constants.py index 0f985d0b56..41d3aadaf7 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,10 +1,44 @@ import pathlib +import numpy as np HERE = pathlib.Path(__file__).parent.absolute() SCRIPTS_DIR = HERE.parent.joinpath("scripts") TESTV_DIR = SCRIPTS_DIR.joinpath("testv") -# regex patterns for parsing the output from cmp_pcm -> mainly for BASOP ci +# regex patterns for parsing the output from comparisons -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)" +MAX_ENC_DIFF_PATTERN = r"total number of differences is \d+ \((\d+\.\d+)%\)" + +MIN_ENC_AUX_FILE_DIFF_THR = ( + 0.1 # minimum ratio of total number of differences in encoder aux file +) +MIN_ENC_AUX_FILE_LENGTH_DIFF = 0.1 # minimum difference of encoder aux file length + +# list of encoder filename patterns with their data type and number of samples per frame +# note: instead of specifying the number of samples per frame, you can use a formula incl. 'fs', e.g. 'fs/50' +ENC_AUX_FILES = [ + ["bits_nominal", np.int16, "fs/50"], + ["bwidth", np.int16, "fs/50"], + ["clas", np.int16, "fs/50"], + ["cng_type", np.int16, "fs/50"], + ["coder_type", np.int16, "fs/50"], + ["core", np.int16, "fs/50"], + ["core_brate", np.float32, "fs/50"], + ["count_SWB", np.int16, "fs/50"], + ["count_WB", np.int16, "fs/50"], + ["element_brate", np.float32, "fs/50"], + ["element_mode", np.int16, "fs/50"], + ["extl", np.int16, "fs/50"], + ["extl_brate", np.float32, "fs/50"], + ["ivas_total_brate", np.float32, "fs/50"], + ["L_frame", np.int16, "fs/50"], + ["localVAD", np.int16, "fs/50"], + ["sp_aud_decision0", np.int16, "fs/50"], + ["sp_aud_decision1", np.int16, "fs/50"], + ["sp_aud_decision2", np.int16, "fs/50"], + ["tdm_LRTD_flag", np.int16, "fs/50"], + ["total_brate", np.float32, "fs/50"], + ["vad_flag", np.int16, "fs/50"], +] -- GitLab From 2e079b587667d794a3a95d9fc4f5801f70b5cf98 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Jul 2024 10:04:51 +0200 Subject: [PATCH 03/28] add --enc_stats cmld arg --- .../codec_be_on_mr_nonselection/test_param_file.py | 13 +++++++------ tests/conftest.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index f58c8828c7..9fa609e148 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -166,6 +166,7 @@ def test_param_file_tests( get_mld_lim, abs_tol, get_ssnr, + get_enc_stats, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -212,11 +213,11 @@ def test_param_file_tests( bitstream_file, enc_split, update_ref, - encoder_only, + get_enc_stats, ) # compare binary files extracted from the encoder - if encoder_only: + if get_enc_stats: print("Comparing encoder auxiliary files") print("=================================\n") @@ -546,7 +547,7 @@ def encode( bitstream_file, enc_opts_list, update_ref, - encoder_only, + get_enc_stats=False, ): """ Call REF and/or DUT encoder. @@ -558,7 +559,7 @@ def encode( ref_out_file = f"{ref_out_dir}/{bitstream_file}" dut_out_file = f"{dut_out_dir}/{bitstream_file}" - if encoder_only: + if get_enc_stats: stats_file = bitstream_file.replace(".192", ".stats") ref_stats_file = f"{ref_out_dir}/{stats_file}" dut_stats_file = f"{dut_out_dir}/{stats_file}" @@ -566,7 +567,7 @@ def encode( ref_stats_file = None dut_stats_file = None - if (update_ref in [1, 2] and not os.path.exists(ref_out_file)) or encoder_only: + if update_ref in [1, 2] and not os.path.exists(ref_out_file): check_and_makedir(ref_out_dir) # call REF encoder @@ -579,7 +580,7 @@ def encode( stats_file=ref_stats_file, ) - if update_ref in [0, 2] or encoder_only: + if update_ref in [0, 2]: check_and_makedir(dut_out_dir) # call DUT encoder diff --git a/tests/conftest.py b/tests/conftest.py index 0b854ba717..67ef298258 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -183,6 +183,12 @@ def pytest_addoption(parser): help="Compute Segmental SNR (SSNR) between ref and dut output instead of just comparing for bitexactness", ) + parser.addoption( + "--enc_stats", + action="store_true", + help="Activate logging and comparison of statistics from the encoder.", + ) + parser.addoption( "--create_ref", action="store_true", @@ -269,6 +275,14 @@ def get_ssnr(request): return request.config.option.ssnr +@pytest.fixture(scope="session", autouse=True) +def get_enc_stats(request): + """ + Return indication to compare statistics of values logged from encoder. + """ + return request.config.option.enc_stats + + @pytest.fixture(scope="session") def abs_tol(request) -> int: """ -- GitLab From 0242d39970314e111e15efafd3e9a3a2322dbcf7 Mon Sep 17 00:00:00 2001 From: malenov Date: Fri, 12 Jul 2024 15:23:21 +0200 Subject: [PATCH 04/28] add wrapper function for comparing .stats files --- tests/cmp_stats_files.py | 145 ++++++++++++++++++ .../test_param_file.py | 124 +++------------ tests/conftest.py | 14 +- tests/constants.py | 8 +- 4 files changed, 176 insertions(+), 115 deletions(-) create mode 100644 tests/cmp_stats_files.py diff --git a/tests/cmp_stats_files.py b/tests/cmp_stats_files.py new file mode 100644 index 0000000000..4d4dd684b8 --- /dev/null +++ b/tests/cmp_stats_files.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys +import json + +import pdb + +THIS_PATH = os.path.join(os.getcwd(), __file__) +sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) + +import numpy as np +import pyaudio3dtools +import pyivastest +from collections import OrderedDict + + + +def str2num(s): + """ + Convert string either to integer or float + """ + try: + return int(s) + except ValueError: + return float(s) + +def cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = 0.1, min_enc_stats_diff = 0.1) -> (int, str): + """ + Compare two .stats files containing encoder statistics (extracted from binary files) + """ + print(f"Comparing {os.path.basename(ref_stats_file)} between Ref and Dut ... \n") + + # open and read the .stats files + with open(ref_stats_file, "r") as f_aux: + ref_stats = json.load(f_aux) + # create dictionary to map "name" to the corresponding dictionaries + ref_stats_names = {d["name"]: d for d in ref_stats} + + with open(dut_stats_file, "r") as f_aux: + dut_stats = json.load(f_aux) + # create dictionary to map "name" to the corresponding dictionaries + dut_stats_names = {d["name"]: d for d in dut_stats} + + # loop over all common aux files + enc_test_result = 0 + enc_test_result_msg = "" + max_total_num_diff = 0 + max_total_num_diff_ratio = 0 + for name in ref_stats_names: + if name in dut_stats_names: + # retrieve the dictionaries + ref_stats_dict = ref_stats_names[name] + dut_stats_dict = dut_stats_names[name] + + msg = f"File {name}" + + # compare the file lengths + result_len_check = 0 + file_length = max(ref_stats_dict["length"], dut_stats_dict["length"]) + if ref_stats_dict["length"] != dut_stats_dict["length"]: + msg += f" has different length between Ref {ref_stats_dict['length']} and DuT {dut_stats_dict['length']}" + + # check if threshold has been exceeded + if abs(ref_stats_dict["length"] - dut_stats_dict["length"]) / file_length > min_enc_file_length_diff: + result_len_check = 1 + + msg += f", " + + # remove the "name" and "length" keys for further processing + del ref_stats_dict["name"] + del dut_stats_dict["name"] + del ref_stats_dict["length"] + del dut_stats_dict["length"] + + # convert keys and values from string to float + ref_hist = {str2num(i) : str2num(j) for i,j in ref_stats_dict.items()} + cut_hist = {str2num(i) : str2num(j) for i,j in dut_stats_dict.items()} + delta_ref = set(cut_hist) - set(ref_hist) + delta_cut = set(ref_hist) - set(cut_hist) + + # append missing keys + for item in delta_cut: + cut_hist[item] = 0 + + for item in delta_ref: + ref_hist[item] = 0 + + ref_hist = dict(sorted(ref_hist.items())) + cut_hist = dict(sorted(cut_hist.items())) + + # caculate difference of statistics + diff_hist = { k : cut_hist[k] - ref_hist[k] for k in ref_hist.keys()} + + # calculate the total number of differences + total_num_diff = sum(np.abs(list(diff_hist.values()))) + total_num_diff_ratio = total_num_diff / (sum(ref_hist.values()) + sum(cut_hist.values())) + + msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)" + if total_num_diff_ratio > min_enc_stats_diff: + result_diff_check = 1 + msg += "! " + else: + result_diff_check = 0 + msg += ". " + + # check if the maximum difference has been exceeded + if total_num_diff_ratio > max_total_num_diff_ratio: + max_total_num_diff = total_num_diff + max_total_num_diff_ratio = total_num_diff_ratio + + # update test result + if result_len_check or result_diff_check: + enc_test_result = 1 + enc_test_result_msg += msg + + print(msg) + + if enc_test_result and max_total_num_diff > 0: + msg = f"MAXIMUM ENC DIFF: {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%) " + enc_test_result_msg += msg + print(msg) + + return enc_test_result, enc_test_result_msg + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("ref_file", type=str) + parser.add_argument("cut_file", type=str) + parser.add_argument("--data_type", type=str, default='int16', dest="dtype") + parser.add_argument("--nsamples_per_frame", type=str, default=960, dest="nsamples_per_frame") + parser.add_argument("--len_check", type=int, default=0, dest="len_check") + parser.add_argument("--min_diff_thr", type=float, default=0.0, dest="min_diff_thr") + args = parser.parse_args() + + # convert dtype from str + if isinstance(args.dtype, str): + args.dtype = np.dtype(getattr(np, args.dtype)) + + result, msg = cmp_bin_files(**vars(args)) + print(msg) + sys.exit(result) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 9fa609e148..904ad82300 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -41,12 +41,16 @@ from pathlib import Path from subprocess import run import pytest import numpy as np +import re + from tests.cmp_pcm import cmp_pcm +from tests.cmp_stats_files import cmp_stats_files from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties from tests.testconfig import PARAM_FILE -from tests.constants import MIN_ENC_AUX_FILE_LENGTH_DIFF, MIN_ENC_AUX_FILE_DIFF_THR +from tests.constants import ENC_AUX_FILES, MAX_ENC_DIFF_PATTERN, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF +import pdb VALID_DEC_OUTPUT_CONF = [ "MONO", @@ -131,15 +135,6 @@ def convert_test_string_to_tag(test_string): return tag_str -def num(s): - """ - Convert string either to integer or float - """ - try: - return int(s) - except ValueError: - return float(s) - @pytest.mark.create_ref @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) @@ -171,6 +166,7 @@ def test_param_file_tests( enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] tag_str = convert_test_string_to_tag(test_tag) + # evaluate encoder options enc_split = enc_opts.split() @@ -200,7 +196,7 @@ def test_param_file_tests( # in the parameter file, only "bit" is used as bitstream file name # -> construct bitstream filename bitstream_file = f"{testv_base}_{tag_str}.192" - + if not decoder_only: encode( dut_encoder_frontend, @@ -215,7 +211,7 @@ def test_param_file_tests( update_ref, get_enc_stats, ) - + # compare binary files extracted from the encoder if get_enc_stats: print("Comparing encoder auxiliary files") @@ -225,100 +221,16 @@ def test_param_file_tests( ref_stats_file = f"{reference_path}/param_file/enc/{stats_file}" dut_stats_file = f"{dut_base_path}/param_file/enc/{stats_file}" - # open and read the .stats file - with open(ref_stats_file, "r") as f_aux: - ref_stats = json.load(f_aux) - # create dictionary to map "name" to the corresponding dictionaries - ref_stats_names = {d["name"]: d for d in ref_stats} - - with open(dut_stats_file, "r") as f_aux: - dut_stats = json.load(f_aux) - # create dictionary to map "name" to the corresponding dictionaries - dut_stats_names = {d["name"]: d for d in dut_stats} - - # loop over all common aux files - enc_test_result = 0 - enc_test_result_msg = "" - max_enc_diff = 0 - for name in ref_stats_names: - if name in dut_stats_names: - # retrieve the dictionaries - ref_stats_dict = ref_stats_names[name] - dut_stats_dict = dut_stats_names[name] - - msg = f"File {name}" - - # compare the file lengths - result_len_check = 0 - file_length = max( - ref_stats_dict["length"], dut_stats_dict["length"] - ) - if ref_stats_dict["length"] != dut_stats_dict["length"]: - msg += f" has different length between Ref {ref_stats_dict['length']} and DuT {dut_stats_dict['length']}" - - # check if threshold has been exceeded - if ( - abs(ref_stats_dict["length"] - dut_stats_dict["length"]) - / file_length - > MIN_ENC_AUX_FILE_LENGTH_DIFF - ): - result_len_check = 1 - - msg += ", " - - # remove the "name" and "length" keys for further processing - del ref_stats_dict["name"] - del dut_stats_dict["name"] - del ref_stats_dict["length"] - del dut_stats_dict["length"] - - # convert keys and values from string to float - ref_hist = {num(i): num(j) for i, j in ref_stats_dict.items()} - cut_hist = {num(i): num(j) for i, j in dut_stats_dict.items()} - delta_ref = set(cut_hist) - set(ref_hist) - delta_cut = set(ref_hist) - set(cut_hist) - - # append missing keys - for item in delta_cut: - cut_hist[item] = 0 - - for item in delta_ref: - ref_hist[item] = 0 - - ref_hist = dict(sorted(ref_hist.items())) - cut_hist = dict(sorted(cut_hist.items())) - - # caculate difference of statistics - diff_hist = {k: cut_hist[k] - ref_hist[k] for k in ref_hist.keys()} - - # calculate the total number of differences - total_num_diff = sum(np.abs(list(diff_hist.values()))) - total_num_diff_ratio = total_num_diff / ( - sum(ref_hist.values()) + sum(cut_hist.values()) - ) - - msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)" - if total_num_diff_ratio > MIN_ENC_AUX_FILE_DIFF_THR: - result_diff_check = 1 - msg += "! " - else: - result_diff_check = 0 - msg += ". " - - # check if the maximum difference has been exceeded - if total_num_diff_ratio > max_enc_diff: - max_enc_diff = total_num_diff_ratio - - # update test result - if result_len_check or result_diff_check: - enc_test_result = 1 - enc_test_result_msg += msg - - print(msg) - - print("") - + # compare ref and dut .stats files + enc_test_result, enc_test_result_msg = cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = MIN_ENC_FILE_LENGTH_DIFF, min_enc_stats_diff = MIN_ENC_STATS_DIFF) + if enc_test_result: + # find the maximum number of differences in the test result message + search_result = re.search(MAX_ENC_DIFF_PATTERN, enc_test_result_msg) + if search_result: + max_enc_diff, max_enc_diff_ratio = search_result.groups(0) + if max_enc_diff: + max_enc_diff = float(max_enc_diff) record_property("MAXIMUM ENC DIFF", max_enc_diff) pytest.fail(enc_test_result_msg) @@ -566,7 +478,7 @@ def encode( else: ref_stats_file = None dut_stats_file = None - + if update_ref in [1, 2] and not os.path.exists(ref_out_file): check_and_makedir(ref_out_dir) diff --git a/tests/conftest.py b/tests/conftest.py index 67ef298258..67e87a2ecc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -230,6 +230,13 @@ def pytest_addoption(parser): default=0, ) + parser.addoption( + "--get_enc_stats", + help="Generate .stats file containing statistics about encoder parameters", + action="store_true", + default=False, + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -276,12 +283,11 @@ def get_ssnr(request): @pytest.fixture(scope="session", autouse=True) -def get_enc_stats(request): +def get_enc_stats(request) -> bool: """ - Return indication to compare statistics of values logged from encoder. + Return value of cmdl param --get_enc_stats """ - return request.config.option.enc_stats - + return request.config.getoption("--get_enc_stats") @pytest.fixture(scope="session") def abs_tol(request) -> int: diff --git a/tests/constants.py b/tests/constants.py index 41d3aadaf7..5233a207d3 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -9,12 +9,10 @@ TESTV_DIR = SCRIPTS_DIR.joinpath("testv") MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)" -MAX_ENC_DIFF_PATTERN = r"total number of differences is \d+ \((\d+\.\d+)%\)" +MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)" -MIN_ENC_AUX_FILE_DIFF_THR = ( - 0.1 # minimum ratio of total number of differences in encoder aux file -) -MIN_ENC_AUX_FILE_LENGTH_DIFF = 0.1 # minimum difference of encoder aux file length +MIN_ENC_FILE_LENGTH_DIFF = 0.1 # minimum difference between ref and dut encoder file lengths +MIN_ENC_STATS_DIFF = 0.1 # minimum difference between the statistics of ref and dut encoder files # list of encoder filename patterns with their data type and number of samples per frame # note: instead of specifying the number of samples per frame, you can use a formula incl. 'fs', e.g. 'fs/50' -- GitLab From cef43e80a10195542a82b35ec27d057f0f16ad52 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 12 Jul 2024 15:41:02 +0200 Subject: [PATCH 05/28] cleanup and remove duplicate cmld arg --- .../test_param_file.py | 24 ++++++++++++------- tests/conftest.py | 10 ++------ tests/constants.py | 8 +++++-- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 904ad82300..de4788b240 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -35,7 +35,6 @@ Execute tests specified via a parameter file. import errno import filecmp import os -import json import platform from pathlib import Path from subprocess import run @@ -48,7 +47,11 @@ from tests.cmp_pcm import cmp_pcm from tests.cmp_stats_files import cmp_stats_files from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties from tests.testconfig import PARAM_FILE -from tests.constants import ENC_AUX_FILES, MAX_ENC_DIFF_PATTERN, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF +from tests.constants import ( + MAX_ENC_DIFF_PATTERN, + MIN_ENC_FILE_LENGTH_DIFF, + MIN_ENC_STATS_DIFF, +) import pdb @@ -135,7 +138,6 @@ def convert_test_string_to_tag(test_string): return tag_str - @pytest.mark.create_ref @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) # hack to have stv/ltv/evs in the test name @@ -166,7 +168,6 @@ def test_param_file_tests( enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] tag_str = convert_test_string_to_tag(test_tag) - # evaluate encoder options enc_split = enc_opts.split() @@ -196,7 +197,7 @@ def test_param_file_tests( # in the parameter file, only "bit" is used as bitstream file name # -> construct bitstream filename bitstream_file = f"{testv_base}_{tag_str}.192" - + if not decoder_only: encode( dut_encoder_frontend, @@ -211,7 +212,7 @@ def test_param_file_tests( update_ref, get_enc_stats, ) - + # compare binary files extracted from the encoder if get_enc_stats: print("Comparing encoder auxiliary files") @@ -222,8 +223,13 @@ def test_param_file_tests( dut_stats_file = f"{dut_base_path}/param_file/enc/{stats_file}" # compare ref and dut .stats files - enc_test_result, enc_test_result_msg = cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = MIN_ENC_FILE_LENGTH_DIFF, min_enc_stats_diff = MIN_ENC_STATS_DIFF) - + enc_test_result, enc_test_result_msg = cmp_stats_files( + ref_stats_file, + dut_stats_file, + min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF, + min_enc_stats_diff=MIN_ENC_STATS_DIFF, + ) + if enc_test_result: # find the maximum number of differences in the test result message search_result = re.search(MAX_ENC_DIFF_PATTERN, enc_test_result_msg) @@ -478,7 +484,7 @@ def encode( else: ref_stats_file = None dut_stats_file = None - + if update_ref in [1, 2] and not os.path.exists(ref_out_file): check_and_makedir(ref_out_dir) diff --git a/tests/conftest.py b/tests/conftest.py index 67e87a2ecc..3d62c91252 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -230,13 +230,6 @@ def pytest_addoption(parser): default=0, ) - parser.addoption( - "--get_enc_stats", - help="Generate .stats file containing statistics about encoder parameters", - action="store_true", - default=False, - ) - @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -287,7 +280,8 @@ def get_enc_stats(request) -> bool: """ Return value of cmdl param --get_enc_stats """ - return request.config.getoption("--get_enc_stats") + return request.config.getoption("--enc_stats") + @pytest.fixture(scope="session") def abs_tol(request) -> int: diff --git a/tests/constants.py b/tests/constants.py index 5233a207d3..15f4d3dd36 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -11,8 +11,12 @@ MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)" MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)" -MIN_ENC_FILE_LENGTH_DIFF = 0.1 # minimum difference between ref and dut encoder file lengths -MIN_ENC_STATS_DIFF = 0.1 # minimum difference between the statistics of ref and dut encoder files +MIN_ENC_FILE_LENGTH_DIFF = ( + 0.1 # minimum difference between ref and dut encoder file lengths +) +MIN_ENC_STATS_DIFF = ( + 0.1 # minimum difference between the statistics of ref and dut encoder files +) # list of encoder filename patterns with their data type and number of samples per frame # note: instead of specifying the number of samples per frame, you can use a formula incl. 'fs', e.g. 'fs/50' -- GitLab From 12ed51e56a31180e2e78edb03829d0e3803a78c3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 12 Jul 2024 15:41:51 +0200 Subject: [PATCH 06/28] run formatter --- tests/cmp_stats_files.py | 83 ++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/tests/cmp_stats_files.py b/tests/cmp_stats_files.py index 4d4dd684b8..2f316be890 100644 --- a/tests/cmp_stats_files.py +++ b/tests/cmp_stats_files.py @@ -16,7 +16,6 @@ import pyivastest from collections import OrderedDict - def str2num(s): """ Convert string either to integer or float @@ -25,8 +24,11 @@ def str2num(s): return int(s) except ValueError: return float(s) - -def cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = 0.1, min_enc_stats_diff = 0.1) -> (int, str): + + +def cmp_stats_files( + ref_stats_file, dut_stats_file, min_enc_file_length_diff=0.1, min_enc_stats_diff=0.1 +) -> (int, str): """ Compare two .stats files containing encoder statistics (extracted from binary files) """ @@ -34,26 +36,26 @@ def cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = 0 # open and read the .stats files with open(ref_stats_file, "r") as f_aux: - ref_stats = json.load(f_aux) - # create dictionary to map "name" to the corresponding dictionaries - ref_stats_names = {d["name"]: d for d in ref_stats} + ref_stats = json.load(f_aux) + # create dictionary to map "name" to the corresponding dictionaries + ref_stats_names = {d["name"]: d for d in ref_stats} with open(dut_stats_file, "r") as f_aux: - dut_stats = json.load(f_aux) - # create dictionary to map "name" to the corresponding dictionaries - dut_stats_names = {d["name"]: d for d in dut_stats} - + dut_stats = json.load(f_aux) + # create dictionary to map "name" to the corresponding dictionaries + dut_stats_names = {d["name"]: d for d in dut_stats} + # loop over all common aux files enc_test_result = 0 enc_test_result_msg = "" max_total_num_diff = 0 max_total_num_diff_ratio = 0 - for name in ref_stats_names: + for name in ref_stats_names: if name in dut_stats_names: - # retrieve the dictionaries + # retrieve the dictionaries ref_stats_dict = ref_stats_names[name] dut_stats_dict = dut_stats_names[name] - + msg = f"File {name}" # compare the file lengths @@ -61,11 +63,15 @@ def cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = 0 file_length = max(ref_stats_dict["length"], dut_stats_dict["length"]) if ref_stats_dict["length"] != dut_stats_dict["length"]: msg += f" has different length between Ref {ref_stats_dict['length']} and DuT {dut_stats_dict['length']}" - + # check if threshold has been exceeded - if abs(ref_stats_dict["length"] - dut_stats_dict["length"]) / file_length > min_enc_file_length_diff: + if ( + abs(ref_stats_dict["length"] - dut_stats_dict["length"]) + / file_length + > min_enc_file_length_diff + ): result_len_check = 1 - + msg += f", " # remove the "name" and "length" keys for further processing @@ -73,30 +79,32 @@ def cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = 0 del dut_stats_dict["name"] del ref_stats_dict["length"] del dut_stats_dict["length"] - + # convert keys and values from string to float - ref_hist = {str2num(i) : str2num(j) for i,j in ref_stats_dict.items()} - cut_hist = {str2num(i) : str2num(j) for i,j in dut_stats_dict.items()} + ref_hist = {str2num(i): str2num(j) for i, j in ref_stats_dict.items()} + cut_hist = {str2num(i): str2num(j) for i, j in dut_stats_dict.items()} delta_ref = set(cut_hist) - set(ref_hist) delta_cut = set(ref_hist) - set(cut_hist) - + # append missing keys for item in delta_cut: cut_hist[item] = 0 - + for item in delta_ref: ref_hist[item] = 0 - + ref_hist = dict(sorted(ref_hist.items())) cut_hist = dict(sorted(cut_hist.items())) - + # caculate difference of statistics - diff_hist = { k : cut_hist[k] - ref_hist[k] for k in ref_hist.keys()} - + diff_hist = {k: cut_hist[k] - ref_hist[k] for k in ref_hist.keys()} + # calculate the total number of differences total_num_diff = sum(np.abs(list(diff_hist.values()))) - total_num_diff_ratio = total_num_diff / (sum(ref_hist.values()) + sum(cut_hist.values())) - + total_num_diff_ratio = total_num_diff / ( + sum(ref_hist.values()) + sum(cut_hist.values()) + ) + msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)" if total_num_diff_ratio > min_enc_stats_diff: result_diff_check = 1 @@ -104,38 +112,39 @@ def cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = 0 else: result_diff_check = 0 msg += ". " - + # check if the maximum difference has been exceeded if total_num_diff_ratio > max_total_num_diff_ratio: max_total_num_diff = total_num_diff max_total_num_diff_ratio = total_num_diff_ratio - + # update test result if result_len_check or result_diff_check: enc_test_result = 1 enc_test_result_msg += msg - + print(msg) - + if enc_test_result and max_total_num_diff > 0: msg = f"MAXIMUM ENC DIFF: {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%) " enc_test_result_msg += msg print(msg) - + return enc_test_result, enc_test_result_msg - - + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("ref_file", type=str) parser.add_argument("cut_file", type=str) - parser.add_argument("--data_type", type=str, default='int16', dest="dtype") - parser.add_argument("--nsamples_per_frame", type=str, default=960, dest="nsamples_per_frame") + parser.add_argument("--data_type", type=str, default="int16", dest="dtype") + parser.add_argument( + "--nsamples_per_frame", type=str, default=960, dest="nsamples_per_frame" + ) parser.add_argument("--len_check", type=int, default=0, dest="len_check") parser.add_argument("--min_diff_thr", type=float, default=0.0, dest="min_diff_thr") args = parser.parse_args() - + # convert dtype from str if isinstance(args.dtype, str): args.dtype = np.dtype(getattr(np, args.dtype)) -- GitLab From 62938fc9136c2f10e0a9a9bed4f75fb151ad4c31 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 12 Jul 2024 15:47:11 +0200 Subject: [PATCH 07/28] more cleanup --- tests/cmp_stats_files.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/cmp_stats_files.py b/tests/cmp_stats_files.py index 2f316be890..459f211b8f 100644 --- a/tests/cmp_stats_files.py +++ b/tests/cmp_stats_files.py @@ -4,16 +4,7 @@ import argparse import os import sys import json - -import pdb - -THIS_PATH = os.path.join(os.getcwd(), __file__) -sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) - import numpy as np -import pyaudio3dtools -import pyivastest -from collections import OrderedDict def str2num(s): @@ -72,7 +63,7 @@ def cmp_stats_files( ): result_len_check = 1 - msg += f", " + msg += ", " # remove the "name" and "length" keys for further processing del ref_stats_dict["name"] -- GitLab From 4514cc068604dc30f1c06139b621929c4062f4f4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 12 Jul 2024 16:13:54 +0200 Subject: [PATCH 08/28] record encoder diff in unified function, too --- .../test_param_file.py | 27 +++++---- tests/conftest.py | 57 ++++++++++++++----- tests/constants.py | 21 +++++-- 3 files changed, 73 insertions(+), 32 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index f3bf4a19c6..7d01c99ba9 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -53,7 +53,6 @@ from tests.constants import ( MIN_ENC_STATS_DIFF, ) -import pdb VALID_DEC_OUTPUT_CONF = [ "MONO", @@ -199,6 +198,9 @@ def test_param_file_tests( # -> construct bitstream filename bitstream_file = f"{testv_base}_{tag_str}.192" + cmp_result_msg = "" + enc_test_result = 0 + if not decoder_only: encode( dut_encoder_frontend, @@ -231,17 +233,15 @@ def test_param_file_tests( min_enc_stats_diff=MIN_ENC_STATS_DIFF, ) - if enc_test_result: - # find the maximum number of differences in the test result message - search_result = re.search(MAX_ENC_DIFF_PATTERN, enc_test_result_msg) - if search_result: - max_enc_diff, max_enc_diff_ratio = search_result.groups(0) - if max_enc_diff: - max_enc_diff = float(max_enc_diff) - record_property("MAXIMUM ENC DIFF", max_enc_diff) - pytest.fail(enc_test_result_msg) + cmp_result_msg += enc_test_result_msg if encoder_only: + props = parse_properties(cmp_result_msg, False, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if enc_test_result: + pytest.fail("Too high difference in encoder statistics found.") return # check for networkSimulator_g192 command line @@ -402,7 +402,9 @@ def test_param_file_tests( ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) - props = parse_properties(reason, output_differs, props_to_record) + cmp_result_msg += reason + + props = parse_properties(cmp_result_msg, output_differs, props_to_record) for k, v in props.items(): record_property(k, v) @@ -439,6 +441,9 @@ def test_param_file_tests( msg += "metadata only" pytest.fail(msg) + if enc_test_result: + pytest.fail("Too high difference in encoder statistics found.") + # remove DUT output files when test result is OK (to save disk space) if not keep_files: os.remove(f"{dut_base_path}/param_file/dec/{output_file}") diff --git a/tests/conftest.py b/tests/conftest.py index 12ef71d703..426e8d9570 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,7 +45,19 @@ from subprocess import TimeoutExpired, run import tempfile from typing import Optional, Union import numpy as np -from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN, ENC_AUX_FILES, ODG_PATTERN +from .constants import ( + MLD_PATTERN, + MAX_DIFF_PATTERN, + SSNR_PATTERN, + ENC_AUX_FILES, + ODG_PATTERN, + MLD, + MAX_ABS_DIFF, + SSNR, + ODG, + MAX_ENC_DIFF, + MAX_ENC_DIFF_PATTERN, +) logger = logging.getLogger(__name__) USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature @@ -187,7 +199,7 @@ def pytest_addoption(parser): "--enc_stats", action="store_true", help="Activate logging and comparison of statistics from the encoder.", - ) + ) parser.addoption( "--odg", @@ -963,14 +975,22 @@ def pytest_configure(config): @pytest.fixture(scope="session") -def props_to_record(request, get_mld, get_ssnr, get_odg) -> str: - props = ["MAXIMUM ABS DIFF"] - if get_mld: - props.append("MLD") - if get_ssnr: - props.append("SSNR") - if get_odg: - props.append("ODG") +def props_to_record( + request, get_mld, get_ssnr, get_odg, get_enc_stats, encoder_only +) -> str: + props = [] + + if get_enc_stats: + props.append(MAX_ENC_DIFF) + + if not encoder_only: + props.append(MAX_ABS_DIFF) + if get_mld: + props.append(MLD) + if get_ssnr: + props.append(SSNR) + if get_odg: + props.append(ODG) return props @@ -983,10 +1003,10 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: props = dict() for prop in props_to_record: - if prop == "MLD": + if prop == MLD: mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0]) props[prop] = mld - elif prop == "MAXIMUM ABS DIFF": + elif prop == MAX_ABS_DIFF: max_diff = 0 if output_differs: if (match := re.search(MAX_DIFF_PATTERN, text_to_parse)) is not None: @@ -994,19 +1014,26 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: else: raise MaxDiffPatternNotFound() props[prop] = max_diff - elif prop == "SSNR": + elif prop == SSNR: ssnrs = re.findall(SSNR_PATTERN, text_to_parse) min_ssnr = min(ssnrs) min_ssnr_channel = ssnrs.index(min_ssnr) props["MIN_SSNR"] = min_ssnr props["MIN_SSNR_CHANNEL"] = min_ssnr_channel - elif prop == "ODG": + elif prop == ODG: odgs = re.findall(ODG_PATTERN, text_to_parse) - print(odgs) min_odg = min(odgs) min_odg_channel = odgs.index(min_odg) props["MIN_ODG"] = min_odg props["MIN_ODG_CHANNEL"] = min_odg_channel + elif prop == MAX_ENC_DIFF: + search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse) + max_enc_diff = 0 + if search_result: + max_enc_diff, max_enc_diff_ratio = search_result.groups(0) + if max_enc_diff: + max_enc_diff = float(max_enc_diff) + props[MAX_ENC_DIFF] = max_enc_diff return props diff --git a/tests/constants.py b/tests/constants.py index d90c452270..a3b3e5fd6d 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -5,6 +5,15 @@ HERE = pathlib.Path(__file__).parent.absolute() SCRIPTS_DIR = HERE.parent.joinpath("scripts") TESTV_DIR = SCRIPTS_DIR.joinpath("testv") + +# Properties to record +MLD = "MLD" +MAX_ABS_DIFF = "MAXIMUM ABS DIFF" +SSNR = "SSNR" +ODG = "ODG" +MAX_ENC_DIFF = "MAXIMUM ENC DIFF" + + # regex patterns for parsing the output from comparisons -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" @@ -13,12 +22,12 @@ ODG_PATTERN = r"ODG: (-*\d*\.\d*)" SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)" MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)" -MIN_ENC_FILE_LENGTH_DIFF = ( - 0.1 # minimum difference between ref and dut encoder file lengths -) -MIN_ENC_STATS_DIFF = ( - 0.1 # minimum difference between the statistics of ref and dut encoder files -) + +# minimum difference between ref and dut encoder file lengths +MIN_ENC_FILE_LENGTH_DIFF = 0.1 +# minimum difference between the statistics of ref and dut encoder files +MIN_ENC_STATS_DIFF = 0.1 + # list of encoder filename patterns with their data type and number of samples per frame # note: instead of specifying the number of samples per frame, you can use a formula incl. 'fs', e.g. 'fs/50' -- GitLab From 6042b4afee74ad658b170ce1f176c037dac4212a Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 15 Jul 2024 10:58:53 +0200 Subject: [PATCH 09/28] fix standalone function --- tests/cmp_stats_files.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/cmp_stats_files.py b/tests/cmp_stats_files.py index 459f211b8f..dfebbd28a5 100644 --- a/tests/cmp_stats_files.py +++ b/tests/cmp_stats_files.py @@ -126,20 +126,14 @@ def cmp_stats_files( if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("ref_file", type=str) - parser.add_argument("cut_file", type=str) - parser.add_argument("--data_type", type=str, default="int16", dest="dtype") - parser.add_argument( - "--nsamples_per_frame", type=str, default=960, dest="nsamples_per_frame" - ) - parser.add_argument("--len_check", type=int, default=0, dest="len_check") - parser.add_argument("--min_diff_thr", type=float, default=0.0, dest="min_diff_thr") + + parser.add_argument("ref_stats_file", type=str) + parser.add_argument("dut_stats_file", type=str) + parser.add_argument("--min_enc_file_length_diff", type=float, default=0.1, dest="min_enc_file_length_diff") + parser.add_argument("--min_enc_stats_diff", type=float, default=0.1, dest="min_enc_stats_diff") args = parser.parse_args() - # convert dtype from str - if isinstance(args.dtype, str): - args.dtype = np.dtype(getattr(np, args.dtype)) - - result, msg = cmp_bin_files(**vars(args)) - print(msg) - sys.exit(result) + enc_test_result, enc_test_result_msg = cmp_stats_files(**vars(args)) + print(enc_test_result_msg) + + sys.exit(enc_test_result) -- GitLab From 8ca8a081ce58d68997b9fbbd3bb190796fa0045d Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 15 Jul 2024 13:12:50 +0200 Subject: [PATCH 10/28] improve formatting --- tests/codec_be_on_mr_nonselection/test_param_file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 7d01c99ba9..0055caddaa 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -218,8 +218,8 @@ def test_param_file_tests( # compare binary files extracted from the encoder if get_enc_stats: - print("Comparing encoder auxiliary files") - print("=================================\n") + print("Comparing encoder files") + print("=======================\n") stats_file = bitstream_file.replace(".192", ".stats") ref_stats_file = f"{reference_path}/param_file/enc/{stats_file}" @@ -233,6 +233,7 @@ def test_param_file_tests( min_enc_stats_diff=MIN_ENC_STATS_DIFF, ) + print("") cmp_result_msg += enc_test_result_msg if encoder_only: -- GitLab From f9e9c9176faabd7f9854c3e3cca9e42929500c38 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 29 Jul 2024 15:24:29 +0200 Subject: [PATCH 11/28] use new wrapper function in pca test, no additions yet --- tests/codec_be_on_mr_nonselection/test_sba.py | 157 +++++++++++++----- 1 file changed, 112 insertions(+), 45 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 0b95d46234..ff1209ae36 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -35,8 +35,6 @@ __doc__ = """ import errno import os -import tempfile -from pathlib import Path import pytest from cut_bs import cut_from_start @@ -96,7 +94,7 @@ def check_and_makedir(dir_path): @pytest.mark.create_ref @pytest.mark.parametrize("tag", tag_list) -@pytest.mark.parametrize("fs", sample_rate_list) +@pytest.mark.parametrize("sampling_rate", sample_rate_list) def test_pca_enc( record_property, props_to_record, @@ -110,71 +108,52 @@ def test_pca_enc( update_ref, keep_files, tag, - fs, + sampling_rate, get_mld, get_mld_lim, + encoder_only, decoder_only, abs_tol, get_ssnr, get_odg, ): pca = True - tag = tag + fs + "c" - ivas_br = "256000" - dtx = "0" - max_bw = "FB" - gain_flag = -1 + tag = tag + sampling_rate + "c" + bitrate = "256000" + ivas_max_bw = "FB" sba_order = "+1" output_config = "FOA" + br_switch_file_path = None + cut_testv = True - if not decoder_only: - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_testv=True, - pca=pca, - ) - - # dec - sba_dec( + run_sba( record_property, props_to_record, + dut_encoder_frontend, dut_decoder_frontend, + ref_encoder_frontend, ref_decoder_frontend, + test_vector_path, reference_path, dut_base_path, + br_switch_file_path, tag, - fs, - ivas_br, - dtx, - None, - max_bw, + sampling_rate, + bitrate, + ivas_max_bw, + sba_order, output_config, update_ref, - gain_flag, keep_files, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_odg, + encoder_only, decoder_only, - get_mld=get_mld, - get_mld_lim=get_mld_lim, + cut_testv=cut_testv, pca=pca, - abs_tol=abs_tol, - get_ssnr=get_ssnr, - get_odg=get_odg, ) @@ -885,3 +864,91 @@ def sba_dec( os.remove(dut_out_raw) if not decoder_only and plc_file is None: os.remove(dut_in_pkt) + + +def run_sba( + record_property, + props_to_record, + dut_encoder_frontend, + dut_decoder_frontend, + ref_encoder_frontend, + ref_decoder_frontend, + test_vector_path, + reference_path, + dut_base_path, + br_switch_file_path, + tag, + sampling_rate, + bitrate, + ivas_max_bw, + sba_order, + output_config, + update_ref, + keep_files, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_odg, + encoder_only, + decoder_only, + dtx="0", + sid=None, + gain_flag=-1, + cut_gain="1.0", + create_dutenc=False, + cut_testv=False, + pca=False, +): + if not decoder_only: + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + br_switch_file_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + ivas_max_bw, + sba_order, + update_ref, + gain_flag, + keep_files, + cut_gain, + create_dutenc, + cut_testv, + pca, + ) + + if not encoder_only: + sba_dec( + record_property, + props_to_record, + dut_decoder_frontend, + ref_decoder_frontend, + reference_path, + dut_base_path, + tag, + sampling_rate, + bitrate, + dtx, + None, + ivas_max_bw, + output_config, + update_ref, + gain_flag, + keep_files, + decoder_only, + get_mld=get_mld, + get_mld_lim=get_mld_lim, + pca=pca, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + ) + + # record properties tbd -- GitLab From 38958c9c52c17597b91b8723db2b3ea10e77c569 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 29 Jul 2024 16:14:59 +0200 Subject: [PATCH 12/28] harmonize name tags --- tests/codec_be_on_mr_nonselection/test_sba.py | 21 ++++--------------- tests/conftest.py | 15 +++++++------ 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index ff1209ae36..7aa0d524da 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -653,17 +653,11 @@ def sba_enc( short_tag_ext += "_SID" if pca: short_tag_ext += "_pca" - # to avoid conflicting names in case of parallel test execution, differentiate all cases - long_tag_ext = "" - if gain_flag != -1: - long_tag_ext += f"_Gain{gain_flag}" - if SID == 1: - long_tag_ext += "_SID" - dut_pkt_file = f"{dut_out_dir}/{tag_out}{long_tag_ext}.192" + dut_pkt_file = f"{dut_out_dir}/{tag_out}{short_tag_ext}.192" ref_pkt_file = f"{ref_out_dir}/{tag_out}{short_tag_ext}.192" ref_pkt_file_dutenc = f"{ref_out_dir}/{tag_out}{short_tag_ext}_dutenc.192" if SID == 1: - dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{long_tag_ext}_cut.192" + dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{short_tag_ext}_cut.192" ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{short_tag_ext}_cut.192" ref_pkt_file_dutenc_cut = ( f"{ref_out_dir}/{tag_out}{short_tag_ext}_dutenc_cut.192" @@ -779,16 +773,10 @@ def sba_dec( short_tag_ext += "_pca" if SID == 1: short_tag_ext += "_SID_cut" - # to avoid conflicting names in case of parallel test execution, differentiate all cases - long_tag_ext = "" - if gain_flag != -1: - long_tag_ext += f"_Gain{gain_flag}" - if SID == 1: - long_tag_ext += "_SID_cut" dut_out_dir = f"{dut_base_path}/sba_bs/raw" ref_out_dir = f"{reference_path}/sba_bs/raw" - dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.192" + dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{short_tag_ext}.192" ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}.192" ref_in_pkt_dutenc = ( f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}_dutenc.192" @@ -804,10 +792,9 @@ def sba_dec( plc_file = None if plc_pattern is not None: plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" - long_tag_ext = f"{long_tag_ext}_{plc_pattern}" short_tag_ext = f"{short_tag_ext}_{plc_pattern}" - dut_out_raw = f"{dut_out_dir}/{tag_out}{long_tag_ext}.wav" + dut_out_raw = f"{dut_out_dir}/{tag_out}{short_tag_ext}.wav" ref_out_raw = f"{ref_out_dir}/{tag_out}{short_tag_ext}.wav" check_and_makedir(dut_out_dir) diff --git a/tests/conftest.py b/tests/conftest.py index 426e8d9570..c943642c0c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,16 +57,15 @@ from .constants import ( ODG, MAX_ENC_DIFF, MAX_ENC_DIFF_PATTERN, + SCRIPTS_DIR, ) logger = logging.getLogger(__name__) USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature -HERE = Path(__file__).parent -SCRIPTS_DIR = str(HERE.parent.joinpath("scripts").absolute()) import sys -sys.path.append(SCRIPTS_DIR) +sys.path.append(str(SCRIPTS_DIR)) import prepare_combined_format_inputs @@ -657,18 +656,18 @@ class DecoderFrontend: system = platform.system() if system == "Windows": - eid_path = "./scripts/tools/Win32/eid-xor.exe" + eid_path = SCRIPTS_DIR.joinpath("tools/Win32/eid-xor.exe") elif system == "Linux": - eid_path = "./scripts/tools/Linux/eid-xor" + eid_path = SCRIPTS_DIR.joinpath("tools/Linux/eid-xor") elif system == "Darwin": - eid_path = "./scripts/tools/Darwin/eid-xor" + eid_path = SCRIPTS_DIR.joinpath("tools/Darwin/eid-xor") else: raise ValueError(f'Wrong system "{system}"!') - if not os.path.isfile(eid_path): + if not eid_path.exists(): raise FileNotFoundError(f"eid-xor binary {eid_path} not found!\n") - eid_command = [eid_path] + eid_command = [str(eid_path)] eid_command.extend(["-fer", "-vbr", "-bs", "g192", "-ep", "g192"]) eid_output_suffix = "." + os.path.basename(plc_file) + ".fer" eid_command += [ -- GitLab From 5ec9da74ee309189f535f68cb077ea1c0cfc29ae Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 30 Jul 2024 18:02:20 +0200 Subject: [PATCH 13/28] add encoder verfication mechanism to the sba tests --- tests/codec_be_on_mr_nonselection/test_sba.py | 420 +++++++++--------- 1 file changed, 203 insertions(+), 217 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 7aa0d524da..d98c26c8ab 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -35,6 +35,7 @@ __doc__ = """ import errno import os +from typing import Tuple import pytest from cut_bs import cut_from_start @@ -42,7 +43,8 @@ from cut_bs import cut_from_start from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend from ..conftest import parse_properties -from ..constants import TESTV_DIR +from ..cmp_stats_files import cmp_stats_files +from ..constants import TESTV_DIR, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF # params @@ -116,6 +118,7 @@ def test_pca_enc( abs_tol, get_ssnr, get_odg, + get_enc_stats, ): pca = True tag = tag + sampling_rate + "c" @@ -123,7 +126,6 @@ def test_pca_enc( ivas_max_bw = "FB" sba_order = "+1" output_config = "FOA" - br_switch_file_path = None cut_testv = True run_sba( @@ -136,7 +138,6 @@ def test_pca_enc( test_vector_path, reference_path, dut_base_path, - br_switch_file_path, tag, sampling_rate, bitrate, @@ -150,6 +151,7 @@ def test_pca_enc( abs_tol, get_ssnr, get_odg, + get_enc_stats, encoder_only, decoder_only, cut_testv=cut_testv, @@ -158,12 +160,12 @@ def test_pca_enc( @pytest.mark.create_ref -@pytest.mark.parametrize("ivas_br", ivas_br_FOA) +@pytest.mark.parametrize("bitrate", ivas_br_FOA) @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list) -@pytest.mark.parametrize("fs", sample_rate_list) +@pytest.mark.parametrize("sampling_rate", sample_rate_list) @pytest.mark.parametrize("gain_flag", gain_list) -@pytest.mark.parametrize("SID", SID_list) +@pytest.mark.parametrize("sid", SID_list) def test_sba_enc_system( record_property, props_to_record, @@ -177,39 +179,41 @@ def test_sba_enc_system( br_switch_file_path, update_ref, keep_files, - ivas_br, + bitrate, dtx, tag, - fs, + sampling_rate, gain_flag, - SID, + sid, get_mld, get_mld_lim, + encoder_only, decoder_only, abs_tol, get_ssnr, get_odg, + get_enc_stats, ): - if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: + if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() - if SID == 1: + if sid == 1: if ( - ivas_br not in ["13200", "16400", "64000"] - or fs == "16" + bitrate not in ["13200", "16400", "64000"] + or sampling_rate == "16" or gain_flag == 1 or dtx == "0" ): pytest.skip() else: - if ivas_br in ["13200", "16400"]: + if bitrate in ["13200", "16400"]: pytest.skip() - if ivas_br == "sw_24k4_256k.bin" and gain_flag != 1: + if bitrate == "sw_24k4_256k.bin" and gain_flag != 1: pytest.skip() - if gain_flag == 1 and ivas_br not in ["13200", "16400", "24400", "32000"]: + if gain_flag == 1 and bitrate not in ["13200", "16400", "24400", "32000"]: pytest.skip() - tag = tag + fs + "c" - max_bw = "FB" + tag = tag + sampling_rate + "c" + ivas_max_bw = "FB" sba_order = "+1" output_config = "FOA" if gain_flag == 1: @@ -219,59 +223,43 @@ def test_sba_enc_system( else: cut_gain = "1.0" - if not decoder_only: - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - br_switch_file_path, - tag, - fs, - ivas_br, - dtx, - SID, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_gain=cut_gain, - create_dutenc=True, - cut_testv=True, - ) - - # dec - sba_dec( + run_sba( record_property, props_to_record, + dut_encoder_frontend, dut_decoder_frontend, + ref_encoder_frontend, ref_decoder_frontend, + test_vector_path, reference_path, dut_base_path, tag, - fs, - ivas_br, - dtx, - SID, - max_bw, + sampling_rate, + bitrate, + ivas_max_bw, + sba_order, output_config, update_ref, - gain_flag, keep_files, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_odg, + get_enc_stats, + encoder_only, decoder_only, - get_mld=get_mld, - get_mld_lim=get_mld_lim, - abs_tol=abs_tol, - get_ssnr=get_ssnr, - get_odg=get_odg, + cut_gain=cut_gain, + br_switch_file_path=br_switch_file_path, + sid=sid, + create_dutenc=True, + gain_flag=gain_flag, + dtx=dtx, ) @pytest.mark.create_ref -@pytest.mark.parametrize("ivas_br", ivas_br_HOA2) +@pytest.mark.parametrize("bitrate", ivas_br_HOA2) @pytest.mark.parametrize("tag", tag_list_HOA2) def test_spar_hoa2_enc_system( record_property, @@ -285,74 +273,55 @@ def test_spar_hoa2_enc_system( ref_decoder_frontend, update_ref, keep_files, - ivas_br, + bitrate, tag, get_mld, get_mld_lim, + encoder_only, decoder_only, abs_tol, get_ssnr, get_odg, + get_enc_stats, ): - fs = "48" - dtx = "0" - gain_flag = -1 + sampling_rate = "48" - tag = tag + fs + "c" - max_bw = "FB" + tag = tag + sampling_rate + "c" + ivas_max_bw = "FB" sba_order = "+2" output_config = "HOA2" - if not decoder_only: - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - ) - - # dec - sba_dec( + run_sba( record_property, props_to_record, + dut_encoder_frontend, dut_decoder_frontend, + ref_encoder_frontend, ref_decoder_frontend, + test_vector_path, reference_path, dut_base_path, tag, - fs, - ivas_br, - dtx, - None, - max_bw, + sampling_rate, + bitrate, + ivas_max_bw, + sba_order, output_config, update_ref, - gain_flag, keep_files, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_odg, + get_enc_stats, + encoder_only, decoder_only, - get_mld=get_mld, - get_mld_lim=get_mld_lim, - abs_tol=abs_tol, - get_ssnr=get_ssnr, - get_odg=get_odg, ) @pytest.mark.create_ref -@pytest.mark.parametrize("ivas_br", ivas_br_HOA3) +@pytest.mark.parametrize("bitrate", ivas_br_HOA3) @pytest.mark.parametrize("tag", tag_list_HOA3) def test_spar_hoa3_enc_system( record_property, @@ -366,74 +335,55 @@ def test_spar_hoa3_enc_system( ref_decoder_frontend, update_ref, keep_files, - ivas_br, + bitrate, tag, get_mld, get_mld_lim, + encoder_only, decoder_only, abs_tol, get_ssnr, get_odg, + get_enc_stats, ): - fs = "48" - dtx = "0" - gain_flag = -1 + sampling_rate = "48" - tag = tag + fs + "c" - max_bw = "FB" + tag = tag + sampling_rate + "c" + ivas_max_bw = "FB" sba_order = "+3" output_config = "HOA3" - if not decoder_only: - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - ) - - # dec - sba_dec( + run_sba( record_property, props_to_record, + dut_encoder_frontend, dut_decoder_frontend, + ref_encoder_frontend, ref_decoder_frontend, + test_vector_path, reference_path, dut_base_path, tag, - fs, - ivas_br, - dtx, - None, - max_bw, + sampling_rate, + bitrate, + ivas_max_bw, + sba_order, output_config, update_ref, - gain_flag, keep_files, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_odg, + get_enc_stats, + encoder_only, decoder_only, - get_mld=get_mld, - get_mld_lim=get_mld_lim, - abs_tol=abs_tol, - get_ssnr=get_ssnr, - get_odg=get_odg, ) @pytest.mark.create_ref -@pytest.mark.parametrize("ivas_br", ivas_br_FOA) +@pytest.mark.parametrize("bitrate", ivas_br_FOA) @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list_bw_force) @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) @@ -449,77 +399,59 @@ def test_sba_enc_BWforce_system( ref_decoder_frontend, update_ref, keep_files, - ivas_br, + bitrate, dtx, tag, sample_rate_bw_idx, get_mld, get_mld_lim, + encoder_only, decoder_only, abs_tol, get_ssnr, get_odg, + get_enc_stats, ): - if dtx == "1" and ivas_br not in ["32000", "64000"]: + if dtx == "1" and bitrate not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() - if ivas_br == "13200" or ivas_br == "16400": + if bitrate == "13200" or bitrate == "16400": pytest.skip() - if ivas_br == "sw_24k4_256k.bin": + if bitrate == "sw_24k4_256k.bin": pytest.skip() - fs = sample_rate_bw_idx[0] - bw = sample_rate_bw_idx[1] - tag = tag + fs + "c" - gain_flag = -1 + sampling_rate = sample_rate_bw_idx[0] + ivas_max_bw = sample_rate_bw_idx[1] + tag = tag + sampling_rate + "c" sba_order = "+1" output_config = "FOA" - if not decoder_only: - # enc - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - None, - tag, - fs, - ivas_br, - dtx, - None, - bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_testv=True, - ) - - # dec - sba_dec( + run_sba( record_property, props_to_record, + dut_encoder_frontend, dut_decoder_frontend, + ref_encoder_frontend, ref_decoder_frontend, + test_vector_path, reference_path, dut_base_path, tag, - fs, - ivas_br, - dtx, - None, - bw, + sampling_rate, + bitrate, + ivas_max_bw, + sba_order, output_config, update_ref, - gain_flag, keep_files, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_odg, + get_enc_stats, + encoder_only, decoder_only, - get_mld=get_mld, - get_mld_lim=get_mld_lim, - abs_tol=abs_tol, - get_ssnr=get_ssnr, - get_odg=get_odg, + dtx=dtx, ) @@ -530,11 +462,11 @@ def test_sba_enc_BWforce_system( # -> the reference generation for this test (reference decoder output) needs to be done after completion of test_sba_enc_system # -> therefore the marker create_ref_part2 @pytest.mark.create_ref_part2 -@pytest.mark.parametrize("ivas_br", ivas_br_plc) +@pytest.mark.parametrize("bitrate", ivas_br_plc) @pytest.mark.parametrize("dtx", dtx_set) @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("plc_pattern", plc_patterns) -@pytest.mark.parametrize("fs", sample_rate_list) +@pytest.mark.parametrize("sampling_rate", sample_rate_list) @pytest.mark.parametrize("gain_flag", gain_list) def test_sba_plc_system( record_property, @@ -546,61 +478,73 @@ def test_sba_plc_system( ref_decoder_path, update_ref, keep_files, - ivas_br, + bitrate, dtx, tag, plc_pattern, - fs, + sampling_rate, gain_flag, get_mld, get_mld_lim, abs_tol, get_ssnr, get_odg, + get_enc_stats, + test_vector_path, ): - SID = 0 - if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: + sid = 0 + if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() - if ivas_br == "13200" or ivas_br == "16400": + if bitrate == "13200" or bitrate == "16400": if ( dtx == "1" and gain_flag == 0 - and fs != "16" + and sampling_rate != "16" and plc_pattern == "PLperc12mblen5" ): - SID = 1 + sid = 1 else: pytest.skip() - if gain_flag == 1 and ivas_br not in ["13200", "16400", "24400", "32000"]: + if gain_flag == 1 and bitrate not in ["13200", "16400", "24400", "32000"]: pytest.skip() - tag = tag + fs + "c" + tag = tag + sampling_rate + "c" - # dec - sba_dec( + ivas_max_bw = "FB" + output_config = "FOA" + sba_order = "+1" + dut_encoder_frontend = None + ref_encoder_frontend = None + + run_sba( record_property, props_to_record, + dut_encoder_frontend, dut_decoder_frontend, + ref_encoder_frontend, ref_decoder_frontend, + test_vector_path, reference_path, dut_base_path, tag, - fs, - ivas_br, - dtx, - SID, - "FB", - "FOA", + sampling_rate, + bitrate, + ivas_max_bw, + sba_order, + output_config, update_ref, - gain_flag, keep_files, - False, - plc_pattern=plc_pattern, - get_mld=get_mld, - get_mld_lim=get_mld_lim, - abs_tol=abs_tol, - get_ssnr=get_ssnr, - get_odg=get_odg, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_odg, + get_enc_stats, + encoder_only=False, + decoder_only=True, + sid=sid, + dtx=dtx, + gain_flag=gain_flag, ) @@ -627,7 +571,8 @@ def sba_enc( create_dutenc=False, cut_testv=False, pca=False, -): + get_enc_stats=False, +) -> Tuple[str, str]: # ------------ run cmd ------------ dut_out_dir = f"{dut_base_path}/sba_bs/pkt" ref_out_dir = f"{reference_path}/sba_bs/pkt" @@ -675,6 +620,9 @@ def sba_enc( input_path = cut_file + stats_file_ref = ref_pkt_file.replace(".192", ".stats") + stats_file_dut = dut_pkt_file.replace(".192", ".stats") + if ref_encoder_frontend: # call REF encoder ref_encoder_frontend.run( @@ -686,6 +634,7 @@ def sba_enc( max_band=ivas_max_bw, pca=pca, dtx_mode=dtx_mode, + stats_file=stats_file_ref if get_enc_stats else None, ) if create_dutenc: # for PLC decoder tests, create bitstream using DUT encoder @@ -711,6 +660,7 @@ def sba_enc( max_band=ivas_max_bw, pca=pca, dtx_mode=dtx_mode, + stats_file=stats_file_dut if get_enc_stats else None, ) if SID == 1: @@ -731,10 +681,13 @@ def sba_enc( if not keep_files: os.remove(dut_pkt_file) + return stats_file_ref, stats_file_dut + def sba_dec( record_property, props_to_record, + cmp_result_msg, decoder_frontend, ref_decoder_frontend, reference_path, @@ -838,13 +791,14 @@ def sba_dec( get_odg=get_odg, ) - props = parse_properties(reason, cmp_result != 0, props_to_record) + text_to_parse = cmp_result_msg + reason + props = parse_properties(text_to_parse, cmp_result != 0, props_to_record) for k, v in props.items(): record_property(k, v) # report compare result if cmp_result != 0: - pytest.fail(reason) + pytest.fail(text_to_parse) # remove DUT output files when test result is OK (to save disk space) if not keep_files: @@ -863,7 +817,6 @@ def run_sba( test_vector_path, reference_path, dut_base_path, - br_switch_file_path, tag, sampling_rate, bitrate, @@ -877,18 +830,21 @@ def run_sba( abs_tol, get_ssnr, get_odg, + get_enc_stats, encoder_only, decoder_only, dtx="0", - sid=None, + sid=0, + br_switch_file_path=None, gain_flag=-1, cut_gain="1.0", create_dutenc=False, cut_testv=False, pca=False, ): + cmp_result_msg = "" if not decoder_only: - sba_enc( + stats_file_ref, stats_file_dut = sba_enc( dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -909,12 +865,44 @@ def run_sba( create_dutenc, cut_testv, pca, + get_enc_stats, ) + if get_enc_stats: + print("Comparing encoder files") + print("=======================\n") + + # compare ref and dut .stats files + enc_test_result, enc_test_result_msg = cmp_stats_files( + stats_file_ref, + stats_file_dut, + min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF, + min_enc_stats_diff=MIN_ENC_STATS_DIFF, + ) + + print("") + cmp_result_msg += enc_test_result_msg + + # TODO: there is cleanup potential here: + # - move filename handling outside of the sba_enc/dec functions into start of this function + # - move file cleanup part out as well + # - move this comparison to the end and the one in sba_dec there as well + if encoder_only: + props = parse_properties( + cmp_result_msg, enc_test_result != 0, props_to_record + ) + for k, v in props.items(): + record_property(k, v) + + # report compare result + if enc_test_result != 0: + pytest.fail(cmp_result_msg) + if not encoder_only: sba_dec( record_property, props_to_record, + cmp_result_msg, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -923,7 +911,7 @@ def run_sba( sampling_rate, bitrate, dtx, - None, + sid, ivas_max_bw, output_config, update_ref, @@ -937,5 +925,3 @@ def run_sba( get_ssnr=get_ssnr, get_odg=get_odg, ) - - # record properties tbd -- GitLab From cf1994c50153df0e5b8ef6c7ea9860978efd1cb5 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 8 Aug 2024 16:21:56 +0200 Subject: [PATCH 14/28] convert paths into absolute ones --- tests/codec_be_on_mr_nonselection/test_param_file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 0055caddaa..f1107009d5 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -40,7 +40,6 @@ from pathlib import Path from subprocess import run import pytest import numpy as np -import re from tests.cmp_pcm import cmp_pcm @@ -48,9 +47,9 @@ from tests.cmp_stats_files import cmp_stats_files from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties from tests.testconfig import PARAM_FILE from tests.constants import ( - MAX_ENC_DIFF_PATTERN, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF, + SCRIPTS_DIR, ) @@ -313,6 +312,8 @@ def test_param_file_tests( ] # remove leading "../" dec_split = [x[3:] if x.startswith("../") else x for x in dec_split] + # convert "scripts/" paths into absolute ones + dec_split = [str(SCRIPTS_DIR.joinpath(x[8:])) if x.startswith("scripts/") else x for x in dec_split] output_file = dec_split.pop() bitstream_file_dec = dec_split.pop() -- GitLab From 14d303d0efa10a9879cbcc6b589c18190820653b Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Mon, 12 Aug 2024 15:39:16 +0200 Subject: [PATCH 15/28] [fix] try to fix networkSimulator_g192 path --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c943642c0c..df6c26c29a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -694,11 +694,11 @@ class DecoderFrontend: # TODO: centralize this in a utils file if system == "Windows": - netsim_path = "./scripts/tools/Win32/networkSimulator_g192.exe" + netsim_path = SCRIPTS_DIR.joinpath("tools/Win32/networkSimulator_g192.exe") elif system == "Linux": - netsim_path = "./scripts/tools/Linux/networkSimulator_g192" + netsim_path = SCRIPTS_DIR.joinpath("tools/Linux/networkSimulator_g192") elif system == "Darwin": - netsim_path = "./scripts/tools/Darwin/networkSimulator_g192" + netsim_path = SCRIPTS_DIR.joinpath("tools/Darwin/networkSimulator_g192") else: raise ValueError(f'Wrong system "{system}"!') -- GitLab From d16853bdc16dee2fb85a5d26919f5eba500f6a95 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Tue, 10 Sep 2024 12:35:26 +0200 Subject: [PATCH 16/28] fix encoder_only option --- .../test_param_file.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 87e17ec7ab..eaec8792cd 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -41,7 +41,6 @@ from subprocess import run import pytest import numpy as np - from tests.cmp_pcm import cmp_pcm from tests.cmp_stats_files import cmp_stats_files from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties @@ -198,7 +197,7 @@ def test_param_file_tests( cmp_result_msg = "" enc_test_result = 0 - + if not decoder_only: encode( dut_encoder_frontend, @@ -215,7 +214,7 @@ def test_param_file_tests( ) # compare binary files extracted from the encoder - if get_enc_stats: + if update_ref in [0, 2] and get_enc_stats: print("Comparing encoder files") print("=======================\n") @@ -233,15 +232,19 @@ def test_param_file_tests( print("") cmp_result_msg += enc_test_result_msg - + props = parse_properties(cmp_result_msg, False, props_to_record) + for k, v in props.items(): + record_property(k, v) + + if enc_test_result: + pytest.fail("Too high difference in encoder statistics found.") + else: + # remove DUT stats file when test result is OK (to save disk space) + if not keep_files: + os.remove(dut_stats_file) + if encoder_only: - props = parse_properties(cmp_result_msg, False, props_to_record) - for k, v in props.items(): - record_property(k, v) - - if enc_test_result: - pytest.fail("Too high difference in encoder statistics found.") - return + return # don't proceed with the decoder if user specified --encoder_only on the command line # check for networkSimulator_g192 command line if sim_opts != "": @@ -275,6 +278,7 @@ def test_param_file_tests( ) # check for eid-xor command line + if eid_opts != "": eid_split = eid_opts.split() assert len(eid_split) >= 3, "eid-xor expects at least 3 parameters" @@ -291,6 +295,7 @@ def test_param_file_tests( # -> construct netsim output file name eid_xor_outfile = f"{testv_base}_{tag_str}.fer.192" eid_split[-1] = eid_xor_outfile + error_insertion( reference_path, dut_base_path, -- GitLab From 0e78465f698c590e29cac428e17e35014e8ab97c Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Thu, 12 Sep 2024 10:42:11 +0200 Subject: [PATCH 17/28] remove repeated input parameter --- tests/codec_be_on_mr_nonselection/test_sba.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 0c02857684..2f3c0f0d8d 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -591,8 +591,7 @@ def test_sba_plc_system( abs_tol, get_ssnr, get_odg, - get_enc_stats, - test_vector_path, + get_enc_stats ): sid = 0 if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: -- GitLab From d84157d147919742a01282db1ab9e9151c5ed3b4 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 16 Sep 2024 12:45:13 +0200 Subject: [PATCH 18/28] fix SBA tests --- tests/codec_be_on_mr_nonselection/test_sba.py | 926 ++++++++++-------- 1 file changed, 514 insertions(+), 412 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 2f3c0f0d8d..04c3fb51e8 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -36,7 +36,6 @@ __doc__ = """ import errno import os from typing import Tuple - import pytest from cut_bs import cut_from_start @@ -47,12 +46,11 @@ from ..cmp_stats_files import cmp_stats_files from ..constants import TESTV_DIR, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF from tests.testconfig import use_ltv -# params +import pdb tag_list = ["ltvFOA" if use_ltv else "stvFOA"] tag_list_HOA2 = ["ltvHOA2" if use_ltv else "stv2OA"] tag_list_HOA3 = ["ltvHOA3" if use_ltv else "stv3OA"] - tag_list_bw_force = tag_list dtx_set = ["0", "1"] dict_fsample_bw = {"48": "3", "32": "2", "16": "1"} @@ -75,9 +73,7 @@ ivas_br_HOA3 = ["256000", "384000", "512000"] SID_list = [0, 1] sample_rate_list = ["48", "32", "16"] gain_list = [0, 1] - sample_rate_bw_idx_list = [("48", "SWB"), ("48", "WB"), ("32", "WB")] - AbsTol = "0" # params for PLC test @@ -107,6 +103,7 @@ def test_pca_enc( dut_base_path, ref_encoder_frontend, ref_decoder_frontend, + br_switch_file_path, update_ref, keep_files, tag, @@ -121,77 +118,79 @@ def test_pca_enc( get_enc_stats, ): pca = True - ivas_br = "256000" + bitrate = "256000" dtx = "0" + sid = 0 max_bw = "FB" gain_flag = -1 sba_order = "+1" output_config = "FOA" cut_testv = True + cut_gain = "1.0" + plc_pattern = None if "ltv" in tag: - tag = f"ltv{fs}_FOA" + tag = f"ltv{sampling_rate}_FOA" cut_testv = False elif "stv" in tag: - tag = tag + fs + "c" + tag = tag + sampling_rate + "c" cut_testv=True else: assert 0 if not decoder_only: - # enc sba_enc( dut_encoder_frontend, test_vector_path, ref_encoder_frontend, reference_path, dut_base_path, - None, + br_switch_file_path, tag, - fs, - ivas_br, + sampling_rate, + bitrate, dtx, - None, + sid, max_bw, sba_order, update_ref, gain_flag, - keep_files, + keep_files=keep_files, + cut_gain=cut_gain, cut_testv=cut_testv, pca=pca, + plc_pattern=plc_pattern, + get_enc_stats=get_enc_stats, ) - - # dec - sba_dec( - record_property, - props_to_record, - dut_encoder_frontend, - dut_decoder_frontend, - ref_encoder_frontend, - ref_decoder_frontend, - test_vector_path, - reference_path, - dut_base_path, - tag, - sampling_rate, - bitrate, - ivas_max_bw, - sba_order, - output_config, - update_ref, - keep_files, - get_mld, - get_mld_lim, - abs_tol, - get_ssnr, - get_odg, - get_enc_stats, - encoder_only, - decoder_only, - cut_testv=cut_testv, - pca=pca, - ) - + + + # sba_dec( + # record_property, + # props_to_record, + # dut_decoder_frontend, + # ref_decoder_frontend, + # reference_path, + # dut_base_path, + # tag, + # sampling_rate, + # bitrate, + # dtx, + # sid, + # max_bw, + # output_config, + # update_ref, + # gain_flag, + # keep_files, + # decoder_only, + # plc_pattern=plc_pattern, + # get_mld=get_mld, + # get_mld_lim=get_mld_lim, + # pca=pca, + # abs_tol=abs_tol, + # get_ssnr=get_ssnr, + # get_odg=get_odg, + # ) + @pytest.mark.parametrize("bitrate", ivas_br_FOA) @pytest.mark.parametrize("dtx", dtx_set) @@ -227,6 +226,16 @@ def test_sba_enc_system( get_odg, get_enc_stats, ): + + plc_pattern = None + pca = False + max_bw = "FB" + sba_order = "+1" + output_config = "FOA" + cut_gain = "1.0" + plc_pattern = None + cut_testv = True + if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() @@ -247,17 +256,14 @@ def test_sba_enc_system( pytest.skip() if "ltv" in tag: - tag = f"ltv{fs}_FOA" + tag = f"ltv{sampling_rate}_FOA" cut_testv = False elif "stv" in tag: - tag = tag + fs + "c" + tag = tag + sampling_rate + "c" cut_testv = True else: assert 0 - max_bw = "FB" - sba_order = "+1" - output_config = "FOA" if gain_flag == 1: cut_gain = "16.0" elif dtx == "1": @@ -266,8 +272,7 @@ def test_sba_enc_system( cut_gain = "1.0" if not decoder_only: - # enc - sba_enc( + ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -275,53 +280,84 @@ def test_sba_enc_system( dut_base_path, br_switch_file_path, tag, - fs, - ivas_br, + sampling_rate, + bitrate, dtx, - SID, + sid, max_bw, sba_order, update_ref, gain_flag, - keep_files, + keep_files=keep_files, cut_gain=cut_gain, cut_testv=cut_testv, + pca=pca, + plc_pattern=plc_pattern, + get_enc_stats=get_enc_stats, ) + + if update_ref == 0 and get_enc_stats: + print("Comparing encoder files") + print("=======================\n") - # dec - sba_dec( - record_property, - props_to_record, - dut_encoder_frontend, - dut_decoder_frontend, - ref_encoder_frontend, - ref_decoder_frontend, - test_vector_path, - reference_path, - dut_base_path, - tag, - sampling_rate, - bitrate, - ivas_max_bw, - sba_order, - output_config, - update_ref, - keep_files, - get_mld, - get_mld_lim, - abs_tol, - get_ssnr, - get_odg, - get_enc_stats, - encoder_only, - decoder_only, - cut_gain=cut_gain, - br_switch_file_path=br_switch_file_path, - sid=sid, - create_dutenc=True, - gain_flag=gain_flag, - dtx=dtx, - ) + # check if both REF and DUT .stats file exist + if ref_stats_file is None or not os.path.exists(ref_stats_file): + pytest.fail("REF .stats file does not exist or has not been generated!") + if dut_stats_file is None or not os.path.exists(dut_stats_file): + pytest.fail("DUT .stats file does not exist or has not been generated!") + + # compare ref and dut .stats files + enc_test_result, enc_test_result_msg = cmp_stats_files( + ref_stats_file, + dut_stats_file, + min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF, + min_enc_stats_diff=MIN_ENC_STATS_DIFF, + ) + + print("") + + # TODO: there is cleanup potential here: + # - move filename handling outside of the sba_enc/dec functions into start of this function + # - move file cleanup part out as well + # - move this comparison to the end and the one in sba_dec there as well + if encoder_only: + props = parse_properties( + enc_test_result_msg, enc_test_result != 0, props_to_record + ) + for k, v in props.items(): + record_property(k, v) + + # report compare result + if enc_test_result != 0: + pytest.fail(enc_test_result_msg) + + + # sba_dec( + # record_property, + # props_to_record, + # dut_decoder_frontend, + # ref_decoder_frontend, + # reference_path, + # dut_base_path, + # tag, + # sampling_rate, + # bitrate, + # dtx, + # sid, + # max_bw, + # output_config, + # update_ref, + # gain_flag, + # keep_files, + # decoder_only, + # plc_pattern=plc_pattern, + # get_mld=get_mld, + # get_mld_lim=get_mld_lim, + # pca=pca, + # abs_tol=abs_tol, + # get_ssnr=get_ssnr, + # get_odg=get_odg, + # ) @pytest.mark.parametrize("bitrate", ivas_br_HOA2) @@ -336,6 +372,7 @@ def test_spar_hoa2_enc_system( dut_base_path, ref_encoder_frontend, ref_decoder_frontend, + br_switch_file_path, update_ref, keep_files, bitrate, @@ -350,45 +387,111 @@ def test_spar_hoa2_enc_system( get_enc_stats, ): sampling_rate = "48" - + pca = False + max_bw = "FB" + sba_order = "+2" + output_config = "HOA2" + dtx = "0" + sid = 0 + gain_flag = -1 + cut_gain = "1.0" + plc_pattern = None + cut_testv = False + if "ltv" in tag: - tag = f"ltv{fs}_HOA2" + tag = f"ltv{sampling_rate}_HOA2" elif "stv" in tag: - tag = tag + fs + "c" + tag = tag + sampling_rate + "c" else: assert 0 + + if not decoder_only: + ref_stats_file, dut_stats_file = sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + br_switch_file_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files=keep_files, + cut_gain=cut_gain, + cut_testv=cut_testv, + pca=pca, + plc_pattern=plc_pattern, + get_enc_stats=get_enc_stats, + ) - max_bw = "FB" - sba_order = "+2" - output_config = "HOA2" + if update_ref == 0 and get_enc_stats: + print("Comparing encoder files") + print("=======================\n") - run_sba( - record_property, - props_to_record, - dut_encoder_frontend, - dut_decoder_frontend, - ref_encoder_frontend, - ref_decoder_frontend, - test_vector_path, - reference_path, - dut_base_path, - tag, - sampling_rate, - bitrate, - ivas_max_bw, - sba_order, - output_config, - update_ref, - keep_files, - get_mld, - get_mld_lim, - abs_tol, - get_ssnr, - get_odg, - get_enc_stats, - encoder_only, - decoder_only, - ) + # check if both REF and DUT .stats file exist + if ref_stats_file is None or not os.path.exists(ref_stats_file): + pytest.fail("REF .stats file does not exist or has not been generated!") + if dut_stats_file is None or not os.path.exists(dut_stats_file): + pytest.fail("DUT .stats file does not exist or has not been generated!") + + # compare ref and dut .stats files + enc_test_result, enc_test_result_msg = cmp_stats_files( + ref_stats_file, + dut_stats_file, + min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF, + min_enc_stats_diff=MIN_ENC_STATS_DIFF, + ) + + print("") + + # TODO: there is cleanup potential here: + # - move filename handling outside of the sba_enc/dec functions into start of this function + # - move file cleanup part out as well + # - move this comparison to the end and the one in sba_dec there as well + if encoder_only: + props = parse_properties( + enc_test_result_msg, enc_test_result != 0, props_to_record + ) + for k, v in props.items(): + record_property(k, v) + + # report compare result + if enc_test_result != 0: + pytest.fail(enc_test_result_msg) + + # if not encoder_only: + # sba_dec( + # record_property, + # props_to_record, + # dut_decoder_frontend, + # ref_decoder_frontend, + # reference_path, + # dut_base_path, + # tag, + # sampling_rate, + # bitrate, + # dtx, + # sid, + # max_bw, + # output_config, + # update_ref, + # gain_flag, + # keep_files, + # decoder_only, + # plc_pattern=plc_pattern, + # get_mld=get_mld, + # get_mld_lim=get_mld_lim, + # pca=pca, + # abs_tol=abs_tol, + # get_ssnr=get_ssnr, + # get_odg=get_odg, + # ) @pytest.mark.parametrize("bitrate", ivas_br_HOA3) @@ -403,6 +506,7 @@ def test_spar_hoa3_enc_system( dut_base_path, ref_encoder_frontend, ref_decoder_frontend, + br_switch_file_path, update_ref, keep_files, bitrate, @@ -417,45 +521,105 @@ def test_spar_hoa3_enc_system( get_enc_stats, ): sampling_rate = "48" + pca = False + max_bw = "FB" + sba_order = "+3" + output_config = "HOA3" + dtx = "0" + sid = 0 + gain_flag = -1 + cut_gain = "1.0" + plc_pattern = None + cut_testv = False if "ltv" in tag: - tag = f"ltv{fs}_HOA3" + tag = f"ltv{sampling_rate}_HOA3" elif "stv" in tag: - tag = tag + fs + "c" + tag = tag + sampling_rate + "c" else: assert 0 - max_bw = "FB" - sba_order = "+3" - output_config = "HOA3" + if not decoder_only: + ref_stats_file, dut_stats_file = sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + br_switch_file_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files=keep_files, + cut_gain=cut_gain, + cut_testv=cut_testv, + pca=pca, + plc_pattern=plc_pattern, + get_enc_stats=get_enc_stats, + ) + + if get_enc_stats: + print("Comparing encoder files") + print("=======================\n") - run_sba( - record_property, - props_to_record, - dut_encoder_frontend, - dut_decoder_frontend, - ref_encoder_frontend, - ref_decoder_frontend, - test_vector_path, - reference_path, - dut_base_path, - tag, - sampling_rate, - bitrate, - ivas_max_bw, - sba_order, - output_config, - update_ref, - keep_files, - get_mld, - get_mld_lim, - abs_tol, - get_ssnr, - get_odg, - get_enc_stats, - encoder_only, - decoder_only, - ) + # compare ref and dut .stats files + enc_test_result, enc_test_result_msg = cmp_stats_files( + ref_stats_file, + dut_stats_file, + min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF, + min_enc_stats_diff=MIN_ENC_STATS_DIFF, + ) + + print("") + + # TODO: there is cleanup potential here: + # - move filename handling outside of the sba_enc/dec functions into start of this function + # - move file cleanup part out as well + # - move this comparison to the end and the one in sba_dec there as well + if encoder_only: + props = parse_properties( + enc_test_result_msg, enc_test_result != 0, props_to_record + ) + for k, v in props.items(): + record_property(k, v) + + # report compare result + if enc_test_result != 0: + pytest.fail(enc_test_result_msg) + + # if not encoder_only: + # sba_dec( + # record_property, + # props_to_record, + # dut_decoder_frontend, + # ref_decoder_frontend, + # reference_path, + # dut_base_path, + # tag, + # sampling_rate, + # bitrate, + # dtx, + # sid, + # max_bw, + # output_config, + # update_ref, + # gain_flag, + # keep_files, + # decoder_only, + # plc_pattern=plc_pattern, + # get_mld=get_mld, + # get_mld_lim=get_mld_lim, + # pca=pca, + # abs_tol=abs_tol, + # get_ssnr=get_ssnr, + # get_odg=get_odg, + # ) @pytest.mark.parametrize("bitrate", ivas_br_FOA) @@ -472,6 +636,7 @@ def test_sba_enc_BWforce_system( dut_base_path, ref_encoder_frontend, ref_decoder_frontend, + br_switch_file_path, update_ref, keep_files, bitrate, @@ -487,6 +652,18 @@ def test_sba_enc_BWforce_system( get_odg, get_enc_stats, ): + sid = 0 + plc_pattern = None + pca = False + gain_flag = -1 + sba_order = "+1" + output_config = "FOA" + dtx = "0" + gain_flag = -1 + cut_gain = "1.0" + plc_pattern = None + cut_testv = False + if dtx == "1" and bitrate not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() @@ -494,71 +671,104 @@ def test_sba_enc_BWforce_system( pytest.skip() if bitrate == "sw_24k4_256k.bin": pytest.skip() - fs = sample_rate_bw_idx[0] - bw = sample_rate_bw_idx[1] + sampling_rate = sample_rate_bw_idx[0] + max_bw = sample_rate_bw_idx[1] if "ltv" in tag: - tag = f"ltv{fs}_FOA" + tag = f"ltv{sampling_rate}_FOA" cut_testv = False elif "stv" in tag: - tag = tag + fs + "c" + tag = tag + sampling_rate + "c" cut_testv = True else: assert 0 - gain_flag = -1 - sba_order = "+1" - output_config = "FOA" if not decoder_only: - # enc - sba_enc( + ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, test_vector_path, ref_encoder_frontend, reference_path, dut_base_path, - None, + br_switch_file_path, tag, - fs, - ivas_br, + sampling_rate, + bitrate, dtx, - None, - bw, + sid, + max_bw, sba_order, update_ref, gain_flag, - keep_files, + keep_files=keep_files, + cut_gain=cut_gain, cut_testv=cut_testv, + pca=pca, + plc_pattern=plc_pattern, + get_enc_stats=get_enc_stats, ) + + if update_ref == 0 and get_enc_stats: + print("Comparing encoder files") + print("=======================\n") - # dec - sba_dec( - record_property, - props_to_record, - dut_encoder_frontend, - dut_decoder_frontend, - ref_encoder_frontend, - ref_decoder_frontend, - test_vector_path, - reference_path, - dut_base_path, - tag, - sampling_rate, - bitrate, - ivas_max_bw, - sba_order, - output_config, - update_ref, - keep_files, - get_mld, - get_mld_lim, - abs_tol, - get_ssnr, - get_odg, - get_enc_stats, - encoder_only, - decoder_only, - dtx=dtx, - ) + # check if both REF and DUT .stats file exist + if ref_stats_file is None or not os.path.exists(ref_stats_file): + pytest.fail("REF .stats file does not exist or has not been generated!") + if dut_stats_file is None or not os.path.exists(dut_stats_file): + pytest.fail("DUT .stats file does not exist or has not been generated!") + + # compare ref and dut .stats files + enc_test_result, enc_test_result_msg = cmp_stats_files( + ref_stats_file, + dut_stats_file, + min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF, + min_enc_stats_diff=MIN_ENC_STATS_DIFF, + ) + + print("") + + # TODO: there is cleanup potential here: + # - move filename handling outside of the sba_enc/dec functions into start of this function + # - move file cleanup part out as well + # - move this comparison to the end and the one in sba_dec there as well + if encoder_only: + props = parse_properties( + enc_test_result_msg, enc_test_result != 0, props_to_record + ) + for k, v in props.items(): + record_property(k, v) + + # report compare result + if enc_test_result != 0: + pytest.fail(enc_test_result_msg) + + + # sba_dec( + # record_property, + # props_to_record, + # dut_decoder_frontend, + # ref_decoder_frontend, + # reference_path, + # dut_base_path, + # tag, + # sampling_rate, + # bitrate, + # dtx, + # sid, + # max_bw, + # output_config, + # update_ref, + # gain_flag, + # keep_files, + # decoder_only, + # plc_pattern=plc_pattern, + # get_mld=get_mld, + # get_mld_lim=get_mld_lim, + # pca=pca, + # abs_tol=abs_tol, + # get_ssnr=get_ssnr, + # get_odg=get_odg, + # ) @pytest.mark.parametrize("bitrate", ivas_br_plc) @@ -588,12 +798,20 @@ def test_sba_plc_system( gain_flag, get_mld, get_mld_lim, + encoder_only, + decoder_only, abs_tol, get_ssnr, get_odg, get_enc_stats ): sid = 0 + pca = False + max_bw = "FB" + sba_order = "+1" + cut_testv = True + output_config = "FOA" + if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() @@ -610,17 +828,14 @@ def test_sba_plc_system( if gain_flag == 1 and bitrate not in ["13200", "16400", "24400", "32000"]: pytest.skip() if "ltv" in tag: - tag = f"ltv{fs}_FOA" + tag = f"ltv{sampling_rate}_FOA" cut_testv = False elif "stv" in tag: - tag = tag + fs + "c" + tag = tag + sampling_rate + "c" cut_testv = True else: assert 0 - # added enc call - max_bw = "FB" - sba_order = "+1" if gain_flag == 1: cut_gain = "16.0" elif dtx == "1": @@ -628,65 +843,64 @@ def test_sba_plc_system( else: cut_gain = "1.0" - sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - br_switch_file_path, - tag, - fs, - ivas_br, - dtx, - SID, - max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_gain=cut_gain, - cut_testv=cut_testv, - plc_pattern=plc_pattern, - ) - - # dec - sba_dec( - record_property, - props_to_record, - dut_encoder_frontend, - dut_decoder_frontend, - ref_encoder_frontend, - ref_decoder_frontend, - test_vector_path, - reference_path, - dut_base_path, - tag, - sampling_rate, - bitrate, - ivas_max_bw, - sba_order, - output_config, - update_ref, - keep_files, - get_mld, - get_mld_lim, - abs_tol, - get_ssnr, - get_odg, - get_enc_stats, - encoder_only=False, - decoder_only=True, - sid=sid, - dtx=dtx, - gain_flag=gain_flag, - ) + if not decoder_only: + sba_enc( + dut_encoder_frontend, + test_vector_path, + ref_encoder_frontend, + reference_path, + dut_base_path, + br_switch_file_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + max_bw, + sba_order, + update_ref, + gain_flag, + keep_files=keep_files, + cut_gain=cut_gain, + cut_testv=cut_testv, + pca=pca, + plc_pattern=plc_pattern, + get_enc_stats=get_enc_stats, + ) + + if not encoder_only: + sba_dec( + record_property, + props_to_record, + dut_decoder_frontend, + ref_decoder_frontend, + reference_path, + dut_base_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + max_bw, + output_config, + update_ref, + gain_flag, + keep_files, + decoder_only, + plc_pattern=plc_pattern, + get_mld=get_mld, + get_mld_lim=get_mld_lim, + pca=pca, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + ) ######################################################### -# -------------------- test function -------------------- + def sba_enc( - encoder_frontend, + dut_encoder_frontend, test_vector_path, ref_encoder_frontend, reference_path, @@ -694,10 +908,10 @@ def sba_enc( br_switch_file_path, tag, sampling_rate, - ivas_br, + bitrate, dtx, - SID, - ivas_max_bw, + sid, + max_bw, sba_order, update_ref, gain_flag, @@ -706,8 +920,8 @@ def sba_enc( cut_testv=False, pca=False, plc_pattern=None, + get_enc_stats=False, ): - # ------------ run cmd ------------ dut_out_dir = f"{dut_base_path}/sba_bs/pkt" ref_out_dir = f"{reference_path}/sba_bs/pkt" @@ -719,12 +933,12 @@ def sba_enc( tag_in = tag # sampling rate to BW mapping bw_idx = dict_fsample_bw[sampling_rate] - if int(dict_bw_idx[ivas_max_bw]) < int(bw_idx): - tag = tag + dict_bw_tag[ivas_max_bw] + if int(dict_bw_idx[max_bw]) < int(bw_idx): + tag = tag + dict_bw_tag[max_bw] - tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}" - if ivas_br == "sw_24k4_256k.bin": - ivas_br = f"{br_switch_file_path}/sw_24k4_256k.bin" + tag_out = f"{tag}_ivasbr{bitrate[:-3]}k_DTX{dtx}" + if bitrate == "sw_24k4_256k.bin": + bitrate = f"{br_switch_file_path}/sw_24k4_256k.bin" # to avoid conflicting names in case of parallel test execution, differentiate all cases tag_ext = "" @@ -732,13 +946,13 @@ def sba_enc( tag_ext += "_pca" if gain_flag != -1: tag_ext += f"_Gain{gain_flag}" - if SID == 1: + if sid == 1: tag_ext += "_SID" if plc_pattern is not None: tag_ext += "_" + plc_pattern dut_pkt_file = f"{dut_out_dir}/{tag_out}{tag_ext}.192" ref_pkt_file = f"{ref_out_dir}/{tag_out}{tag_ext}.192" - if SID == 1: + if sid == 1: dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{tag_ext}_cut.192" ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{tag_ext}_cut.192" input_path = f"{test_vector_path}/{tag_in}{in_extension}" @@ -754,35 +968,42 @@ def sba_enc( input_path = cut_file + if get_enc_stats: + ref_stats_file = f"{ref_out_dir}/{tag_out}{tag_ext}.stats" + dut_stats_file = f"{dut_out_dir}/{tag_out}{tag_ext}.stats" + else: + ref_stats_file = None + dut_stats_file = None + if update_ref == 1: # call REF encoder ref_encoder_frontend.run( - ivas_br, + bitrate, sampling_rate, input_path, ref_pkt_file, sba_order=sba_order, - max_band=ivas_max_bw, + max_band=max_bw, pca=pca, dtx_mode=dtx_mode, - stats_file=stats_file_ref if get_enc_stats else None, + stats_file=ref_stats_file, ) if update_ref == 0: # call DUT encoder - encoder_frontend.run( - ivas_br, + dut_encoder_frontend.run( + bitrate, sampling_rate, input_path, dut_pkt_file, sba_order=sba_order, - max_band=ivas_max_bw, + max_band=max_bw, pca=pca, dtx_mode=dtx_mode, - stats_file=stats_file_dut if get_enc_stats else None, + stats_file=dut_stats_file, ) - if SID == 1: + if sid == 1: if update_ref == 1: with open(ref_pkt_file, "rb") as fp_in: with open(ref_pkt_file_cut, "wb") as fp_out: @@ -796,23 +1017,22 @@ def sba_enc( if not keep_files: os.remove(dut_pkt_file) - return stats_file_ref, stats_file_dut + return ref_stats_file, dut_stats_file def sba_dec( record_property, props_to_record, - cmp_result_msg, - decoder_frontend, + dut_decoder_frontend, ref_decoder_frontend, reference_path, dut_base_path, tag, sampling_rate, - ivas_br, + bitrate, dtx, - SID, - ivas_max_bw, + sid, + max_bw, output_config, update_ref, gain_flag, @@ -829,10 +1049,10 @@ def sba_dec( # -------- run cmd ------------ # sampling rate to BW mapping bw_idx = dict_fsample_bw[sampling_rate] - if int(dict_bw_idx[ivas_max_bw]) < int(bw_idx): - tag = tag + dict_bw_tag[ivas_max_bw] + if int(dict_bw_idx[max_bw]) < int(bw_idx): + tag = tag + dict_bw_tag[max_bw] - tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}" + tag_out = f"{tag}_ivasbr{bitrate[:-3]}k_DTX{dtx}" # to avoid conflicting names in case of parallel test execution, differentiate all cases tag_ext = "" @@ -840,11 +1060,11 @@ def sba_dec( tag_ext += "_pca" if gain_flag != -1: tag_ext += f"_Gain{gain_flag}" - if SID == 1: + if sid == 1: tag_ext += "_SID" if plc_pattern is not None: tag_ext += "_" + plc_pattern - if SID == 1: + if sid == 1: tag_ext += "_cut" dut_out_dir = f"{dut_base_path}/sba_bs/raw" @@ -880,7 +1100,7 @@ def sba_dec( dut_in_pkt = ref_in_pkt # call DUT decoder - decoder_frontend.run( + dut_decoder_frontend.run( output_config, sampling_rate, dut_in_pkt, @@ -888,12 +1108,12 @@ def sba_dec( plc_file=plc_file, ) - fs = int(sampling_rate) * 1000 + sampling_rate_Hz = int(sampling_rate) * 1000 cmp_result, reason = cmp_pcm( dut_out_raw, ref_out_raw, output_config, - fs, + sampling_rate_Hz, get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, @@ -901,7 +1121,7 @@ def sba_dec( get_odg=get_odg, ) - text_to_parse = cmp_result_msg + reason + text_to_parse = reason props = parse_properties(text_to_parse, cmp_result != 0, props_to_record) for k, v in props.items(): record_property(k, v) @@ -917,121 +1137,3 @@ def sba_dec( os.remove(dut_in_pkt) -def run_sba( - record_property, - props_to_record, - dut_encoder_frontend, - dut_decoder_frontend, - ref_encoder_frontend, - ref_decoder_frontend, - test_vector_path, - reference_path, - dut_base_path, - tag, - sampling_rate, - bitrate, - ivas_max_bw, - sba_order, - output_config, - update_ref, - keep_files, - get_mld, - get_mld_lim, - abs_tol, - get_ssnr, - get_odg, - get_enc_stats, - encoder_only, - decoder_only, - dtx="0", - sid=0, - br_switch_file_path=None, - gain_flag=-1, - cut_gain="1.0", - create_dutenc=False, - cut_testv=False, - pca=False, -): - cmp_result_msg = "" - if not decoder_only: - stats_file_ref, stats_file_dut = sba_enc( - dut_encoder_frontend, - test_vector_path, - ref_encoder_frontend, - reference_path, - dut_base_path, - br_switch_file_path, - tag, - sampling_rate, - bitrate, - dtx, - sid, - ivas_max_bw, - sba_order, - update_ref, - gain_flag, - keep_files, - cut_gain, - create_dutenc, - cut_testv, - pca, - get_enc_stats, - ) - - if get_enc_stats: - print("Comparing encoder files") - print("=======================\n") - - # compare ref and dut .stats files - enc_test_result, enc_test_result_msg = cmp_stats_files( - stats_file_ref, - stats_file_dut, - min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF, - min_enc_stats_diff=MIN_ENC_STATS_DIFF, - ) - - print("") - cmp_result_msg += enc_test_result_msg - - # TODO: there is cleanup potential here: - # - move filename handling outside of the sba_enc/dec functions into start of this function - # - move file cleanup part out as well - # - move this comparison to the end and the one in sba_dec there as well - if encoder_only: - props = parse_properties( - cmp_result_msg, enc_test_result != 0, props_to_record - ) - for k, v in props.items(): - record_property(k, v) - - # report compare result - if enc_test_result != 0: - pytest.fail(cmp_result_msg) - - if not encoder_only: - sba_dec( - record_property, - props_to_record, - cmp_result_msg, - dut_decoder_frontend, - ref_decoder_frontend, - reference_path, - dut_base_path, - tag, - sampling_rate, - bitrate, - dtx, - sid, - ivas_max_bw, - output_config, - update_ref, - gain_flag, - keep_files, - decoder_only, - get_mld=get_mld, - get_mld_lim=get_mld_lim, - pca=pca, - abs_tol=abs_tol, - get_ssnr=get_ssnr, - get_odg=get_odg, - ) -- GitLab From 3f04f6500b76f20dcf9964269b5ab9cb20b85a13 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 16 Sep 2024 12:49:46 +0200 Subject: [PATCH 19/28] add sba_dec() calls --- tests/codec_be_on_mr_nonselection/test_sba.py | 163 +++++++++--------- 1 file changed, 81 insertions(+), 82 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 04c3fb51e8..2122bf13ea 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -465,33 +465,33 @@ def test_spar_hoa2_enc_system( if enc_test_result != 0: pytest.fail(enc_test_result_msg) - # if not encoder_only: - # sba_dec( - # record_property, - # props_to_record, - # dut_decoder_frontend, - # ref_decoder_frontend, - # reference_path, - # dut_base_path, - # tag, - # sampling_rate, - # bitrate, - # dtx, - # sid, - # max_bw, - # output_config, - # update_ref, - # gain_flag, - # keep_files, - # decoder_only, - # plc_pattern=plc_pattern, - # get_mld=get_mld, - # get_mld_lim=get_mld_lim, - # pca=pca, - # abs_tol=abs_tol, - # get_ssnr=get_ssnr, - # get_odg=get_odg, - # ) + if not encoder_only: + sba_dec( + record_property, + props_to_record, + dut_decoder_frontend, + ref_decoder_frontend, + reference_path, + dut_base_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + max_bw, + output_config, + update_ref, + gain_flag, + keep_files, + decoder_only, + plc_pattern=plc_pattern, + get_mld=get_mld, + get_mld_lim=get_mld_lim, + pca=pca, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + ) @pytest.mark.parametrize("bitrate", ivas_br_HOA3) @@ -593,33 +593,33 @@ def test_spar_hoa3_enc_system( if enc_test_result != 0: pytest.fail(enc_test_result_msg) - # if not encoder_only: - # sba_dec( - # record_property, - # props_to_record, - # dut_decoder_frontend, - # ref_decoder_frontend, - # reference_path, - # dut_base_path, - # tag, - # sampling_rate, - # bitrate, - # dtx, - # sid, - # max_bw, - # output_config, - # update_ref, - # gain_flag, - # keep_files, - # decoder_only, - # plc_pattern=plc_pattern, - # get_mld=get_mld, - # get_mld_lim=get_mld_lim, - # pca=pca, - # abs_tol=abs_tol, - # get_ssnr=get_ssnr, - # get_odg=get_odg, - # ) + if not encoder_only: + sba_dec( + record_property, + props_to_record, + dut_decoder_frontend, + ref_decoder_frontend, + reference_path, + dut_base_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + max_bw, + output_config, + update_ref, + gain_flag, + keep_files, + decoder_only, + plc_pattern=plc_pattern, + get_mld=get_mld, + get_mld_lim=get_mld_lim, + pca=pca, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + ) @pytest.mark.parametrize("bitrate", ivas_br_FOA) @@ -742,33 +742,33 @@ def test_sba_enc_BWforce_system( if enc_test_result != 0: pytest.fail(enc_test_result_msg) - - # sba_dec( - # record_property, - # props_to_record, - # dut_decoder_frontend, - # ref_decoder_frontend, - # reference_path, - # dut_base_path, - # tag, - # sampling_rate, - # bitrate, - # dtx, - # sid, - # max_bw, - # output_config, - # update_ref, - # gain_flag, - # keep_files, - # decoder_only, - # plc_pattern=plc_pattern, - # get_mld=get_mld, - # get_mld_lim=get_mld_lim, - # pca=pca, - # abs_tol=abs_tol, - # get_ssnr=get_ssnr, - # get_odg=get_odg, - # ) + if not encoder_only: + sba_dec( + record_property, + props_to_record, + dut_decoder_frontend, + ref_decoder_frontend, + reference_path, + dut_base_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + max_bw, + output_config, + update_ref, + gain_flag, + keep_files, + decoder_only, + plc_pattern=plc_pattern, + get_mld=get_mld, + get_mld_lim=get_mld_lim, + pca=pca, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + ) @pytest.mark.parametrize("bitrate", ivas_br_plc) @@ -1046,7 +1046,6 @@ def sba_dec( get_ssnr=False, get_odg=False, ): - # -------- run cmd ------------ # sampling rate to BW mapping bw_idx = dict_fsample_bw[sampling_rate] if int(dict_bw_idx[max_bw]) < int(bw_idx): -- GitLab From 9dbe414cd75de9e758a0ff9d94ae22f216fc5fe0 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 16 Sep 2024 15:04:13 +0200 Subject: [PATCH 20/28] add sba_dec() calls --- tests/codec_be_on_mr_nonselection/test_sba.py | 106 +++++++++--------- 1 file changed, 54 insertions(+), 52 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 2122bf13ea..651dd76cfa 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -164,32 +164,33 @@ def test_pca_enc( ) - # sba_dec( - # record_property, - # props_to_record, - # dut_decoder_frontend, - # ref_decoder_frontend, - # reference_path, - # dut_base_path, - # tag, - # sampling_rate, - # bitrate, - # dtx, - # sid, - # max_bw, - # output_config, - # update_ref, - # gain_flag, - # keep_files, - # decoder_only, - # plc_pattern=plc_pattern, - # get_mld=get_mld, - # get_mld_lim=get_mld_lim, - # pca=pca, - # abs_tol=abs_tol, - # get_ssnr=get_ssnr, - # get_odg=get_odg, - # ) + if not encoder_only: + sba_dec( + record_property, + props_to_record, + dut_decoder_frontend, + ref_decoder_frontend, + reference_path, + dut_base_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + max_bw, + output_config, + update_ref, + gain_flag, + keep_files, + decoder_only, + plc_pattern=plc_pattern, + get_mld=get_mld, + get_mld_lim=get_mld_lim, + pca=pca, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + ) @pytest.mark.parametrize("bitrate", ivas_br_FOA) @@ -332,32 +333,33 @@ def test_sba_enc_system( pytest.fail(enc_test_result_msg) - # sba_dec( - # record_property, - # props_to_record, - # dut_decoder_frontend, - # ref_decoder_frontend, - # reference_path, - # dut_base_path, - # tag, - # sampling_rate, - # bitrate, - # dtx, - # sid, - # max_bw, - # output_config, - # update_ref, - # gain_flag, - # keep_files, - # decoder_only, - # plc_pattern=plc_pattern, - # get_mld=get_mld, - # get_mld_lim=get_mld_lim, - # pca=pca, - # abs_tol=abs_tol, - # get_ssnr=get_ssnr, - # get_odg=get_odg, - # ) + if not encoder_only: + sba_dec( + record_property, + props_to_record, + dut_decoder_frontend, + ref_decoder_frontend, + reference_path, + dut_base_path, + tag, + sampling_rate, + bitrate, + dtx, + sid, + max_bw, + output_config, + update_ref, + gain_flag, + keep_files, + decoder_only, + plc_pattern=plc_pattern, + get_mld=get_mld, + get_mld_lim=get_mld_lim, + pca=pca, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + ) @pytest.mark.parametrize("bitrate", ivas_br_HOA2) -- GitLab From 9eaa3a96b8ecb4dea235f972323cda6f8089deb0 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 16 Sep 2024 16:50:54 +0200 Subject: [PATCH 21/28] fix BWforce tests --- tests/codec_be_on_mr_nonselection/test_sba.py | 130 ++++++++++-------- 1 file changed, 69 insertions(+), 61 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 651dd76cfa..8f3b85e9ce 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -660,21 +660,20 @@ def test_sba_enc_BWforce_system( gain_flag = -1 sba_order = "+1" output_config = "FOA" - dtx = "0" - gain_flag = -1 cut_gain = "1.0" - plc_pattern = None cut_testv = False + sampling_rate = sample_rate_bw_idx[0] + max_bw = sample_rate_bw_idx[1] if dtx == "1" and bitrate not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() + if bitrate == "13200" or bitrate == "16400": pytest.skip() - if bitrate == "sw_24k4_256k.bin": + elif bitrate == "sw_24k4_256k.bin": pytest.skip() - sampling_rate = sample_rate_bw_idx[0] - max_bw = sample_rate_bw_idx[1] + if "ltv" in tag: tag = f"ltv{sampling_rate}_FOA" cut_testv = False @@ -817,18 +816,16 @@ def test_sba_plc_system( if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() + if bitrate == "13200" or bitrate == "16400": - if ( - dtx == "1" - and gain_flag == 0 - and sampling_rate != "16" - and plc_pattern == "PLperc12mblen5" - ): + if dtx == "1" and gain_flag == 0 and sampling_rate != "16" and plc_pattern == "PLperc12mblen5": sid = 1 else: pytest.skip() + if gain_flag == 1 and bitrate not in ["13200", "16400", "24400", "32000"]: pytest.skip() + if "ltv" in tag: tag = f"ltv{sampling_rate}_FOA" cut_testv = False @@ -930,49 +927,47 @@ def sba_enc( check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) - in_extension = ".wav" - - tag_in = tag + tag_out = tag + # sampling rate to BW mapping bw_idx = dict_fsample_bw[sampling_rate] if int(dict_bw_idx[max_bw]) < int(bw_idx): - tag = tag + dict_bw_tag[max_bw] + tag_out += dict_bw_tag[max_bw] - tag_out = f"{tag}_ivasbr{bitrate[:-3]}k_DTX{dtx}" + tag_out += f"_ivasbr{bitrate[:-3]}k_DTX{dtx}" if bitrate == "sw_24k4_256k.bin": bitrate = f"{br_switch_file_path}/sw_24k4_256k.bin" # to avoid conflicting names in case of parallel test execution, differentiate all cases - tag_ext = "" if pca: - tag_ext += "_pca" + tag_out += "_pca" if gain_flag != -1: - tag_ext += f"_Gain{gain_flag}" + tag_out += f"_Gain{gain_flag}" if sid == 1: - tag_ext += "_SID" + tag_out += "_SID" if plc_pattern is not None: - tag_ext += "_" + plc_pattern - dut_pkt_file = f"{dut_out_dir}/{tag_out}{tag_ext}.192" - ref_pkt_file = f"{ref_out_dir}/{tag_out}{tag_ext}.192" + tag_out += "_" + plc_pattern + dut_pkt_file = f"{dut_out_dir}/{tag_out}.192" + ref_pkt_file = f"{ref_out_dir}/{tag_out}.192" if sid == 1: - dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}{tag_ext}_cut.192" - ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}{tag_ext}_cut.192" - input_path = f"{test_vector_path}/{tag_in}{in_extension}" + dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}_cut.192" + ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}_cut.192" + input_path = f"{test_vector_path}/{tag}.wav" dtx_mode = dtx == "1" if cut_testv: # use shortened and potentially gain adjusted input PCM file - create if not present # cut input PCM file: currently with mostly fixed (i.e. not test dependant) values if cut_gain == "1.0": - cut_file = f"{test_vector_path}/{tag_in}_cut{in_extension}" + cut_file = f"{test_vector_path}/{tag}_cut.wav" else: - cut_file = f"{test_vector_path}/{tag_in}_cut_{cut_gain}{in_extension}" + cut_file = f"{test_vector_path}/{tag}_cut_{cut_gain}.wav" input_path = cut_file if get_enc_stats: - ref_stats_file = f"{ref_out_dir}/{tag_out}{tag_ext}.stats" - dut_stats_file = f"{dut_out_dir}/{tag_out}{tag_ext}.stats" + ref_stats_file = f"{ref_out_dir}/{tag_out}.stats" + dut_stats_file = f"{dut_out_dir}/{tag_out}.stats" else: ref_stats_file = None dut_stats_file = None @@ -1048,71 +1043,84 @@ def sba_dec( get_ssnr=False, get_odg=False, ): + + dut_out_dir = f"{dut_base_path}/sba_bs/raw" + ref_out_dir = f"{reference_path}/sba_bs/raw" + + check_and_makedir(dut_out_dir) + check_and_makedir(ref_out_dir) + + tag_in = tag + tag_out = tag + # sampling rate to BW mapping bw_idx = dict_fsample_bw[sampling_rate] if int(dict_bw_idx[max_bw]) < int(bw_idx): - tag = tag + dict_bw_tag[max_bw] - - tag_out = f"{tag}_ivasbr{bitrate[:-3]}k_DTX{dtx}" - + tag_out += dict_bw_tag[max_bw] + + tag_out += f"_ivasbr{bitrate[:-3]}k_DTX{dtx}" + # to avoid conflicting names in case of parallel test execution, differentiate all cases - tag_ext = "" if pca: - tag_ext += "_pca" + tag_out += "_pca" if gain_flag != -1: - tag_ext += f"_Gain{gain_flag}" + tag_out += f"_Gain{gain_flag}" if sid == 1: - tag_ext += "_SID" + tag_out += "_SID" if plc_pattern is not None: - tag_ext += "_" + plc_pattern + tag_out += "_" + plc_pattern if sid == 1: - tag_ext += "_cut" - - dut_out_dir = f"{dut_base_path}/sba_bs/raw" - ref_out_dir = f"{reference_path}/sba_bs/raw" + tag_out += "_cut" + + # to avoid conflicting names in case of parallel test execution, differentiate all cases + if pca: + tag_out += "_pca" + if gain_flag != -1: + tag_out += f"_Gain{gain_flag}" + if sid == 1: + tag_out += "_SID" + if plc_pattern is not None: + tag_out += "_" + plc_pattern - dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{tag_ext}.192" - ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{tag_ext}.192" + dut_pkt_file = f"{dut_base_path}/sba_bs/pkt/{tag_out}.192" + ref_pkt_file = f"{reference_path}/sba_bs/pkt/{tag_out}.192" plc_file = None if plc_pattern is not None: plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" - dut_out_raw = f"{dut_out_dir}/{tag_out}{tag_ext}.wav" - ref_out_raw = f"{ref_out_dir}/{tag_out}{tag_ext}.wav" - - check_and_makedir(dut_out_dir) - check_and_makedir(ref_out_dir) + dut_out_file = f"{dut_out_dir}/{tag_out}.wav" + ref_out_file = f"{ref_out_dir}/{tag_out}.wav" if update_ref == 1: # call REF decoder ref_decoder_frontend.run( output_config, sampling_rate, - ref_in_pkt, - ref_out_raw, + ref_pkt_file, + ref_out_file, plc_file=plc_file, ) if update_ref == 0: if plc_file is not None: - dut_in_pkt = dut_in_pkt + dut_pkt_file = dut_pkt_file elif decoder_only: - dut_in_pkt = ref_in_pkt + dut_pkt_file = ref_pkt_file # call DUT decoder dut_decoder_frontend.run( output_config, sampling_rate, - dut_in_pkt, - dut_out_raw, + dut_pkt_file, + dut_out_file, plc_file=plc_file, ) sampling_rate_Hz = int(sampling_rate) * 1000 cmp_result, reason = cmp_pcm( - dut_out_raw, - ref_out_raw, + dut_out_file, + ref_out_file, output_config, sampling_rate_Hz, get_mld=get_mld, @@ -1133,8 +1141,8 @@ def sba_dec( # remove DUT output files when test result is OK (to save disk space) if not keep_files: - os.remove(dut_out_raw) + os.remove(dut_out_file) if not decoder_only and plc_file is None: - os.remove(dut_in_pkt) + os.remove(dut_pkt_file) -- GitLab From 45517869695fe8cd1f794adb74280ca0dae98e94 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 16 Sep 2024 17:09:09 +0200 Subject: [PATCH 22/28] fix SBA PLC dec tests --- tests/codec_be_on_mr_nonselection/test_sba.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 8f3b85e9ce..7e9ead51c1 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1071,16 +1071,6 @@ def sba_dec( tag_out += "_" + plc_pattern if sid == 1: tag_out += "_cut" - - # to avoid conflicting names in case of parallel test execution, differentiate all cases - if pca: - tag_out += "_pca" - if gain_flag != -1: - tag_out += f"_Gain{gain_flag}" - if sid == 1: - tag_out += "_SID" - if plc_pattern is not None: - tag_out += "_" + plc_pattern dut_pkt_file = f"{dut_base_path}/sba_bs/pkt/{tag_out}.192" ref_pkt_file = f"{reference_path}/sba_bs/pkt/{tag_out}.192" -- GitLab From 116d27af5e91f6fe37084d3c99ded279ba941303 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 16 Sep 2024 18:43:55 +0200 Subject: [PATCH 23/28] run SBA PLC decoder with DUT .pkt file --- tests/codec_be_on_mr_nonselection/test_sba.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 7e9ead51c1..6da1adf482 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1093,10 +1093,10 @@ def sba_dec( ) if update_ref == 0: - if plc_file is not None: - dut_pkt_file = dut_pkt_file - elif decoder_only: - dut_pkt_file = ref_pkt_file + # if plc_file is not None: + # dut_pkt_file = dut_pkt_file + # elif decoder_only: + # dut_pkt_file = ref_pkt_file # call DUT decoder dut_decoder_frontend.run( -- GitLab From f4a8bde82d8d54f0900ed83157f9bebf89b26535 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 16 Sep 2024 20:00:49 +0200 Subject: [PATCH 24/28] try to fix conflicting names of some .192 files --- tests/codec_be_on_mr_nonselection/test_sba.py | 64 ++++++++++--------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 6da1adf482..13be91368f 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -134,7 +134,7 @@ def test_pca_enc( cut_testv = False elif "stv" in tag: tag = tag + sampling_rate + "c" - cut_testv=True + cut_testv = True else: assert 0 @@ -921,11 +921,15 @@ def sba_enc( plc_pattern=None, get_enc_stats=False, ): - dut_out_dir = f"{dut_base_path}/sba_bs/pkt" - ref_out_dir = f"{reference_path}/sba_bs/pkt" - check_and_makedir(dut_out_dir) - check_and_makedir(ref_out_dir) + input_path = f"{test_vector_path}/{tag}.wav" + dtx_mode = dtx == "1" + + dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" + ref_pkt_dir = f"{reference_path}/sba_bs/pkt" + + check_and_makedir(dut_pkt_dir) + check_and_makedir(ref_pkt_dir) tag_out = tag @@ -947,13 +951,9 @@ def sba_enc( tag_out += "_SID" if plc_pattern is not None: tag_out += "_" + plc_pattern - dut_pkt_file = f"{dut_out_dir}/{tag_out}.192" - ref_pkt_file = f"{ref_out_dir}/{tag_out}.192" - if sid == 1: - dut_pkt_file_cut = f"{dut_out_dir}/{tag_out}_cut.192" - ref_pkt_file_cut = f"{ref_out_dir}/{tag_out}_cut.192" - input_path = f"{test_vector_path}/{tag}.wav" - dtx_mode = dtx == "1" + + dut_pkt_file = f"{dut_pkt_dir}/{tag_out}.192" + ref_pkt_file = f"{ref_pkt_dir}/{tag_out}.192" if cut_testv: # use shortened and potentially gain adjusted input PCM file - create if not present @@ -966,8 +966,8 @@ def sba_enc( input_path = cut_file if get_enc_stats: - ref_stats_file = f"{ref_out_dir}/{tag_out}.stats" - dut_stats_file = f"{dut_out_dir}/{tag_out}.stats" + ref_stats_file = f"{ref_pkt_dir}/{tag_out}.stats" + dut_stats_file = f"{dut_pkt_dir}/{tag_out}.stats" else: ref_stats_file = None dut_stats_file = None @@ -1001,6 +1001,10 @@ def sba_enc( ) if sid == 1: + # cut .pkt files such that they start with SID frame + dut_pkt_file_cut = f"{dut_pkt_dir}/{tag_out}_cut.192" + ref_pkt_file_cut = f"{ref_pkt_dir}/{tag_out}_cut.192" + if update_ref == 1: with open(ref_pkt_file, "rb") as fp_in: with open(ref_pkt_file_cut, "wb") as fp_out: @@ -1043,9 +1047,10 @@ def sba_dec( get_ssnr=False, get_odg=False, ): - - dut_out_dir = f"{dut_base_path}/sba_bs/raw" - ref_out_dir = f"{reference_path}/sba_bs/raw" + dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" + ref_pkt_dir = f"{reference_path}/sba_bs/pkt" + dut_out_dir = f"{dut_base_path}/sba_bs/syn" + ref_out_dir = f"{reference_path}/sba_bs/syn" check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) @@ -1069,18 +1074,20 @@ def sba_dec( tag_out += "_SID" if plc_pattern is not None: tag_out += "_" + plc_pattern - if sid == 1: - tag_out += "_cut" - - dut_pkt_file = f"{dut_base_path}/sba_bs/pkt/{tag_out}.192" - ref_pkt_file = f"{reference_path}/sba_bs/pkt/{tag_out}.192" - + plc_file = None if plc_pattern is not None: plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" - + + if sid == 1: + dut_pkt_file = f"{dut_pkt_dir}/{tag_out}_cut.192" + ref_pkt_file = f"{ref_pkt_dir}/{tag_out}_cut.192" + else: + dut_pkt_file = f"{dut_pkt_dir}/{tag_out}.192" + ref_pkt_file = f"{ref_pkt_dir}/{tag_out}.192" + dut_out_file = f"{dut_out_dir}/{tag_out}.wav" - ref_out_file = f"{ref_out_dir}/{tag_out}.wav" + ref_out_file = f"{ref_out_dir}/{tag_out}.wav" if update_ref == 1: # call REF decoder @@ -1130,9 +1137,8 @@ def sba_dec( pytest.fail(text_to_parse) # remove DUT output files when test result is OK (to save disk space) - if not keep_files: - os.remove(dut_out_file) - if not decoder_only and plc_file is None: - os.remove(dut_pkt_file) + # if not keep_files: + # os.remove(dut_out_file) + # os.remove(dut_pkt_file) -- GitLab From 3645562c5fc0d81f25bbd65dfbe54b1c42d08561 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 16 Sep 2024 20:43:21 +0200 Subject: [PATCH 25/28] fix --decoder_only SBS tests --- tests/codec_be_on_mr_nonselection/test_sba.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 13be91368f..25c0130b58 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1100,10 +1100,9 @@ def sba_dec( ) if update_ref == 0: - # if plc_file is not None: - # dut_pkt_file = dut_pkt_file - # elif decoder_only: - # dut_pkt_file = ref_pkt_file + if decoder_only: + # DUT .pkt file has not been generated -> use REF .pkt file instead + dut_pkt_file = ref_pkt_file # call DUT decoder dut_decoder_frontend.run( -- GitLab From b9bf9387735f84447e2d3c1442a36b480242b4b7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 17 Sep 2024 10:05:26 +0200 Subject: [PATCH 26/28] run formatter --- tests/cmp_stats_files.py | 15 +++- .../test_param_file.py | 15 ++-- tests/codec_be_on_mr_nonselection/test_sba.py | 90 ++++++++++--------- tests/conftest.py | 4 +- 4 files changed, 69 insertions(+), 55 deletions(-) diff --git a/tests/cmp_stats_files.py b/tests/cmp_stats_files.py index dfebbd28a5..bc191c87ee 100644 --- a/tests/cmp_stats_files.py +++ b/tests/cmp_stats_files.py @@ -126,14 +126,21 @@ def cmp_stats_files( if __name__ == "__main__": parser = argparse.ArgumentParser() - + parser.add_argument("ref_stats_file", type=str) parser.add_argument("dut_stats_file", type=str) - parser.add_argument("--min_enc_file_length_diff", type=float, default=0.1, dest="min_enc_file_length_diff") - parser.add_argument("--min_enc_stats_diff", type=float, default=0.1, dest="min_enc_stats_diff") + parser.add_argument( + "--min_enc_file_length_diff", + type=float, + default=0.1, + dest="min_enc_file_length_diff", + ) + parser.add_argument( + "--min_enc_stats_diff", type=float, default=0.1, dest="min_enc_stats_diff" + ) args = parser.parse_args() enc_test_result, enc_test_result_msg = cmp_stats_files(**vars(args)) print(enc_test_result_msg) - + sys.exit(enc_test_result) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index eaec8792cd..9a11b4d6d4 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -197,7 +197,7 @@ def test_param_file_tests( cmp_result_msg = "" enc_test_result = 0 - + if not decoder_only: encode( dut_encoder_frontend, @@ -242,9 +242,9 @@ def test_param_file_tests( # remove DUT stats file when test result is OK (to save disk space) if not keep_files: os.remove(dut_stats_file) - + if encoder_only: - return # don't proceed with the decoder if user specified --encoder_only on the command line + return # don't proceed with the decoder if user specified --encoder_only on the command line # check for networkSimulator_g192 command line if sim_opts != "": @@ -278,7 +278,7 @@ def test_param_file_tests( ) # check for eid-xor command line - + if eid_opts != "": eid_split = eid_opts.split() assert len(eid_split) >= 3, "eid-xor expects at least 3 parameters" @@ -295,7 +295,7 @@ def test_param_file_tests( # -> construct netsim output file name eid_xor_outfile = f"{testv_base}_{tag_str}.fer.192" eid_split[-1] = eid_xor_outfile - + error_insertion( reference_path, dut_base_path, @@ -317,7 +317,10 @@ def test_param_file_tests( # remove leading "../" dec_split = [x[3:] if x.startswith("../") else x for x in dec_split] # convert "scripts/" paths into absolute ones - dec_split = [str(SCRIPTS_DIR.joinpath(x[8:])) if x.startswith("scripts/") else x for x in dec_split] + dec_split = [ + str(SCRIPTS_DIR.joinpath(x[8:])) if x.startswith("scripts/") else x + for x in dec_split + ] output_file = dec_split.pop() bitstream_file_dec = dec_split.pop() diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 25c0130b58..ee14c09c22 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -136,7 +136,7 @@ def test_pca_enc( tag = tag + sampling_rate + "c" cut_testv = True else: - assert 0 + assert 0 if not decoder_only: sba_enc( @@ -162,7 +162,6 @@ def test_pca_enc( plc_pattern=plc_pattern, get_enc_stats=get_enc_stats, ) - if not encoder_only: sba_dec( @@ -191,7 +190,7 @@ def test_pca_enc( get_ssnr=get_ssnr, get_odg=get_odg, ) - + @pytest.mark.parametrize("bitrate", ivas_br_FOA) @pytest.mark.parametrize("dtx", dtx_set) @@ -236,7 +235,7 @@ def test_sba_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = True - + if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() @@ -263,7 +262,7 @@ def test_sba_enc_system( tag = tag + sampling_rate + "c" cut_testv = True else: - assert 0 + assert 0 if gain_flag == 1: cut_gain = "16.0" @@ -296,7 +295,7 @@ def test_sba_enc_system( plc_pattern=plc_pattern, get_enc_stats=get_enc_stats, ) - + if update_ref == 0 and get_enc_stats: print("Comparing encoder files") print("=======================\n") @@ -331,7 +330,6 @@ def test_sba_enc_system( # report compare result if enc_test_result != 0: pytest.fail(enc_test_result_msg) - if not encoder_only: sba_dec( @@ -399,14 +397,14 @@ def test_spar_hoa2_enc_system( cut_gain = "1.0" plc_pattern = None cut_testv = False - + if "ltv" in tag: tag = f"ltv{sampling_rate}_HOA2" elif "stv" in tag: tag = tag + sampling_rate + "c" else: - assert 0 - + assert 0 + if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, @@ -539,7 +537,7 @@ def test_spar_hoa3_enc_system( elif "stv" in tag: tag = tag + sampling_rate + "c" else: - assert 0 + assert 0 if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -664,16 +662,16 @@ def test_sba_enc_BWforce_system( cut_testv = False sampling_rate = sample_rate_bw_idx[0] max_bw = sample_rate_bw_idx[1] - + if dtx == "1" and bitrate not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() - + if bitrate == "13200" or bitrate == "16400": pytest.skip() elif bitrate == "sw_24k4_256k.bin": pytest.skip() - + if "ltv" in tag: tag = f"ltv{sampling_rate}_FOA" cut_testv = False @@ -681,7 +679,7 @@ def test_sba_enc_BWforce_system( tag = tag + sampling_rate + "c" cut_testv = True else: - assert 0 + assert 0 if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -707,7 +705,7 @@ def test_sba_enc_BWforce_system( plc_pattern=plc_pattern, get_enc_stats=get_enc_stats, ) - + if update_ref == 0 and get_enc_stats: print("Comparing encoder files") print("=======================\n") @@ -742,7 +740,7 @@ def test_sba_enc_BWforce_system( # report compare result if enc_test_result != 0: pytest.fail(enc_test_result_msg) - + if not encoder_only: sba_dec( record_property, @@ -804,7 +802,7 @@ def test_sba_plc_system( abs_tol, get_ssnr, get_odg, - get_enc_stats + get_enc_stats, ): sid = 0 pca = False @@ -812,20 +810,25 @@ def test_sba_plc_system( sba_order = "+1" cut_testv = True output_config = "FOA" - + if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved pytest.skip() - + if bitrate == "13200" or bitrate == "16400": - if dtx == "1" and gain_flag == 0 and sampling_rate != "16" and plc_pattern == "PLperc12mblen5": + if ( + dtx == "1" + and gain_flag == 0 + and sampling_rate != "16" + and plc_pattern == "PLperc12mblen5" + ): sid = 1 else: pytest.skip() - + if gain_flag == 1 and bitrate not in ["13200", "16400", "24400", "32000"]: pytest.skip() - + if "ltv" in tag: tag = f"ltv{sampling_rate}_FOA" cut_testv = False @@ -833,7 +836,7 @@ def test_sba_plc_system( tag = tag + sampling_rate + "c" cut_testv = True else: - assert 0 + assert 0 if gain_flag == 1: cut_gain = "16.0" @@ -898,6 +901,7 @@ def test_sba_plc_system( ######################################################### + def sba_enc( dut_encoder_frontend, test_vector_path, @@ -932,7 +936,7 @@ def sba_enc( check_and_makedir(ref_pkt_dir) tag_out = tag - + # sampling rate to BW mapping bw_idx = dict_fsample_bw[sampling_rate] if int(dict_bw_idx[max_bw]) < int(bw_idx): @@ -951,9 +955,9 @@ def sba_enc( tag_out += "_SID" if plc_pattern is not None: tag_out += "_" + plc_pattern - + dut_pkt_file = f"{dut_pkt_dir}/{tag_out}.192" - ref_pkt_file = f"{ref_pkt_dir}/{tag_out}.192" + ref_pkt_file = f"{ref_pkt_dir}/{tag_out}.192" if cut_testv: # use shortened and potentially gain adjusted input PCM file - create if not present @@ -971,7 +975,7 @@ def sba_enc( else: ref_stats_file = None dut_stats_file = None - + if update_ref == 1: # call REF encoder ref_encoder_frontend.run( @@ -1001,10 +1005,10 @@ def sba_enc( ) if sid == 1: - # cut .pkt files such that they start with SID frame + # cut .pkt files such that they start with SID frame dut_pkt_file_cut = f"{dut_pkt_dir}/{tag_out}_cut.192" ref_pkt_file_cut = f"{ref_pkt_dir}/{tag_out}_cut.192" - + if update_ref == 1: with open(ref_pkt_file, "rb") as fp_in: with open(ref_pkt_file_cut, "wb") as fp_out: @@ -1051,20 +1055,20 @@ def sba_dec( ref_pkt_dir = f"{reference_path}/sba_bs/pkt" dut_out_dir = f"{dut_base_path}/sba_bs/syn" ref_out_dir = f"{reference_path}/sba_bs/syn" - + check_and_makedir(dut_out_dir) check_and_makedir(ref_out_dir) tag_in = tag tag_out = tag - + # sampling rate to BW mapping bw_idx = dict_fsample_bw[sampling_rate] if int(dict_bw_idx[max_bw]) < int(bw_idx): tag_out += dict_bw_tag[max_bw] - + tag_out += f"_ivasbr{bitrate[:-3]}k_DTX{dtx}" - + # to avoid conflicting names in case of parallel test execution, differentiate all cases if pca: tag_out += "_pca" @@ -1074,20 +1078,20 @@ def sba_dec( tag_out += "_SID" if plc_pattern is not None: tag_out += "_" + plc_pattern - + plc_file = None if plc_pattern is not None: plc_file = f"{TESTV_DIR}/{plc_pattern}.g192" - + if sid == 1: dut_pkt_file = f"{dut_pkt_dir}/{tag_out}_cut.192" - ref_pkt_file = f"{ref_pkt_dir}/{tag_out}_cut.192" - else: + ref_pkt_file = f"{ref_pkt_dir}/{tag_out}_cut.192" + else: dut_pkt_file = f"{dut_pkt_dir}/{tag_out}.192" ref_pkt_file = f"{ref_pkt_dir}/{tag_out}.192" - + dut_out_file = f"{dut_out_dir}/{tag_out}.wav" - ref_out_file = f"{ref_out_dir}/{tag_out}.wav" + ref_out_file = f"{ref_out_dir}/{tag_out}.wav" if update_ref == 1: # call REF decoder @@ -1137,7 +1141,5 @@ def sba_dec( # remove DUT output files when test result is OK (to save disk space) # if not keep_files: - # os.remove(dut_out_file) - # os.remove(dut_pkt_file) - - + # os.remove(dut_out_file) + # os.remove(dut_pkt_file) diff --git a/tests/conftest.py b/tests/conftest.py index 802373c65a..777b7025b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -707,7 +707,9 @@ class DecoderFrontend: # TODO: centralize this in a utils file if system == "Windows": - netsim_path = SCRIPTS_DIR.joinpath("tools/Win32/networkSimulator_g192.exe") + netsim_path = SCRIPTS_DIR.joinpath( + "tools/Win32/networkSimulator_g192.exe" + ) elif system == "Linux": netsim_path = SCRIPTS_DIR.joinpath("tools/Linux/networkSimulator_g192") elif system == "Darwin": -- GitLab From f8caaff4196017a21548c3cafb851d094c3f0a4a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 17 Sep 2024 11:53:39 +0200 Subject: [PATCH 27/28] remove obsolete imports --- tests/codec_be_on_mr_nonselection/test_sba.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index ee14c09c22..667169d4bc 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -35,7 +35,6 @@ __doc__ = """ import errno import os -from typing import Tuple import pytest from cut_bs import cut_from_start @@ -46,7 +45,6 @@ from ..cmp_stats_files import cmp_stats_files from ..constants import TESTV_DIR, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF from tests.testconfig import use_ltv -import pdb tag_list = ["ltvFOA" if use_ltv else "stvFOA"] tag_list_HOA2 = ["ltvHOA2" if use_ltv else "stv2OA"] -- GitLab From cd241c5ec84d080af8cad786792651c1c354f6bc Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Tue, 17 Sep 2024 15:12:11 +0200 Subject: [PATCH 28/28] add printout when res/ folder is empty --- tests/conftest.py | 94 ++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 45 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 802373c65a..2dee2f8096 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -392,57 +392,61 @@ class EncoderFrontend: """ hist_dicts = [] - for f in ENC_AUX_FILES: - filename = f[0] - dtype = f[1] - fs = int(sampling_rate) * 1000 - if isinstance(f[2], str): - nsamples_per_frame = np.int16(eval(f[2])) - else: - nsamples_per_frame = np.int16(f[2]) - - # aux_files = glob.glob(os.path.join(dbg_tweak_folder, filename + '\.*')) - aux_files = [ - f - for f in os.listdir(dbg_tweak_folder) - if re.search(rf"^{filename}(\..*)?$", f) - ] - # aux_files = [os.path.basename(f) for f in aux_files] - - for aux_file in aux_files: - # extract statistics from the aux file based on histogram of values - print( - f"Extracting statistics from {os.path.basename(aux_file)} ... ", - end="", - ) + + if not os.path.exists(dbg_tweak_folder): + print(f"No statistics have been extracted from the res/ folder to the {stats_file} file!") + else: + for f in ENC_AUX_FILES: + filename = f[0] + dtype = f[1] + fs = int(sampling_rate) * 1000 + if isinstance(f[2], str): + nsamples_per_frame = np.int16(eval(f[2])) + else: + nsamples_per_frame = np.int16(f[2]) + + # aux_files = glob.glob(os.path.join(dbg_tweak_folder, filename + '\.*')) + aux_files = [ + f + for f in os.listdir(dbg_tweak_folder) + if re.search(rf"^{filename}(\..*)?$", f) + ] + # aux_files = [os.path.basename(f) for f in aux_files] + + for aux_file in aux_files: + # extract statistics from the aux file based on histogram of values + print( + f"Extracting statistics from {os.path.basename(aux_file)} ... ", + end="", + ) - # read the aux file - with open(os.path.join(dbg_tweak_folder, aux_file), "r") as f_aux: - data = np.fromfile(f_aux, dtype=dtype) + # read the aux file + with open(os.path.join(dbg_tweak_folder, aux_file), "r") as f_aux: + data = np.fromfile(f_aux, dtype=dtype) - # get file length - data_len = data.shape[0] + # get file length + data_len = data.shape[0] - # remove the duplicates of each value per frame - if nsamples_per_frame > 1: - data = data[::nsamples_per_frame] + # remove the duplicates of each value per frame + if nsamples_per_frame > 1: + data = data[::nsamples_per_frame] - # calculate histogram from data - unique_values = np.sort(np.unique(data)) - hist, _ = np.histogram( - data, bins=np.append(unique_values, unique_values[-1] + 10) - ) + # calculate histogram from data + unique_values = np.sort(np.unique(data)) + hist, _ = np.histogram( + data, bins=np.append(unique_values, unique_values[-1] + 10) + ) - # convert to dict and sort by absolute value of difference - hist_dict = {"name": os.path.basename(aux_file), "length": data_len} - dict_values = { - str(unique_values[i]): str(hist[i]) - for i in range(len(unique_values)) - } - hist_dict.update(dict_values) - hist_dicts.append(hist_dict) + # convert to dict and sort by absolute value of difference + hist_dict = {"name": os.path.basename(aux_file), "length": data_len} + dict_values = { + str(unique_values[i]): str(hist[i]) + for i in range(len(unique_values)) + } + hist_dict.update(dict_values) + hist_dicts.append(hist_dict) - print("DONE") + print("DONE") print("") -- GitLab