diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 399cb6f22e5bf2de53ec28b4f8f670dff7cf4259..3e97e863bb4376a4323a0e859a221e7307980a10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1093,9 +1093,9 @@ check-first-frame-is-sid: - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep -E "HOA") - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 70:80 --oc stereo --timeout 10 || exit_code_hoa=$? - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep "FOA") - - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 75:110 --oc stereo timeout 10 || exit_code_foa=$? + - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 75:110 --oc stereo --timeout 10 || exit_code_foa=$? - - if [ $exit_code_no_sba -ne 0 ] || [ $exit_code_hoa -ne 0 ] || [ $exit_code_hoa -ne 0 ]; then exit 1; fi + - if [ $exit_code_no_sba -ne 0 ] || [ $exit_code_hoa -ne 0 ] || [ $exit_code_foa -ne 0 ]; then exit 1; fi artifacts: paths: - out/logs diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index d958327ccda7ddd9da99f0737133a3bcd9d8b4b2..9783e8b5613cfe89a5ba054b95e3d9cfb5f5c68e 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -12,7 +12,16 @@ import pyaudio3dtools import pyivastest -def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0, abs_tol=0) -> (int, str): +def cmp_pcm( + file1, + file2, + out_config, + fs, + get_mld=False, + allow_differing_lengths=False, + mld_lim=0, + abs_tol=0, +) -> (int, str): """ Compare 2 PCM files for bitexactness """ @@ -22,24 +31,26 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0, abs_tol=0) - out_config = "MONO" if out_config == "" else out_config # out_config may be a string or a Path. Wrap in str() to avoid error in case it is a Path. if str(out_config).upper() not in pyivastest.constants.OC_TO_NCHANNELS: - out_config_in_file_names = os.path.splitext(os.path.basename(out_config))[0] nchannels = ( pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout( out_config ) ) else: - out_config_in_file_names = out_config nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs, outdtype=np.int16) s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs, outdtype=np.int16) - nchannels = s1.shape[ - 1 - ] # In case of wav input, override the nchannels with the one from the wav header + # In case of wav input, override the nchannels with the one from the wav header + nchannels = s1.shape[1] - if s1.shape != s2.shape: + if allow_differing_lengths: + # to allow for MLD comparison, shorten longer file + min_len = min(s1.shape[0], s2.shape[0]) + s1 = s1[:min_len, :] + s2 = s2[:min_len, :] + elif s1.shape != s2.shape: print( f"file size in samples: file 1 = {s1.shape[0]},", f"file 2 = {s2.shape[0]}", @@ -54,7 +65,6 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0, abs_tol=0) - s1, s2, fs, per_frame=False, get_mld=get_mld ) - output_differs = 0 reason = "SUCCESS: Files are bitexact" diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 5b0f7f696a278ec51462e4d9a7bfec191e413914..619e25276734d59b4da909b556358a3020fc817b 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -40,6 +40,7 @@ import platform from pathlib import Path from subprocess import run import pytest +import numpy as np from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend @@ -63,7 +64,11 @@ VALID_DEC_OUTPUT_CONF = [ "EXT", ] -PARAM_FILE_ID = "stv" if PARAM_FILE.stem == "self_test" else PARAM_FILE.stem.replace("self_test_", "") +PARAM_FILE_ID = ( + "stv" + if PARAM_FILE.stem == "self_test" + else PARAM_FILE.stem.replace("self_test_", "") +) param_file_test_dict = {} with open(PARAM_FILE, "r", encoding="UTF-8") as fp: @@ -277,11 +282,6 @@ def test_param_file_tests( else: output_config = "" - output_config_name = output_config - if "/" in output_config: - # the output config is a file - output_config_name = os.path.splitext(os.path.basename(output_config))[0] - tracefile_dec = "" if sim_opts != "": assert bitstream_file_dec == "netsimoutput" @@ -303,7 +303,7 @@ def test_param_file_tests( # -> construct output filename output_file = f"{testv_base}_{tag_str}.dec.wav" - stdout = decode( + decode( dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -321,6 +321,24 @@ def test_param_file_tests( if update_ref in [0, 2]: dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" ref_output_file = f"{reference_path}/param_file/dec/{output_file}" + + # set to false per default even if this is no JBM case - makes later check for failure easier + tracefile_last_rtp_numbers_differ = False + + if len(tracefile_dec) > 0: + dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}" + ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}" + + # check for same RTP sequence number in last line of tracefile + dut_rtp_num_last = np.genfromtxt(dut_tracefile_dec, delimiter=";")[-1,0] + ref_rtp_num_last = np.genfromtxt(ref_tracefile_dec, delimiter=";")[-1,0] + tracefile_last_rtp_numbers_differ = dut_rtp_num_last != ref_rtp_num_last + + # same sequence number -> likely no crash, assume length difference is due to difference in TSM + # to get MLD and abs diff values for now - even though they might be meaningless due to + # shift differences between the two signals - cut longer signal to shorter size + allow_differing_lengths = not tracefile_last_rtp_numbers_differ + fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( dut_output_file, @@ -330,6 +348,7 @@ def test_param_file_tests( get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) @@ -337,7 +356,7 @@ def test_param_file_tests( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) - max_diff =0 + max_diff = 0 if output_differs: search_result = re.search(MAX_DIFF_PATTERN, reason) if search_result: @@ -363,7 +382,11 @@ def test_param_file_tests( print("REF output metadata missing for expected file: " + md_file) metadata_differs = True - if get_mld and get_mld_lim > 0: + if tracefile_last_rtp_numbers_differ: + pytest.fail( + "Last RTP sequence num in tracefiles differ for JBM decoding - Not all frames were decoded in both ref and dut." + ) + elif get_mld and get_mld_lim > 0: if output_differs: pytest.fail(reason) else: