From cfd9a01f15f6b2eabf7e7cd7a630794e08866b80 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 29 May 2024 11:22:46 +0200 Subject: [PATCH 1/9] add tracefile comparison for JBM cases --- .../test_param_file.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 34e92b715a..a808a31df8 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -40,6 +40,7 @@ import platform from pathlib import Path from subprocess import run import pytest +import numpy as np from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend @@ -316,6 +317,16 @@ def test_param_file_tests( if update_ref in [0, 2]: dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" ref_output_file = f"{reference_path}/param_file/dec/{output_file}" + dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}" + ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}" + + # set to false per default even if this is no JBM case - makes later check for failure easier + tracefile_lengths_differ = False + if len(tracefile_dec) > 0: + dut_tracefile_len = np.genfromtxt(dut_tracefile_dec, delimiter=";") + ref_tracefile_len = np.genfromtxt(ref_tracefile_dec, delimiter=";") + tracefile_lengths_differ = dut_tracefile_len.shape[0] != ref_tracefile_len.shape[0] + fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( dut_output_file, @@ -358,7 +369,9 @@ def test_param_file_tests( print("REF output metadata missing for expected file: " + md_file) metadata_differs = True - if get_mld and get_mld_lim > 0: + if tracefile_lengths_differ: + pytest.fail("Different tracefile lengths for JBM decoding - Not all frames were decoded in both ref and dut.") + elif get_mld and get_mld_lim > 0: if output_differs: pytest.fail(reason) else: -- GitLab From a740ed607c9ed84a8e4b466de74385276d9129d0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 29 May 2024 13:39:02 +0200 Subject: [PATCH 2/9] cleanup --- tests/codec_be_on_mr_nonselection/test_param_file.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 8f1c2aa28a..c40345cbe0 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -278,11 +278,6 @@ def test_param_file_tests( else: output_config = "" - output_config_name = output_config - if "/" in output_config: - # the output config is a file - output_config_name = os.path.splitext(os.path.basename(output_config))[0] - tracefile_dec = "" if sim_opts != "": assert bitstream_file_dec == "netsimoutput" @@ -304,7 +299,7 @@ def test_param_file_tests( # -> construct output filename output_file = f"{testv_base}_{tag_str}.dec.wav" - stdout = decode( + decode( dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -322,12 +317,13 @@ def test_param_file_tests( if update_ref in [0, 2]: dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" ref_output_file = f"{reference_path}/param_file/dec/{output_file}" - dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}" - ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}" # set to false per default even if this is no JBM case - makes later check for failure easier tracefile_lengths_differ = False if len(tracefile_dec) > 0: + dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}" + ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}" + dut_tracefile_len = np.genfromtxt(dut_tracefile_dec, delimiter=";") ref_tracefile_len = np.genfromtxt(ref_tracefile_dec, delimiter=";") tracefile_lengths_differ = dut_tracefile_len.shape[0] != ref_tracefile_len.shape[0] -- GitLab From 72827b60f81eca4def802f98e16d3cf1953d2a14 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 29 May 2024 13:39:28 +0200 Subject: [PATCH 3/9] run formatter --- .../test_param_file.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index c40345cbe0..a78e55ca66 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -64,7 +64,11 @@ VALID_DEC_OUTPUT_CONF = [ "EXT", ] -PARAM_FILE_ID = "stv" if PARAM_FILE.stem == "self_test" else PARAM_FILE.stem.replace("self_test_", "") +PARAM_FILE_ID = ( + "stv" + if PARAM_FILE.stem == "self_test" + else PARAM_FILE.stem.replace("self_test_", "") +) param_file_test_dict = {} with open(PARAM_FILE, "r", encoding="UTF-8") as fp: @@ -326,7 +330,9 @@ def test_param_file_tests( dut_tracefile_len = np.genfromtxt(dut_tracefile_dec, delimiter=";") ref_tracefile_len = np.genfromtxt(ref_tracefile_dec, delimiter=";") - tracefile_lengths_differ = dut_tracefile_len.shape[0] != ref_tracefile_len.shape[0] + tracefile_lengths_differ = ( + dut_tracefile_len.shape[0] != ref_tracefile_len.shape[0] + ) fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( @@ -344,7 +350,7 @@ def test_param_file_tests( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) - max_diff =0 + max_diff = 0 if output_differs: search_result = re.search(MAX_DIFF_PATTERN, reason) if search_result: @@ -371,7 +377,9 @@ def test_param_file_tests( metadata_differs = True if tracefile_lengths_differ: - pytest.fail("Different tracefile lengths for JBM decoding - Not all frames were decoded in both ref and dut.") + pytest.fail( + "Different tracefile lengths for JBM decoding - Not all frames were decoded in both ref and dut." + ) elif get_mld and get_mld_lim > 0: if output_differs: pytest.fail(reason) -- GitLab From 56f3db826ede7c03e7dfc0afd15c6312183fb154 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 29 May 2024 16:24:09 +0200 Subject: [PATCH 4/9] do comp for diff in len if tracefiles are of same length --- tests/cmp_pcm.py | 26 +++++++++++++------ .../test_param_file.py | 7 +++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index d958327ccd..9783e8b561 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -12,7 +12,16 @@ import pyaudio3dtools import pyivastest -def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0, abs_tol=0) -> (int, str): +def cmp_pcm( + file1, + file2, + out_config, + fs, + get_mld=False, + allow_differing_lengths=False, + mld_lim=0, + abs_tol=0, +) -> (int, str): """ Compare 2 PCM files for bitexactness """ @@ -22,24 +31,26 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0, abs_tol=0) - out_config = "MONO" if out_config == "" else out_config # out_config may be a string or a Path. Wrap in str() to avoid error in case it is a Path. if str(out_config).upper() not in pyivastest.constants.OC_TO_NCHANNELS: - out_config_in_file_names = os.path.splitext(os.path.basename(out_config))[0] nchannels = ( pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout( out_config ) ) else: - out_config_in_file_names = out_config nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs, outdtype=np.int16) s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs, outdtype=np.int16) - nchannels = s1.shape[ - 1 - ] # In case of wav input, override the nchannels with the one from the wav header + # In case of wav input, override the nchannels with the one from the wav header + nchannels = s1.shape[1] - if s1.shape != s2.shape: + if allow_differing_lengths: + # to allow for MLD comparison, shorten longer file + min_len = min(s1.shape[0], s2.shape[0]) + s1 = s1[:min_len, :] + s2 = s2[:min_len, :] + elif s1.shape != s2.shape: print( f"file size in samples: file 1 = {s1.shape[0]},", f"file 2 = {s2.shape[0]}", @@ -54,7 +65,6 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0, abs_tol=0) - s1, s2, fs, per_frame=False, get_mld=get_mld ) - output_differs = 0 reason = "SUCCESS: Files are bitexact" diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index a78e55ca66..fd076a63f3 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -324,6 +324,7 @@ def test_param_file_tests( # set to false per default even if this is no JBM case - makes later check for failure easier tracefile_lengths_differ = False + if len(tracefile_dec) > 0: dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}" ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}" @@ -334,6 +335,11 @@ def test_param_file_tests( dut_tracefile_len.shape[0] != ref_tracefile_len.shape[0] ) + # same tf lengths -> likely no crash, assume length difference is due to difference in TSM + # to get MLD and abs diff values for now - even though they might be meaningless due to + # shift differences between the two signals - cut longer signal to shorter size + allow_differing_lengths = not tracefile_lengths_differ + fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( dut_output_file, @@ -343,6 +349,7 @@ def test_param_file_tests( get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) -- GitLab From 20e313779549860878bdbe06c3bd54152fa543e9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 14:48:49 +0200 Subject: [PATCH 5/9] compare RTP seq number instead of file length --- tests/codec_be_on_mr_nonselection/test_param_file.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index fd076a63f3..c6a2c4fa75 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -329,13 +329,14 @@ def test_param_file_tests( dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}" ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}" - dut_tracefile_len = np.genfromtxt(dut_tracefile_dec, delimiter=";") - ref_tracefile_len = np.genfromtxt(ref_tracefile_dec, delimiter=";") + # check for same RTP sequence number in last line of tracefile + dut_tracefile_last_line = np.genfromtxt(dut_tracefile_dec, delimiter=";")[-1,:] + ref_tracefile_last_line = np.genfromtxt(ref_tracefile_dec, delimiter=";")[-1,:] tracefile_lengths_differ = ( - dut_tracefile_len.shape[0] != ref_tracefile_len.shape[0] + dut_tracefile_last_line[0] != ref_tracefile_last_line[0] ) - # same tf lengths -> likely no crash, assume length difference is due to difference in TSM + # same sequence number -> likely no crash, assume length difference is due to difference in TSM # to get MLD and abs diff values for now - even though they might be meaningless due to # shift differences between the two signals - cut longer signal to shorter size allow_differing_lengths = not tracefile_lengths_differ -- GitLab From e241bea335ad27b3351a936c3cc345ac19779ed1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Jun 2024 09:58:37 +0200 Subject: [PATCH 6/9] correct printout --- tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index c6a2c4fa75..224eac2db5 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -386,7 +386,7 @@ def test_param_file_tests( if tracefile_lengths_differ: pytest.fail( - "Different tracefile lengths for JBM decoding - Not all frames were decoded in both ref and dut." + "Last RTP sequence num in tracefiles differ for JBM decoding - Not all frames were decoded in both ref and dut." ) elif get_mld and get_mld_lim > 0: if output_differs: -- GitLab From e1870219c8a59445b915d5ff21cbfefee89bb64b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Jun 2024 13:01:28 +0200 Subject: [PATCH 7/9] cleanup and naming improvements --- tests/codec_be_on_mr_nonselection/test_param_file.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 224eac2db5..e7571b14a0 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -323,23 +323,21 @@ def test_param_file_tests( ref_output_file = f"{reference_path}/param_file/dec/{output_file}" # set to false per default even if this is no JBM case - makes later check for failure easier - tracefile_lengths_differ = False + tracefile_last_rtp_numbers_differ = False if len(tracefile_dec) > 0: dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}" ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}" # check for same RTP sequence number in last line of tracefile - dut_tracefile_last_line = np.genfromtxt(dut_tracefile_dec, delimiter=";")[-1,:] - ref_tracefile_last_line = np.genfromtxt(ref_tracefile_dec, delimiter=";")[-1,:] - tracefile_lengths_differ = ( - dut_tracefile_last_line[0] != ref_tracefile_last_line[0] - ) + dut_rtp_num_last = np.genfromtxt(dut_tracefile_dec, delimiter=";")[-1,0] + ref_rtp_num_last = np.genfromtxt(ref_tracefile_dec, delimiter=";")[-1,0] + tracefile_last_rtp_numbers_differ = dut_rtp_num_last != ref_rtp_num_last # same sequence number -> likely no crash, assume length difference is due to difference in TSM # to get MLD and abs diff values for now - even though they might be meaningless due to # shift differences between the two signals - cut longer signal to shorter size - allow_differing_lengths = not tracefile_lengths_differ + allow_differing_lengths = not tracefile_last_rtp_numbers_differ fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( -- GitLab From 53de5a2fc43225a0681bc7b40d6680aea3dacc41 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Jun 2024 13:02:46 +0200 Subject: [PATCH 8/9] fix typos in first-frame-is-sid test --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 399cb6f22e..3e97e863bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1093,9 +1093,9 @@ check-first-frame-is-sid: - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep -E "HOA") - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 70:80 --oc stereo --timeout 10 || exit_code_hoa=$? - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep "FOA") - - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 75:110 --oc stereo timeout 10 || exit_code_foa=$? + - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 75:110 --oc stereo --timeout 10 || exit_code_foa=$? - - if [ $exit_code_no_sba -ne 0 ] || [ $exit_code_hoa -ne 0 ] || [ $exit_code_hoa -ne 0 ]; then exit 1; fi + - if [ $exit_code_no_sba -ne 0 ] || [ $exit_code_hoa -ne 0 ] || [ $exit_code_foa -ne 0 ]; then exit 1; fi artifacts: paths: - out/logs -- GitLab From a9f7d244f65b79a355b47830aa23644d27b9eb06 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 10 Jun 2024 13:18:41 +0200 Subject: [PATCH 9/9] fix forgotten rename --- tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index e7571b14a0..619e252767 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -382,7 +382,7 @@ def test_param_file_tests( print("REF output metadata missing for expected file: " + md_file) metadata_differs = True - if tracefile_lengths_differ: + if tracefile_last_rtp_numbers_differ: pytest.fail( "Last RTP sequence num in tracefiles differ for JBM decoding - Not all frames were decoded in both ref and dut." ) -- GitLab