From 2dcf95c115ae724f37e2370e0b9bbbfb53afe271 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 15 Jan 2024 00:29:03 +0100 Subject: [PATCH 01/12] Add option to use MLD limit to pass test --- scripts/parse_mld_xml.py | 7 +------ tests/cmp_pcm.py | 10 +++++++--- .../codec_be_on_mr_nonselection/test_param_file.py | 14 +++++++++++--- tests/conftest.py | 12 ++++++++++++ 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index bbc47754ca..6273bfd03e 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -26,10 +26,5 @@ if __name__ == '__main__': with open(csv_file,'w') as outfile: for testcase in testcases: fulltestname = testcase.get('file') + "::" + testcase.get('name') - system_out = testcase.find(".//system-out") - mld_val = 0 - if system_out is not None: - for line in system_out.text.split('\n'): - if line.startswith('MLD:'): - mld_val = float(line.split()[1]) + mld_val = testcase.find('.//property').get('value') outfile.write(fulltestname + ';' + str(mld_val)+'\n') diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 817ac9c502..0d160f66b9 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -12,7 +12,7 @@ import pyivastest import numpy as np -def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str): +def cmp_pcm(file1, file2, out_config, fs, get_mld = False, mld_lim = 0) -> (int, str): """ Compare 2 PCM files for bitexactness """ @@ -54,8 +54,12 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str): print(first_msg) if get_mld: mld_msg = f"MLD: {cmp_result['MLD']}" - print(mld_msg) - return 1, "FAIL: Files have different content" + print(mld_msg) + if cmp_result['MLD'] <= mld_lim: + return 0, f"MLD: {cmp_result['MLD']} <= {mld_lim}" + else: + return 1, f"MLD: {cmp_result['MLD']} > {mld_lim}" + return 1, "Non-BE" if __name__ == "__main__": diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 1dfda043d7..2237e9d796 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -125,6 +125,7 @@ def convert_test_string_to_tag(test_string): @pytest.mark.create_ref @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) def test_param_file_tests( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -137,6 +138,7 @@ def test_param_file_tests( keep_files, test_tag, get_mld, + get_mld_lim, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -325,10 +327,16 @@ def test_param_file_tests( ref_output_file = f"{reference_path}/param_file/dec/{output_file}" fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( - dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld + dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) + metadata_differs = False for md_file in md_out_files: dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") @@ -347,9 +355,9 @@ def test_param_file_tests( if output_differs or metadata_differs: msg = "Difference between ref and dut in " if output_differs and metadata_differs: - msg += "output and metadata" + msg += f"output ({reason}) and metadata" elif output_differs: - msg += "output only" + msg += f"output only ({reason})" elif metadata_differs: msg += "metadata only" diff --git a/tests/conftest.py b/tests/conftest.py index 5f7ed04a82..bec51d0214 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -158,6 +158,12 @@ def pytest_addoption(parser): help="Run the MLD tool instead of just comparing for bitexactness", ) + parser.addoption( + "--mld-lim", + action="store", + help="MLD limit for comparison (default: 0)", + default="0", + ) @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -177,6 +183,12 @@ def get_mld(request): """ return request.config.option.mld +@pytest.fixture(scope="session", autouse=True) +def get_mld_lim(request): + """ + Return MLD limit for MLD comparison + """ + return float(request.config.getoption("--mld-lim")) @pytest.fixture(scope="session") def keep_files(request) -> bool: -- GitLab From b4b4c542e85fc8c400eeaf6b8031688589b3c485 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 15 Jan 2024 10:02:00 +0100 Subject: [PATCH 02/12] Add MLD property setting in tests --- .../test_masa_enc_dec.py | 14 ++++++++++++++ .../test_sba_bs_dec_plc.py | 7 +++++++ .../codec_be_on_mr_nonselection/test_sba_bs_enc.py | 9 +++++++++ tests/test_param_file_ltv.py | 14 +++++++++++--- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 0614651653..5df246946c 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -93,6 +93,7 @@ def check_and_makedir(dir_path): masa_metadata_audio_ndir_ntransportch_dtx_list, ) def test_masa_enc_dec( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -106,6 +107,7 @@ def test_masa_enc_dec( test_vector_path, output_mode, get_mld, + get_mld_lim, ): # Input parameters in_fs = 48 @@ -203,6 +205,11 @@ def test_masa_enc_dec( pcmcmp_res, reason = cmp_pcm( dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld ) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) # Fail if compare fails compare result if metacmp_res == False and pcmcmp_res != 0: @@ -221,9 +228,16 @@ def test_masa_enc_dec( cmp_result, reason = cmp_pcm( dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld ) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) # Report compare result assert cmp_result == 0, reason else: + if get_mld: + record_property("MLD","0") print("Comparison bit exact") # remove_output( diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 422f87d9c7..113c2f54d4 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -75,6 +75,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("fs", sampling_rate_list) @pytest.mark.parametrize("gain_flag", gain_list) def test_sba_plc_system( + record_property, dut_decoder_frontend: DecoderFrontend, test_vector_path, reference_path, @@ -89,6 +90,7 @@ def test_sba_plc_system( fs, gain_flag, get_mld, + get_mld_lim, ): SID = 0 if dtx == '1' and ivas_br not in ['13200','16400','24400','32000', '64000']: @@ -192,6 +194,11 @@ def sba_dec_plc( # -------------- compare cmd -------------- fs = int(sampling_rate) * 1000 cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) # report compare result assert cmp_result == 0, reason diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 3e84e6b283..05d90d28e1 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -89,6 +89,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("fs", sample_rate_list) def test_pca_enc( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -101,6 +102,7 @@ def test_pca_enc( tag, fs, get_mld, + get_mld_lim, ): pca = True tag = tag + fs + "c" @@ -161,6 +163,7 @@ def test_pca_enc( @pytest.mark.parametrize("gain_flag", gain_list) @pytest.mark.parametrize("SID", SID_list) def test_sba_enc_system( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -178,6 +181,7 @@ def test_sba_enc_system( gain_flag, SID, get_mld, + get_mld_lim, ): if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: @@ -664,6 +668,11 @@ def sba_dec( fs = int(sampling_rate) * 1000 cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) # report compare result assert cmp_result == 0, reason diff --git a/tests/test_param_file_ltv.py b/tests/test_param_file_ltv.py index 055cd24154..6c9b396e83 100644 --- a/tests/test_param_file_ltv.py +++ b/tests/test_param_file_ltv.py @@ -126,6 +126,7 @@ def convert_test_string_to_tag(test_string): @pytest.mark.create_ref @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) def test_param_file_tests( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -138,6 +139,7 @@ def test_param_file_tests( keep_files, test_tag, get_mld, + get_mld_lim, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -326,10 +328,16 @@ def test_param_file_tests( ref_output_file = f"{reference_path}/param_file/dec/{output_file}" fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( - dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld + dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) + metadata_differs = False for md_file in md_out_files: dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") @@ -348,9 +356,9 @@ def test_param_file_tests( if output_differs or metadata_differs: msg = "Difference between ref and dut in " if output_differs and metadata_differs: - msg += "output and metadata" + msg += f"output ({reason}) and metadata" elif output_differs: - msg += "output only" + msg += f"output only ({reason})" elif metadata_differs: msg += "metadata only" -- GitLab From af0f9d5b4737fb3e89af6356b00f2bdf50d9edc1 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 16 Jan 2024 13:42:46 +0100 Subject: [PATCH 03/12] Fix missing get_mld_lim --- tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py | 4 ++-- tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py | 4 +++- tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py | 6 +++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 5df246946c..8a01b18026 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -203,7 +203,7 @@ def test_masa_enc_dec( # Compare audio outputs pcmcmp_res, reason = cmp_pcm( - dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld + dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim ) if get_mld: mld = 0 @@ -226,7 +226,7 @@ def test_masa_enc_dec( filecmp_res = cmp(dec_output_ref, dec_output_dut) if filecmp_res == False: cmp_result, reason = cmp_pcm( - dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld + dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim ) if get_mld: mld = 0 diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 113c2f54d4..e7ec112300 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -122,6 +122,7 @@ def test_sba_plc_system( gain_flag, keep_files, get_mld=get_mld, + get_mld_lim=get_mld_lim, ) @@ -143,6 +144,7 @@ def sba_dec_plc( gain_flag, keep_files, get_mld=False, + get_mld_lim=0, ): # ------------ run cmd ------------ @@ -193,7 +195,7 @@ def sba_dec_plc( # -------------- compare cmd -------------- fs = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld) + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim) if get_mld: mld = 0 if "MLD" in reason: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 05d90d28e1..da262d6608 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -151,6 +151,7 @@ def test_pca_enc( gain_flag, keep_files, get_mld=get_mld, + get_mld_lim=get_mld_lim, pca=pca, ) @@ -246,6 +247,7 @@ def test_sba_enc_system( gain_flag, keep_files, get_mld=get_mld + get_mld_lim=get_mld_lim ) @@ -311,6 +313,7 @@ def test_spar_hoa2_enc_system( gain_flag, keep_files, get_mld=get_mld, + get_mld_lim=get_mld_lim, ) @@ -613,6 +616,7 @@ def sba_dec( gain_flag, keep_files, get_mld=False, + get_mld_lim=0, pca=False, ): # -------- run cmd ------------ @@ -667,7 +671,7 @@ def sba_dec( ) fs = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld) + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim ) if get_mld: mld = 0 if "MLD" in reason: -- GitLab From b1e856f15c7bd69d9fab2d7decbf27aed0c0c3a6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 16 Jan 2024 14:24:17 +0100 Subject: [PATCH 04/12] Correct syntax error (missing comma) --- tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index da262d6608..2342e5a664 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -246,7 +246,7 @@ def test_sba_enc_system( update_ref, gain_flag, keep_files, - get_mld=get_mld + get_mld=get_mld, get_mld_lim=get_mld_lim ) -- GitLab From d75ba4cde1133fcf9ab982a23dbb449d24724480 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 16 Jan 2024 15:52:33 +0100 Subject: [PATCH 05/12] Fix syntax error related to get_mld_lim. Add command line option for cmp_pcm.py when using from command line --- tests/cmp_pcm.py | 1 + tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 0d160f66b9..9db7ccd39e 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -75,6 +75,7 @@ if __name__ == "__main__": ) parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs") parser.add_argument("--get_mld", action="store_true") + parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim") args = parser.parse_args() result, msg = cmp_pcm(**vars(args)) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 2342e5a664..12c9e483e3 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -267,6 +267,7 @@ def test_spar_hoa2_enc_system( ivas_br, tag, get_mld, + get_mld_lim, ): fs = "48" dtx = "0" -- GitLab From 0fcd1f1f551b2a4846b3de2cab6add880f83fed1 Mon Sep 17 00:00:00 2001 From: azmill Date: Tue, 30 Jan 2024 14:18:54 +1100 Subject: [PATCH 06/12] Fixing tests scripts so that get_mld_lim is passed to all --- .../test_sba_bs_dec_plc.py | 2 ++ .../test_sba_bs_enc.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index e7ec112300..6d03ad7536 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -107,6 +107,7 @@ def test_sba_plc_system( # dec sba_dec_plc( + record_property, dut_decoder_frontend, test_vector_path, reference_path, @@ -129,6 +130,7 @@ def test_sba_plc_system( ######################################################### # -------------------- test function -------------------- def sba_dec_plc( + record_property, decoder_frontend, test_vector_path, reference_path, diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 12c9e483e3..0f98e5f156 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -136,6 +136,7 @@ def test_pca_enc( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -232,6 +233,7 @@ def test_sba_enc_system( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -255,6 +257,7 @@ def test_sba_enc_system( @pytest.mark.parametrize("ivas_br", ivas_br_HOA2) @pytest.mark.parametrize("tag", tag_list_HOA2) def test_spar_hoa2_enc_system( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -299,6 +302,7 @@ def test_spar_hoa2_enc_system( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -322,6 +326,7 @@ def test_spar_hoa2_enc_system( @pytest.mark.parametrize("ivas_br", ivas_br_HOA3) @pytest.mark.parametrize("tag", tag_list_HOA3) def test_spar_hoa3_enc_system( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -334,6 +339,7 @@ def test_spar_hoa3_enc_system( ivas_br, tag, get_mld, + get_mld_lim, ): fs = "48" dtx = "0" @@ -365,6 +371,7 @@ def test_spar_hoa3_enc_system( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -379,7 +386,8 @@ def test_spar_hoa3_enc_system( update_ref, gain_flag, keep_files, - get_mld=get_mld + get_mld=get_mld, + get_mld_lim=get_mld_lim, ) @@ -389,6 +397,7 @@ def test_spar_hoa3_enc_system( @pytest.mark.parametrize("tag", tag_list_bw_force) @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) def test_sba_enc_BWforce_system( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -403,6 +412,7 @@ def test_sba_enc_BWforce_system( tag, sample_rate_bw_idx, get_mld, + get_mld_lim, ): if dtx == "1" and ivas_br not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -440,6 +450,7 @@ def test_sba_enc_BWforce_system( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -455,6 +466,7 @@ def test_sba_enc_BWforce_system( gain_flag, keep_files, get_mld=get_mld, + get_mld_lim=get_mld_lim, ) @@ -602,6 +614,7 @@ def sba_enc( def sba_dec( + record_property, decoder_frontend, ref_decoder_frontend, reference_path, -- GitLab From 1639a8d661f353be5a65ec6c7f5908a191a9e806 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 30 Jan 2024 17:02:20 +0100 Subject: [PATCH 07/12] Change tests from asserts to pytest.fail --- tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py | 3 ++- tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +- tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py | 3 ++- tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py | 3 ++- tests/test_26444.py | 2 +- tests/test_param_file_ltv.py | 3 +-- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 8a01b18026..3c49f9f21c 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -234,7 +234,8 @@ def test_masa_enc_dec( mld = float(reason.split(':')[1].split()[0]) record_property("MLD",mld) # Report compare result - assert cmp_result == 0, reason + if cmp_result != 0: + pytest.fail(reason) else: if get_mld: record_property("MLD","0") diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 2237e9d796..c5fe72ee50 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -361,7 +361,7 @@ def test_param_file_tests( elif metadata_differs: msg += "metadata only" - assert False, msg + pytest.fail(msg) # remove DUT output files when test result is OK (to save disk space) if not keep_files: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 6d03ad7536..484ada6c8a 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -205,7 +205,8 @@ def sba_dec_plc( record_property("MLD",mld) # report compare result - assert cmp_result == 0, reason + if cmp_result != 0: + pytest.fail(reason) # remove DUT output files when test result is OK (to save disk space) if not keep_files: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 0f98e5f156..650bddb17d 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -693,7 +693,8 @@ def sba_dec( record_property("MLD",mld) # report compare result - assert cmp_result == 0, reason + if cmp_result != 0: + pytest.fail(reason) # remove DUT output files when test result is OK (to save disk space) if not keep_files: diff --git a/tests/test_26444.py b/tests/test_26444.py index fcbf2cfc7f..ccf8d23fad 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -88,7 +88,7 @@ def test_evs_26444(test_tag): result1 = filecmp.cmp(cmd1[0], cmd1[1]) result2 = True if not (result1 and result2): - assert False, "Output differs" + pytest.fail("Output differs") \ No newline at end of file diff --git a/tests/test_param_file_ltv.py b/tests/test_param_file_ltv.py index 6c9b396e83..d74a32aa76 100644 --- a/tests/test_param_file_ltv.py +++ b/tests/test_param_file_ltv.py @@ -361,8 +361,7 @@ def test_param_file_tests( msg += f"output only ({reason})" elif metadata_differs: msg += "metadata only" - - assert False, msg + pytest.fail(msg) # remove DUT output files when test result is OK (to save disk space) if not keep_files: -- GitLab From 31ef92d389e433728959a721589e598fc7bc7640 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 31 Jan 2024 09:10:51 +0100 Subject: [PATCH 08/12] Fix parse_mld_xml.py --- scripts/parse_mld_xml.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index 6273bfd03e..a8868f21ee 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -25,6 +25,10 @@ if __name__ == '__main__': with open(csv_file,'w') as outfile: for testcase in testcases: - fulltestname = testcase.get('file') + "::" + testcase.get('name') - mld_val = testcase.find('.//property').get('value') - outfile.write(fulltestname + ';' + str(mld_val)+'\n') + if testcase.find('.//skipped') == None: + fulltestname = testcase.get('classname') + "::" + testcase.get('name') + if testcase.find('.//property') == None: + mld_val = None + else: + mld_val = testcase.find('.//property').get('value') # Currently MLD is the only set property. If more are added updates are needed here. + outfile.write(fulltestname + ';' + str(mld_val)+'\n') -- GitLab From 3cbe209b400747f98921506ae490e0550c77c996 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 31 Jan 2024 09:32:29 +0100 Subject: [PATCH 09/12] Fix parse_mld_xml.py to work both for pytest files and single tests --- scripts/parse_mld_xml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index a8868f21ee..e3b6dd57ae 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -26,7 +26,10 @@ if __name__ == '__main__': with open(csv_file,'w') as outfile: for testcase in testcases: if testcase.find('.//skipped') == None: - fulltestname = testcase.get('classname') + "::" + testcase.get('name') + if testcase.get('file') == None: + fulltestname = testcase.get('classname').replace('.','/') + ".py::" + testcase.get('name') + else: + fulltestname = testcase.get('file') + "::" + testcase.get('name') if testcase.find('.//property') == None: mld_val = None else: -- GitLab From 0861c7d66f8d13bfcf7e48613c8a2d5f0aca57dc Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 1 Feb 2024 10:28:54 +0100 Subject: [PATCH 10/12] Update test_renderer.py to use MLD and also mark with create_ref to be bundled with other tests --- tests/renderer/test_renderer.py | 64 +++++++++++++++++++++------------ tests/renderer/utils.py | 29 ++++++++++----- 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 1ddba22695..c4717e31be 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -42,11 +42,11 @@ from .utils import * ############################################################################## """ Ambisonics """ - +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ambisonics(test_info, in_fmt, out_fmt, frame_size): +def test_ambisonics(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( test_info, in_fmt, @@ -55,10 +55,11 @@ def test_ambisonics(test_info, in_fmt, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt, frame_size): +def test_ambisonics_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( test_info, in_fmt, @@ -67,12 +68,13 @@ def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics_binaural_headrotation( - test_info, in_fmt, out_fmt, trj_file, frame_size + record_property, test_info, in_fmt, out_fmt, trj_file, frame_size, get_mld, get_mld_lim ): run_renderer( test_info, @@ -86,10 +88,11 @@ def test_ambisonics_binaural_headrotation( """ Multichannel """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_multichannel(test_info, in_fmt, out_fmt, frame_size): +def test_multichannel(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( test_info, in_fmt, @@ -98,10 +101,11 @@ def test_multichannel(test_info, in_fmt, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, frame_size): +def test_multichannel_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -113,12 +117,13 @@ def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_multichannel_binaural_headrotation( - test_info, in_fmt, out_fmt, trj_file, frame_size + record_property, test_info, in_fmt, out_fmt, trj_file, frame_size, get_mld, get_mld_lim ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -135,10 +140,11 @@ def test_multichannel_binaural_headrotation( """ ISM """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ism(test_info, in_fmt, out_fmt, frame_size): +def test_ism(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( test_info, in_fmt, @@ -148,10 +154,11 @@ def test_ism(test_info, in_fmt, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ism_binaural_static(test_info, in_fmt, out_fmt, frame_size): +def test_ism_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] except KeyError: @@ -166,11 +173,12 @@ def test_ism_binaural_static(test_info, in_fmt, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, frame_size): +def test_ism_binaural_headrotation(record_property, test_info, in_fmt, out_fmt, trj_file, frame_size, get_mld, get_mld_lim): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] except KeyError: @@ -189,10 +197,11 @@ def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, frame_s """ MASA """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_masa(test_info, in_fmt, out_fmt, frame_size): +def test_masa(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( test_info, in_fmt, @@ -202,10 +211,11 @@ def test_masa(test_info, in_fmt, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_masa_binaural_static(test_info, in_fmt, out_fmt, frame_size): +def test_masa_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -218,11 +228,12 @@ def test_masa_binaural_static(test_info, in_fmt, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_masa_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, frame_size): +def test_masa_binaural_headrotation(record_property, test_info, in_fmt, out_fmt, trj_file, frame_size, get_mld, get_mld_lim): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -236,8 +247,9 @@ def test_masa_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, frame_ ) +@pytest.mark.create_ref @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) -def test_masa_prerend(test_info, in_fmt): +def test_masa_prerend(record_property, test_info, in_fmt, get_mld, get_mld_lim): run_renderer( test_info, "META", @@ -249,10 +261,11 @@ def test_masa_prerend(test_info, in_fmt): """ Custom loudspeaker layouts """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_custom_ls_input(test_info, in_layout, out_fmt, frame_size): +def test_custom_ls_input(record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), @@ -261,9 +274,10 @@ def test_custom_ls_input(test_info, in_layout, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) -def test_custom_ls_output(test_info, in_fmt, out_fmt): +def test_custom_ls_output(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): run_renderer( test_info, in_fmt, @@ -271,9 +285,10 @@ def test_custom_ls_output(test_info, in_fmt, out_fmt): ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) -def test_custom_ls_input_output(test_info, in_fmt, out_fmt): +def test_custom_ls_input_output(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): run_renderer( test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), @@ -281,10 +296,11 @@ def test_custom_ls_input_output(test_info, in_fmt, out_fmt): ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_custom_ls_input_binaural(test_info, in_layout, out_fmt, frame_size): +def test_custom_ls_input_binaural(record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), @@ -293,12 +309,13 @@ def test_custom_ls_input_binaural(test_info, in_layout, out_fmt, frame_size): ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input_binaural_headrotation( - test_info, in_layout, out_fmt, trj_file, frame_size + record_property, test_info, in_layout, out_fmt, trj_file, frame_size, get_mld, get_mld_lim ): run_renderer( test_info, @@ -312,10 +329,11 @@ def test_custom_ls_input_binaural_headrotation( """ Metadata / scene description input """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_metadata(test_info, in_fmt, out_fmt, frame_size): +def test_metadata(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( test_info, "META", @@ -328,10 +346,11 @@ def test_metadata(test_info, in_fmt, out_fmt, frame_size): """ non diegetic pan """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) -def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): +def test_non_diegetic_pan_static(record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim): run_renderer( test_info, in_fmt, @@ -340,10 +359,11 @@ def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) -def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan): +def test_non_diegetic_pan_ism_static(record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim): run_renderer( test_info, in_fmt, diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 1c8e0f2198..afb26ef7b8 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -94,6 +94,7 @@ def check_BE( def run_renderer( + record_property, test_info, in_fmt: str, out_fmt: str, @@ -109,6 +110,8 @@ def run_renderer( binary_suffix: str = "", frame_size: Optional[str] = "20ms", hrtf_file: Optional[str] = None, + get_mld = False, + mld_lim = 0, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -226,16 +229,26 @@ def run_renderer( # CUT creation mode will run a comparison with REF out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) - try: - ref, ref_fs = readfile(out_file_ref) - except FileNotFoundError: - pytest.fail( - f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" - ) + if get_mld: + output_differs, reason = cmp_pcm(out_file, out_file_ref, out_fmt, ref_fs, get_mld=get_mld, mld_lim=get_mld_lim) + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) + if output_differs: + pytest.fail(f"Output differs: ({reason})") + ) + else: + try: + ref, ref_fs = readfile(out_file_ref) + except FileNotFoundError: + pytest.fail( + f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" + ) - cut, cut_fs = readfile(out_file) + cut, cut_fs = readfile(out_file) - check_BE(test_info, ref, ref_fs, cut, cut_fs) + check_BE(test_info, ref, ref_fs, cut, cut_fs) # compare metadata files in case of MASA prerendering if "MASA" in str(out_fmt): -- GitLab From 7f4569ec18aba869efc2138f1db2e5676261ed02 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 1 Feb 2024 11:34:36 +0100 Subject: [PATCH 11/12] Fix test_renderer.py --- tests/renderer/test_renderer.py | 63 +++++++++++++++++++++++++++++++++ tests/renderer/utils.py | 1 - 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index c4717e31be..00ebc68158 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -48,10 +48,13 @@ from .utils import * @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -61,10 +64,13 @@ def test_ambisonics(record_property, test_info, in_fmt, out_fmt, frame_size, get @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -77,11 +83,14 @@ def test_ambisonics_binaural_headrotation( record_property, test_info, in_fmt, out_fmt, trj_file, frame_size, get_mld, get_mld_lim ): run_renderer( + record_property, test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -94,10 +103,13 @@ def test_ambisonics_binaural_headrotation( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_multichannel(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -110,10 +122,13 @@ def test_multichannel_binaural_static(record_property, test_info, in_fmt, out_fm pytest.skip("MONO or STEREO to Binaural rendering unsupported") run_renderer( + record_property, test_info, in_fmt, out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -129,11 +144,14 @@ def test_multichannel_binaural_headrotation( pytest.skip("MONO or STEREO to Binaural rendering unsupported") run_renderer( + record_property, test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -146,11 +164,14 @@ def test_multichannel_binaural_headrotation( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ism(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -165,11 +186,14 @@ def test_ism_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_ in_meta_files = None run_renderer( + record_property, test_info, in_fmt, out_fmt, in_meta_files=in_meta_files, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -185,12 +209,15 @@ def test_ism_binaural_headrotation(record_property, test_info, in_fmt, out_fmt, in_meta_files = None run_renderer( + record_property, test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), in_meta_files=in_meta_files, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -203,11 +230,14 @@ def test_ism_binaural_headrotation(record_property, test_info, in_fmt, out_fmt, @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_masa(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -220,11 +250,14 @@ def test_masa_binaural_static(record_property, test_info, in_fmt, out_fmt, frame pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") run_renderer( + record_property, test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -238,12 +271,15 @@ def test_masa_binaural_headrotation(record_property, test_info, in_fmt, out_fmt, pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") run_renderer( + record_property, test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -251,10 +287,13 @@ def test_masa_binaural_headrotation(record_property, test_info, in_fmt, out_fmt, @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) def test_masa_prerend(record_property, test_info, in_fmt, get_mld, get_mld_lim): run_renderer( + record_property, test_info, "META", "MASA2", metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -267,10 +306,13 @@ def test_masa_prerend(record_property, test_info, in_fmt, get_mld, get_mld_lim): @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input(record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -279,9 +321,12 @@ def test_custom_ls_input(record_property, test_info, in_layout, out_fmt, frame_s @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -290,9 +335,12 @@ def test_custom_ls_output(record_property, test_info, in_fmt, out_fmt, get_mld, @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): run_renderer( + record_property, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -302,10 +350,13 @@ def test_custom_ls_input_output(record_property, test_info, in_fmt, out_fmt, get @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input_binaural(record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -318,11 +369,14 @@ def test_custom_ls_input_binaural_headrotation( record_property, test_info, in_layout, out_fmt, trj_file, frame_size, get_mld, get_mld_lim ): run_renderer( + record_property, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -335,11 +389,14 @@ def test_custom_ls_input_binaural_headrotation( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_metadata(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, "META", out_fmt, metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -352,10 +409,13 @@ def test_metadata(record_property, test_info, in_fmt, out_fmt, frame_size, get_m @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_static(record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -365,10 +425,13 @@ def test_non_diegetic_pan_static(record_property, test_info, in_fmt, out_fmt, no @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_ism_static(record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan, + get_mld=get_mld, + mld_lim=get_mld_lim, ) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index afb26ef7b8..d7b0faf07f 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -237,7 +237,6 @@ def run_renderer( record_property("MLD",mld) if output_differs: pytest.fail(f"Output differs: ({reason})") - ) else: try: ref, ref_fs = readfile(out_file_ref) -- GitLab From 74f3cf6851b9ab5f7785e707e43d9344d3f5934b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 1 Feb 2024 11:54:05 +0100 Subject: [PATCH 12/12] Fix in test_renderer.py. Smoke tests without MLD. --- tests/renderer/test_renderer.py | 27 ++++++++++++++++++--------- tests/renderer/utils.py | 4 +++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 00ebc68158..9750cc11f4 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -449,12 +449,13 @@ def test_non_diegetic_pan_ism_static(record_property, test_info, in_fmt, out_fmt @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotzero( - test_info, in_fmt, out_fmt, trj_file + record_property, test_info, in_fmt, out_fmt, trj_file, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -476,11 +477,12 @@ def test_ambisonics_binaural_headrotation_refrotzero( # Note that reference rotation is supplied per 4 subframes; head rotation per subframe. @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_headrotation_refrotequal(test_info, in_fmt, out_fmt): +def test_ambisonics_binaural_headrotation_refrotequal(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -508,12 +510,13 @@ def test_ambisonics_binaural_headrotation_refrotequal(test_info, in_fmt, out_fmt @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveczero( - test_info, in_fmt, out_fmt, trj_file + record_property, test_info, in_fmt, out_fmt, trj_file, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -536,7 +539,7 @@ def test_ambisonics_binaural_headrotation_refveczero( # looking-direction of the head rotation and therefore compensates it (OTR=REF_VEC) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt): +def test_ambisonics_binaural_headrotation_refvecequal(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -545,6 +548,7 @@ def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt pytest.xfail("WIP : minor differences to be resolved") else: compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -570,7 +574,7 @@ def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt # in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): +def test_ambisonics_binaural_headrotation_refvec_rotating(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -579,6 +583,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating(test_info, in_fmt, out pytest.xfail("WIP : minor differences to be resolved") else: compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -608,12 +613,13 @@ def test_ambisonics_binaural_headrotation_refvec_rotating(test_info, in_fmt, out @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( - test_info, in_fmt, out_fmt + record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -642,12 +648,13 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( - test_info, in_fmt, out_fmt + record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -673,7 +680,7 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( # in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): +def test_multichannel_binaural_headrotation_refvec_rotating(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -681,6 +688,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, o pytest.skip("MONO or STEREO to Binaural rendering unsupported") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -707,7 +715,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, o # in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): +def test_ism_binaural_headrotation_refvec_rotating(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -717,6 +725,7 @@ def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): in_meta_files = None compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index d7b0faf07f..a70546ddec 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -260,9 +260,10 @@ def run_renderer( def compare_renderer_args( - test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict + record_property, test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict ): out_file_ref = run_renderer( + record_property, test_info, in_fmt, out_fmt, @@ -270,6 +271,7 @@ def compare_renderer_args( ) ref, ref_fs = readfile(out_file_ref) out_file_cut = run_renderer( + record_property, test_info, in_fmt, out_fmt, -- GitLab