diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index 34371deabc2e586bb1525a6b7259bbe329c2c51b..e3b6dd57ae87232b7be468a9b1d37147d0318413 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -25,11 +25,13 @@ if __name__ == '__main__': with open(csv_file,'w') as outfile: for testcase in testcases: - fulltestname = testcase.get('file') + "::" + testcase.get('name') - system_out = testcase.find(".//system-out") - mld_val = None - if system_out is not None: - for line in system_out.text.split('\n'): - if line.startswith('MLD:'): - mld_val = float(line.split()[1]) - outfile.write(fulltestname + ';' + str(mld_val)+'\n') + if testcase.find('.//skipped') == None: + if testcase.get('file') == None: + fulltestname = testcase.get('classname').replace('.','/') + ".py::" + testcase.get('name') + else: + fulltestname = testcase.get('file') + "::" + testcase.get('name') + if testcase.find('.//property') == None: + mld_val = None + else: + mld_val = testcase.find('.//property').get('value') # Currently MLD is the only set property. If more are added updates are needed here. + outfile.write(fulltestname + ';' + str(mld_val)+'\n') diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 817ac9c5026c249e8155353a9654bb2c659666cf..9db7ccd39e3750cc10cf21eb3134e4f226cd495d 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -12,7 +12,7 @@ import pyivastest import numpy as np -def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str): +def cmp_pcm(file1, file2, out_config, fs, get_mld = False, mld_lim = 0) -> (int, str): """ Compare 2 PCM files for bitexactness """ @@ -54,8 +54,12 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld = False) -> (int, str): print(first_msg) if get_mld: mld_msg = f"MLD: {cmp_result['MLD']}" - print(mld_msg) - return 1, "FAIL: Files have different content" + print(mld_msg) + if cmp_result['MLD'] <= mld_lim: + return 0, f"MLD: {cmp_result['MLD']} <= {mld_lim}" + else: + return 1, f"MLD: {cmp_result['MLD']} > {mld_lim}" + return 1, "Non-BE" if __name__ == "__main__": @@ -71,6 +75,7 @@ if __name__ == "__main__": ) parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs") parser.add_argument("--get_mld", action="store_true") + parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim") args = parser.parse_args() result, msg = cmp_pcm(**vars(args)) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 0614651653e14d33efe61fa28071ff9aec87b8d1..3c49f9f21c6357d749664186439c2bce806984f0 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -93,6 +93,7 @@ def check_and_makedir(dir_path): masa_metadata_audio_ndir_ntransportch_dtx_list, ) def test_masa_enc_dec( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -106,6 +107,7 @@ def test_masa_enc_dec( test_vector_path, output_mode, get_mld, + get_mld_lim, ): # Input parameters in_fs = 48 @@ -201,8 +203,13 @@ def test_masa_enc_dec( # Compare audio outputs pcmcmp_res, reason = cmp_pcm( - dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld + dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim ) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) # Fail if compare fails compare result if metacmp_res == False and pcmcmp_res != 0: @@ -219,11 +226,19 @@ def test_masa_enc_dec( filecmp_res = cmp(dec_output_ref, dec_output_dut) if filecmp_res == False: cmp_result, reason = cmp_pcm( - dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld + dec_output_dut, dec_output_ref, output_mode, int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim ) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) # Report compare result - assert cmp_result == 0, reason + if cmp_result != 0: + pytest.fail(reason) else: + if get_mld: + record_property("MLD","0") print("Comparison bit exact") # remove_output( diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 1dfda043d7d99730759751aa8aa495b84f637910..c5fe72ee505f51c23917f7cd801ebcd1cf84a311 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -125,6 +125,7 @@ def convert_test_string_to_tag(test_string): @pytest.mark.create_ref @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) def test_param_file_tests( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -137,6 +138,7 @@ def test_param_file_tests( keep_files, test_tag, get_mld, + get_mld_lim, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -325,10 +327,16 @@ def test_param_file_tests( ref_output_file = f"{reference_path}/param_file/dec/{output_file}" fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( - dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld + dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) + metadata_differs = False for md_file in md_out_files: dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") @@ -347,13 +355,13 @@ def test_param_file_tests( if output_differs or metadata_differs: msg = "Difference between ref and dut in " if output_differs and metadata_differs: - msg += "output and metadata" + msg += f"output ({reason}) and metadata" elif output_differs: - msg += "output only" + msg += f"output only ({reason})" elif metadata_differs: msg += "metadata only" - assert False, msg + pytest.fail(msg) # remove DUT output files when test result is OK (to save disk space) if not keep_files: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 422f87d9c7cebb502957120e6a909808288a8c08..484ada6c8aba66fadc3c868ff13e004888a0f494 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -75,6 +75,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("fs", sampling_rate_list) @pytest.mark.parametrize("gain_flag", gain_list) def test_sba_plc_system( + record_property, dut_decoder_frontend: DecoderFrontend, test_vector_path, reference_path, @@ -89,6 +90,7 @@ def test_sba_plc_system( fs, gain_flag, get_mld, + get_mld_lim, ): SID = 0 if dtx == '1' and ivas_br not in ['13200','16400','24400','32000', '64000']: @@ -105,6 +107,7 @@ def test_sba_plc_system( # dec sba_dec_plc( + record_property, dut_decoder_frontend, test_vector_path, reference_path, @@ -120,12 +123,14 @@ def test_sba_plc_system( gain_flag, keep_files, get_mld=get_mld, + get_mld_lim=get_mld_lim, ) ######################################################### # -------------------- test function -------------------- def sba_dec_plc( + record_property, decoder_frontend, test_vector_path, reference_path, @@ -141,6 +146,7 @@ def sba_dec_plc( gain_flag, keep_files, get_mld=False, + get_mld_lim=0, ): # ------------ run cmd ------------ @@ -191,10 +197,16 @@ def sba_dec_plc( # -------------- compare cmd -------------- fs = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld) + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) # report compare result - assert cmp_result == 0, reason + if cmp_result != 0: + pytest.fail(reason) # remove DUT output files when test result is OK (to save disk space) if not keep_files: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 3e84e6b2839d488a17afcd1a42e8ce015685ce1b..650bddb17d2cefc2ae4a68758cd6b193b409d59e 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -89,6 +89,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("fs", sample_rate_list) def test_pca_enc( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -101,6 +102,7 @@ def test_pca_enc( tag, fs, get_mld, + get_mld_lim, ): pca = True tag = tag + fs + "c" @@ -134,6 +136,7 @@ def test_pca_enc( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -149,6 +152,7 @@ def test_pca_enc( gain_flag, keep_files, get_mld=get_mld, + get_mld_lim=get_mld_lim, pca=pca, ) @@ -161,6 +165,7 @@ def test_pca_enc( @pytest.mark.parametrize("gain_flag", gain_list) @pytest.mark.parametrize("SID", SID_list) def test_sba_enc_system( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -178,6 +183,7 @@ def test_sba_enc_system( gain_flag, SID, get_mld, + get_mld_lim, ): if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: @@ -227,6 +233,7 @@ def test_sba_enc_system( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -241,7 +248,8 @@ def test_sba_enc_system( update_ref, gain_flag, keep_files, - get_mld=get_mld + get_mld=get_mld, + get_mld_lim=get_mld_lim ) @@ -249,6 +257,7 @@ def test_sba_enc_system( @pytest.mark.parametrize("ivas_br", ivas_br_HOA2) @pytest.mark.parametrize("tag", tag_list_HOA2) def test_spar_hoa2_enc_system( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -261,6 +270,7 @@ def test_spar_hoa2_enc_system( ivas_br, tag, get_mld, + get_mld_lim, ): fs = "48" dtx = "0" @@ -292,6 +302,7 @@ def test_spar_hoa2_enc_system( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -307,6 +318,7 @@ def test_spar_hoa2_enc_system( gain_flag, keep_files, get_mld=get_mld, + get_mld_lim=get_mld_lim, ) @@ -314,6 +326,7 @@ def test_spar_hoa2_enc_system( @pytest.mark.parametrize("ivas_br", ivas_br_HOA3) @pytest.mark.parametrize("tag", tag_list_HOA3) def test_spar_hoa3_enc_system( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -326,6 +339,7 @@ def test_spar_hoa3_enc_system( ivas_br, tag, get_mld, + get_mld_lim, ): fs = "48" dtx = "0" @@ -357,6 +371,7 @@ def test_spar_hoa3_enc_system( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -371,7 +386,8 @@ def test_spar_hoa3_enc_system( update_ref, gain_flag, keep_files, - get_mld=get_mld + get_mld=get_mld, + get_mld_lim=get_mld_lim, ) @@ -381,6 +397,7 @@ def test_spar_hoa3_enc_system( @pytest.mark.parametrize("tag", tag_list_bw_force) @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) def test_sba_enc_BWforce_system( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -395,6 +412,7 @@ def test_sba_enc_BWforce_system( tag, sample_rate_bw_idx, get_mld, + get_mld_lim, ): if dtx == "1" and ivas_br not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -432,6 +450,7 @@ def test_sba_enc_BWforce_system( # dec sba_dec( + record_property, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -447,6 +466,7 @@ def test_sba_enc_BWforce_system( gain_flag, keep_files, get_mld=get_mld, + get_mld_lim=get_mld_lim, ) @@ -594,6 +614,7 @@ def sba_enc( def sba_dec( + record_property, decoder_frontend, ref_decoder_frontend, reference_path, @@ -609,6 +630,7 @@ def sba_dec( gain_flag, keep_files, get_mld=False, + get_mld_lim=0, pca=False, ): # -------- run cmd ------------ @@ -663,10 +685,16 @@ def sba_dec( ) fs = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld) + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim ) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) # report compare result - assert cmp_result == 0, reason + if cmp_result != 0: + pytest.fail(reason) # remove DUT output files when test result is OK (to save disk space) if not keep_files: diff --git a/tests/conftest.py b/tests/conftest.py index cccae2e90662e37b3d34ec772c588074ce5b8d58..f5a2ee267ac2519de8ae3c7e444cce392467f3d5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -158,6 +158,12 @@ def pytest_addoption(parser): help="Run the MLD tool instead of just comparing for bitexactness", ) + parser.addoption( + "--mld-lim", + action="store", + help="MLD limit for comparison (default: 0)", + default="0", + ) @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -177,6 +183,12 @@ def get_mld(request): """ return request.config.option.mld +@pytest.fixture(scope="session", autouse=True) +def get_mld_lim(request): + """ + Return MLD limit for MLD comparison + """ + return float(request.config.getoption("--mld-lim")) @pytest.fixture(scope="session") def keep_files(request) -> bool: diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 1ddba2269586d99d02d0ec82567d23f4b0321b6b..9750cc11f4ee5b547c1e36912596a7b6bd8f340f 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -42,313 +42,396 @@ from .utils import * ############################################################################## """ Ambisonics """ - +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ambisonics(test_info, in_fmt, out_fmt, frame_size): +def test_ambisonics(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt, frame_size): +def test_ambisonics_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics_binaural_headrotation( - test_info, in_fmt, out_fmt, trj_file, frame_size + record_property, test_info, in_fmt, out_fmt, trj_file, frame_size, get_mld, get_mld_lim ): run_renderer( + record_property, test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) """ Multichannel """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_multichannel(test_info, in_fmt, out_fmt, frame_size): +def test_multichannel(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, frame_size): +def test_multichannel_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") run_renderer( + record_property, test_info, in_fmt, out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_multichannel_binaural_headrotation( - test_info, in_fmt, out_fmt, trj_file, frame_size + record_property, test_info, in_fmt, out_fmt, trj_file, frame_size, get_mld, get_mld_lim ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") run_renderer( + record_property, test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) """ ISM """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ism(test_info, in_fmt, out_fmt, frame_size): +def test_ism(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ism_binaural_static(test_info, in_fmt, out_fmt, frame_size): +def test_ism_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] except KeyError: in_meta_files = None run_renderer( + record_property, test_info, in_fmt, out_fmt, in_meta_files=in_meta_files, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, frame_size): +def test_ism_binaural_headrotation(record_property, test_info, in_fmt, out_fmt, trj_file, frame_size, get_mld, get_mld_lim): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] except KeyError: in_meta_files = None run_renderer( + record_property, test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), in_meta_files=in_meta_files, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) """ MASA """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_masa(test_info, in_fmt, out_fmt, frame_size): +def test_masa(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_masa_binaural_static(test_info, in_fmt, out_fmt, frame_size): +def test_masa_binaural_static(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") run_renderer( + record_property, test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_masa_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, frame_size): +def test_masa_binaural_headrotation(record_property, test_info, in_fmt, out_fmt, trj_file, frame_size, get_mld, get_mld_lim): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") run_renderer( + record_property, test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) -def test_masa_prerend(test_info, in_fmt): +def test_masa_prerend(record_property, test_info, in_fmt, get_mld, get_mld_lim): run_renderer( + record_property, test_info, "META", "MASA2", metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + get_mld=get_mld, + mld_lim=get_mld_lim, ) """ Custom loudspeaker layouts """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_custom_ls_input(test_info, in_layout, out_fmt, frame_size): +def test_custom_ls_input(record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) -def test_custom_ls_output(test_info, in_fmt, out_fmt): +def test_custom_ls_output(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) -def test_custom_ls_input_output(test_info, in_fmt, out_fmt): +def test_custom_ls_input_output(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): run_renderer( + record_property, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_custom_ls_input_binaural(test_info, in_layout, out_fmt, frame_size): +def test_custom_ls_input_binaural(record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input_binaural_headrotation( - test_info, in_layout, out_fmt, trj_file, frame_size + record_property, test_info, in_layout, out_fmt, trj_file, frame_size, get_mld, get_mld_lim ): run_renderer( + record_property, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) """ Metadata / scene description input """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -def test_metadata(test_info, in_fmt, out_fmt, frame_size): +def test_metadata(record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim): run_renderer( + record_property, test_info, "META", out_fmt, metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, ) """ non diegetic pan """ +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) -def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): +def test_non_diegetic_pan_static(record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan, + get_mld=get_mld, + mld_lim=get_mld_lim, ) +@pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) -def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan): +def test_non_diegetic_pan_ism_static(record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim): run_renderer( + record_property, test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan, + get_mld=get_mld, + mld_lim=get_mld_lim, ) @@ -366,12 +449,13 @@ def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pa @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotzero( - test_info, in_fmt, out_fmt, trj_file + record_property, test_info, in_fmt, out_fmt, trj_file, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -393,11 +477,12 @@ def test_ambisonics_binaural_headrotation_refrotzero( # Note that reference rotation is supplied per 4 subframes; head rotation per subframe. @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_headrotation_refrotequal(test_info, in_fmt, out_fmt): +def test_ambisonics_binaural_headrotation_refrotequal(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -425,12 +510,13 @@ def test_ambisonics_binaural_headrotation_refrotequal(test_info, in_fmt, out_fmt @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveczero( - test_info, in_fmt, out_fmt, trj_file + record_property, test_info, in_fmt, out_fmt, trj_file, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -453,7 +539,7 @@ def test_ambisonics_binaural_headrotation_refveczero( # looking-direction of the head rotation and therefore compensates it (OTR=REF_VEC) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt): +def test_ambisonics_binaural_headrotation_refvecequal(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -462,6 +548,7 @@ def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt pytest.xfail("WIP : minor differences to be resolved") else: compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -487,7 +574,7 @@ def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt # in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): +def test_ambisonics_binaural_headrotation_refvec_rotating(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -496,6 +583,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating(test_info, in_fmt, out pytest.xfail("WIP : minor differences to be resolved") else: compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -525,12 +613,13 @@ def test_ambisonics_binaural_headrotation_refvec_rotating(test_info, in_fmt, out @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( - test_info, in_fmt, out_fmt + record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -559,12 +648,13 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( - test_info, in_fmt, out_fmt + record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -590,7 +680,7 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( # in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): +def test_multichannel_binaural_headrotation_refvec_rotating(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -598,6 +688,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, o pytest.skip("MONO or STEREO to Binaural rendering unsupported") compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, @@ -624,7 +715,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, o # in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): +def test_ism_binaural_headrotation_refvec_rotating(record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -634,6 +725,7 @@ def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): in_meta_files = None compare_renderer_args( + record_property, test_info, in_fmt, out_fmt, diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 1c8e0f2198b619a37185ccc6e9a7003607a4acf2..a70546ddecb90dfa2f4a6538970db5757d286855 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -94,6 +94,7 @@ def check_BE( def run_renderer( + record_property, test_info, in_fmt: str, out_fmt: str, @@ -109,6 +110,8 @@ def run_renderer( binary_suffix: str = "", frame_size: Optional[str] = "20ms", hrtf_file: Optional[str] = None, + get_mld = False, + mld_lim = 0, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -226,16 +229,25 @@ def run_renderer( # CUT creation mode will run a comparison with REF out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) - try: - ref, ref_fs = readfile(out_file_ref) - except FileNotFoundError: - pytest.fail( - f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" - ) - - cut, cut_fs = readfile(out_file) - - check_BE(test_info, ref, ref_fs, cut, cut_fs) + if get_mld: + output_differs, reason = cmp_pcm(out_file, out_file_ref, out_fmt, ref_fs, get_mld=get_mld, mld_lim=get_mld_lim) + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) + if output_differs: + pytest.fail(f"Output differs: ({reason})") + else: + try: + ref, ref_fs = readfile(out_file_ref) + except FileNotFoundError: + pytest.fail( + f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" + ) + + cut, cut_fs = readfile(out_file) + + check_BE(test_info, ref, ref_fs, cut, cut_fs) # compare metadata files in case of MASA prerendering if "MASA" in str(out_fmt): @@ -248,9 +260,10 @@ def run_renderer( def compare_renderer_args( - test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict + record_property, test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict ): out_file_ref = run_renderer( + record_property, test_info, in_fmt, out_fmt, @@ -258,6 +271,7 @@ def compare_renderer_args( ) ref, ref_fs = readfile(out_file_ref) out_file_cut = run_renderer( + record_property, test_info, in_fmt, out_fmt, diff --git a/tests/test_26444.py b/tests/test_26444.py index fcbf2cfc7f167b18139fb0ff46945892e538ea72..ccf8d23fad876f40306052e04be14b1744fd46a4 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -88,7 +88,7 @@ def test_evs_26444(test_tag): result1 = filecmp.cmp(cmd1[0], cmd1[1]) result2 = True if not (result1 and result2): - assert False, "Output differs" + pytest.fail("Output differs") \ No newline at end of file diff --git a/tests/test_param_file_ltv.py b/tests/test_param_file_ltv.py index 055cd2415471b1aa5dd7e8c723783fb8b93cd127..d74a32aa76d8f5ec721e430270e7192ac2ecc388 100644 --- a/tests/test_param_file_ltv.py +++ b/tests/test_param_file_ltv.py @@ -126,6 +126,7 @@ def convert_test_string_to_tag(test_string): @pytest.mark.create_ref @pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys())) def test_param_file_tests( + record_property, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -138,6 +139,7 @@ def test_param_file_tests( keep_files, test_tag, get_mld, + get_mld_lim, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -326,10 +328,16 @@ def test_param_file_tests( ref_output_file = f"{reference_path}/param_file/dec/{output_file}" fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( - dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld + dut_output_file, ref_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) + if get_mld: + mld = 0 + if "MLD" in reason: + mld = float(reason.split(':')[1].split()[0]) + record_property("MLD",mld) + metadata_differs = False for md_file in md_out_files: dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") @@ -348,13 +356,12 @@ def test_param_file_tests( if output_differs or metadata_differs: msg = "Difference between ref and dut in " if output_differs and metadata_differs: - msg += "output and metadata" + msg += f"output ({reason}) and metadata" elif output_differs: - msg += "output only" + msg += f"output only ({reason})" elif metadata_differs: msg += "metadata only" - - assert False, msg + pytest.fail(msg) # remove DUT output files when test result is OK (to save disk space) if not keep_files: