diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index fe0d01034b0257caa8fc3bbb400ab85219d1e6c0..af937355ef4054fbf24561bf0aee9aa0285ee776 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -198,10 +198,12 @@ def tr_from_row(row, id_current, id_previous): try: if float(curr) > float(prev): curr += f" {ARROW_UP}" - td_tmpl_curr = TD_TMPL_INCREASE + # increase is bad -> mark in red, execpt for SNR for which it is good -> mark in green + td_tmpl_curr = TD_TMPL_REDUCE if c == "MIN_SSNR" else TD_TMPL_INCREASE elif float(curr) < float(prev): curr += f" {ARROW_DOWN}" - td_tmpl_curr = TD_TMPL_REDUCE + # reduce is good -> mark in green, execpt for SNR for which it is bad -> mark in red + td_tmpl_curr = TD_TMPL_INCREASE if c == "MIN_SSNR" else TD_TMPL_REDUCE except ValueError: # if we land here, one of the cells is not a number, this indicates a crash # or some error in the scripts, so mark with red as well @@ -258,7 +260,12 @@ def merge_and_cleanup_mld_reports( for col_pair in other_col_pairs: col_prev = col_pair[0] col_curr = col_pair[1] - diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + + try: + diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + except ValueError: + # can't make float from the column contents, probably NONE -> put to top + diff_other += float("inf") if diff_other > 0: diff = -1000000 diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index 09ddbc0a019dce056cebaf39210a19fe51b30716..2a1e9ed7551ebfde23d521514411c6105ae741db 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -54,6 +54,13 @@ TESTCASES = [ "Multi-channel 5_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 32kHz out, EXT out", "Multi-channel 7_1 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 16kHz out, EXT out", "Multi-channel 7_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, EXT out", + "SBA planar 2OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, EXT out", + "SBA planar FOA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out", + "SBA 2OA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out", + "SBA planar 3OA bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, random FER at 5%, EXT out", + "Multi-channel 5_1 at 512 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (CREND)", + "Multi-channel 5_1 at 64 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (FastConv)", + "Multi-channel 5_1 at 32 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (ParamBin)", ] diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 3139a32dc91242216dbc2f6a5ac7c01b38cfee25..e299047492f8a3d5922d2c1d750253556f07616c 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -109,10 +109,10 @@ def cmp_pcm( reason += f" > {mld_lim}" if get_ssnr: + reason += " - " for i, s in enumerate(cmp_result["SSNR"], start=1): msg = f"Channel {i} SSNR: {s}" - reason += " - " + msg - print(msg) + reason += msg + " - " if get_odg: for n in range(nchannels): diff --git a/tests/conftest.py b/tests/conftest.py index c5d827569f63a4361ead420df5e83d07357f11c1..b75b3ba9979b4bafc1646d70e8e91b446fc0f374 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -872,7 +872,6 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: props["MIN_SSNR_CHANNEL"] = min_ssnr_channel elif prop == "ODG": odgs = re.findall(ODG_PATTERN, text_to_parse) - print(odgs) min_odg = min(odgs) min_odg_channel = odgs.index(min_odg) props["MIN_ODG"] = min_odg diff --git a/tests/constants.py b/tests/constants.py index e6ea1c1589fbfb9e16d71776737599ac0bfa25e5..2f1746e76b33953a0db82cb47cdbfc313d077cce 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -9,4 +9,4 @@ MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" ODG_PATTERN_PQEVALAUDIO = r"Objective Difference Grade: (-*\d*\.\d*)" ODG_PATTERN = r"ODG: (-*\d*\.\d*)" -SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)" +SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[-*\d\.]*)" diff --git a/tests/hrtf_binary_loading/utils.py b/tests/hrtf_binary_loading/utils.py index 04b25640a7842d46e5dac76e86e308bc22d67e63..2ba8b39e6b43225b30e280648cec15a58f32be28 100644 --- a/tests/hrtf_binary_loading/utils.py +++ b/tests/hrtf_binary_loading/utils.py @@ -369,6 +369,7 @@ def compare_renderer_vs_renderer_with_binary_hrir( hrtf_file_path = hrtf_file_dir.joinpath(hrtf_file.format(48)) ref_out = run_renderer( None, + list(), test_info, in_fmt, out_fmt, @@ -385,6 +386,7 @@ def compare_renderer_vs_renderer_with_binary_hrir( ) cut_out = run_renderer( None, + list(), test_info, in_fmt, out_fmt, @@ -458,6 +460,7 @@ def compare_renderer_with_binary_fix_vs_with_binary_float( ref_out = run_renderer( None, + list(), test_info, in_fmt, out_fmt, @@ -475,6 +478,7 @@ def compare_renderer_with_binary_fix_vs_with_binary_float( ) cut_out = run_renderer( None, + list(), test_info, in_fmt, out_fmt, diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index e89096ceed6ff4a76620601766a1aaf49cd9a990..31a125f9a81e2d56d61322151d0342638cf932f0 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -1,36 +1,35 @@ #!/usr/bin/env python3 """ - (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ -from pathlib import Path import pytest from .constants import ( @@ -52,6 +51,7 @@ from .constants import ( METADATA_SCENES_TO_TEST, ) from .utils import run_renderer, compare_renderer_args, test_info +from ..conftest import props_to_record ############################################################################## # Bit-exactness tests @@ -67,10 +67,20 @@ from .utils import run_renderer, compare_renderer_args, test_info @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -78,6 +88,8 @@ def test_ambisonics( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -86,10 +98,20 @@ def test_ambisonics( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics_binaural_static( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -97,6 +119,8 @@ def test_ambisonics_binaural_static( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -107,6 +131,7 @@ def test_ambisonics_binaural_static( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics_binaural_headrotation( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -114,9 +139,12 @@ def test_ambisonics_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -125,22 +153,35 @@ def test_ambisonics_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) +@pytest.mark.skip(reason="Not supported for BASOP code currently") @pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -@pytest.mark.parametrize("aeid", ["1", "0"]) +@pytest.mark.parametrize("aeid", ["1", "0"]) def test_dynamic_acoustic_environment( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim, aeid + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + aeid, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") - + run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -148,24 +189,38 @@ def test_dynamic_acoustic_environment( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, config_file=rend_config_path, - aeid=aeid, + aeid=aeid, ) + +@pytest.mark.skip(reason="Not supported for BASOP code currently") @pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_dynamic_acoustic_environment_file( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") aeid = TEST_VECTOR_DIR.joinpath(f"aeid1.txt") - + run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -173,8 +228,10 @@ def test_dynamic_acoustic_environment_file( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, config_file=rend_config_path, - aeid=aeid, + aeid=aeid, ) @@ -186,10 +243,20 @@ def test_dynamic_acoustic_environment_file( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_multichannel( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -197,6 +264,8 @@ def test_multichannel( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -205,13 +274,23 @@ def test_multichannel( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_multichannel_binaural_static( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -219,6 +298,8 @@ def test_multichannel_binaural_static( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -229,6 +310,7 @@ def test_multichannel_binaural_static( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_multichannel_binaural_headrotation( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -236,12 +318,15 @@ def test_multichannel_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, + get_odg, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -250,6 +335,8 @@ def test_multichannel_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -261,10 +348,20 @@ def test_multichannel_binaural_headrotation( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ism( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -273,6 +370,8 @@ def test_ism( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -281,7 +380,16 @@ def test_ism( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ism_binaural_static( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -290,6 +398,7 @@ def test_ism_binaural_static( run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -298,6 +407,8 @@ def test_ism_binaural_static( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -308,6 +419,7 @@ def test_ism_binaural_static( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ism_binaural_headrotation( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -315,6 +427,8 @@ def test_ism_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, + get_odg, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -323,6 +437,7 @@ def test_ism_binaural_headrotation( run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -332,6 +447,8 @@ def test_ism_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -343,10 +460,20 @@ def test_ism_binaural_headrotation( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_masa( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -355,6 +482,8 @@ def test_masa( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -363,13 +492,23 @@ def test_masa( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_masa_binaural_static( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -378,6 +517,8 @@ def test_masa_binaural_static( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -388,6 +529,7 @@ def test_masa_binaural_static( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_masa_binaural_headrotation( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -395,12 +537,15 @@ def test_masa_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, + get_odg, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -410,14 +555,26 @@ def test_masa_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @pytest.mark.create_ref @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) -def test_masa_prerend(record_property, test_info, in_fmt, get_mld, get_mld_lim): +def test_masa_prerend( + record_property, + props_to_record, + test_info, + in_fmt, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, +): run_renderer( record_property, + props_to_record, test_info, "META", "MASA2", @@ -425,6 +582,8 @@ def test_masa_prerend(record_property, test_info, in_fmt, get_mld, get_mld_lim): binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -436,10 +595,20 @@ def test_masa_prerend(record_property, test_info, in_fmt, get_mld, get_mld_lim): @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input( - record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, @@ -447,6 +616,8 @@ def test_custom_ls_input( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -454,16 +625,27 @@ def test_custom_ls_input( @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -471,16 +653,27 @@ def test_custom_ls_output( @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -489,10 +682,20 @@ def test_custom_ls_input_output( @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input_binaural( - record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, @@ -500,6 +703,8 @@ def test_custom_ls_input_binaural( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -510,6 +715,7 @@ def test_custom_ls_input_binaural( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input_binaural_headrotation( record_property, + props_to_record, test_info, in_layout, out_fmt, @@ -517,9 +723,12 @@ def test_custom_ls_input_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, @@ -528,6 +737,8 @@ def test_custom_ls_input_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -539,10 +750,20 @@ def test_custom_ls_input_binaural_headrotation( @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_metadata( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, "META", out_fmt, @@ -551,6 +772,8 @@ def test_metadata( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -562,10 +785,20 @@ def test_metadata( @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_static( - record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -573,6 +806,8 @@ def test_non_diegetic_pan_static( binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -581,10 +816,20 @@ def test_non_diegetic_pan_static( @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_ism_static( - record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -592,6 +837,8 @@ def test_non_diegetic_pan_ism_static( binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -609,13 +856,22 @@ def test_non_diegetic_pan_ism_static( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotzero( - record_property, test_info, in_fmt, out_fmt, trj_file, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + get_mld, + get_mld_lim, + get_ssnr, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -638,13 +894,14 @@ def test_ambisonics_binaural_headrotation_refrotzero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotequal( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -672,13 +929,21 @@ def test_ambisonics_binaural_headrotation_refrotequal( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveczero( - record_property, test_info, in_fmt, out_fmt, trj_file, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + get_mld, + get_mld_lim, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -702,7 +967,7 @@ def test_ambisonics_binaural_headrotation_refveczero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvecequal( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -713,6 +978,7 @@ def test_ambisonics_binaural_headrotation_refvecequal( else: compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -739,7 +1005,7 @@ def test_ambisonics_binaural_headrotation_refvecequal( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -750,6 +1016,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( else: compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -779,13 +1046,14 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -814,13 +1082,14 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -847,7 +1116,7 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) def test_multichannel_binaural_headrotation_refvec_rotating( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -857,6 +1126,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating( compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -884,7 +1154,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) def test_ism_binaural_headrotation_refvec_rotating( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -896,6 +1166,7 @@ def test_ism_binaural_headrotation_refvec_rotating( compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 82f5a6130231f37c53be39df58dd5eb9487d688e..d94f75bf8fce939bca6d471912581f97e1a4bf18 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import filecmp @@ -40,7 +40,6 @@ from typing import Dict, Optional, Union import numpy as np import pytest -import re from .compare_audio import compare_audio_arrays from .constants import ( @@ -56,7 +55,7 @@ from .constants import ( sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile from ..cmp_pcm import cmp_pcm -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN +from ..conftest import parse_properties # fixture returns test information, enabling per-testcase SNR @@ -74,6 +73,7 @@ def run_cmd(cmd, env=None): f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" ) + def run_isar_ext_rend_cmd(cmd, env=None): logging.info(f"\nRunning ISAR EXT REND command\n{' '.join(cmd)}\n") try: @@ -83,6 +83,7 @@ def run_isar_ext_rend_cmd(cmd, env=None): f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" ) + def run_ivas_isar_enc_cmd(cmd, env=None): logging.info(f"\nRunning IVAS ISAR encoder command\n{' '.join(cmd)}\n") try: @@ -92,6 +93,7 @@ def run_ivas_isar_enc_cmd(cmd, env=None): f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" ) + def run_ivas_isar_dec_cmd(cmd, env=None): logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") try: @@ -100,7 +102,8 @@ def run_ivas_isar_dec_cmd(cmd, env=None): raise SystemError( f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" ) - + + def run_isar_post_rend_cmd(cmd, env=None): logging.info(f"\nRunning ISAR post renderer command\n{' '.join(cmd)}\n") try: @@ -108,7 +111,8 @@ def run_isar_post_rend_cmd(cmd, env=None): except sp.CalledProcessError as e: raise SystemError( f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" - ) + ) + def check_BE( test_info, @@ -141,6 +145,7 @@ def check_BE( def run_renderer( record_property, + props_to_record, test_info, in_fmt: str, out_fmt: str, @@ -159,6 +164,9 @@ def run_renderer( get_mld=False, mld_lim=0, get_mld_lim=0, + abs_tol=0, + get_ssnr=False, + get_odg=False, aeid: Optional[Union[Path, int]] = None, ) -> str: # prepare arguments and filepaths @@ -291,47 +299,26 @@ def run_renderer( # CUT creation mode will run a comparison with REF out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) - if get_mld: - # see constants.py - ref_fs = int(cmd[10]) * 1000 - output_differs, reason = cmp_pcm( - out_file, - out_file_ref, - out_fmt, - ref_fs, - get_mld=get_mld, - mld_lim=get_mld_lim, - ) - mld = 0 - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - max_diff = 0 - if output_differs: - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) - - if output_differs: - pytest.fail(f"Output differs: ({reason})") - else: - try: - ref, ref_fs = readfile(out_file_ref) - except FileNotFoundError: - pytest.fail( - f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" - ) - - cut, cut_fs = readfile(out_file) + # see constants.py + ref_fs = int(cmd[10]) * 1000 + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_ssnr, + ) - [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) - - if diff_found : - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" - ) + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + if output_differs: + pytest.fail(f"Output differs: ({reason})") # compare metadata files in case of MASA prerendering if "MASA" in str(out_fmt): @@ -344,10 +331,17 @@ def run_renderer( def compare_renderer_args( - record_property, test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs: Dict, + cut_kwargs: Dict, ): out_file_ref = run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -356,6 +350,7 @@ def compare_renderer_args( ref, ref_fs = readfile(out_file_ref) out_file_cut = run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -366,4 +361,4 @@ def compare_renderer_args( if diff_found : pytest.fail( f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" - ) \ No newline at end of file + ) diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py index 64e7773a60f3d2be8fdfdc3dc02d1e7aa0fdc90c..61bb168a2fe0cc14a4c7adcce7522328db278e02 100644 --- a/tests/test_be_for_jbm_neutral_dly_profile.py +++ b/tests/test_be_for_jbm_neutral_dly_profile.py @@ -197,4 +197,4 @@ def test_be_for_jbm_neutral_dly_profile( test_start_offset_ms=JBM_NEUTRAL_DELAY_MS, ) if not cmp_result["bitexact"]: - pytest.fail("Difference between no jbm and zero-delay jbm decoding!") + pytest.fail(f"Difference between no jbm and zero-delay jbm decoding found! Max abs diff: {cmp_result['max_abs_diff']}")