From 864ee1b679eafcd321bed8b70954387da8d73314 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 4 Dec 2024 15:16:07 +0100 Subject: [PATCH 01/40] Updates for binauralizing audio --- .../test_param_file.py | 160 +++++++++++++++++- tests/codec_be_on_mr_nonselection/test_sba.py | 6 + tests/conftest.py | 18 ++ tests/renderer/utils.py | 33 ++-- 4 files changed, 197 insertions(+), 20 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 9fe43a40b2..ae3afa6d27 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -52,6 +52,7 @@ from tests.constants import ( SCRIPTS_DIR, MAX_ENC_DIFF, ) +import tests.renderer.utils as rend PASSTHROUGH_CONF = [ (r"[\s\S]*", "EXT"), # Any input with EXT output is pass-through @@ -82,6 +83,47 @@ VALID_DEC_OUTPUT_CONF = [ "EXT", ] +INPUT_FMT = [ + (r"-stereo", "STEREO"), + (r"-sba\s?[-+]?1", "FOA"), + (r"-sba\s?[-+]?2", "HOA2"), + (r"-sba\s?[-+]?3", "HOA3"), + (r"-mc\s5_1\s", "5_1"), + (r"-mc\s7_1\s", "7_1"), + (r"-mc\s5_1_2", "5_1_2"), + (r"-mc\s5_1_4", "5_1_4"), + (r"-mc\s7_1_4", "7_1_4"), + (r"-ism\s?[-+]?1", "ISM1"), + (r"-ism\s?[-+]?2", "ISM2"), + (r"-ism\s?[-+]?3", "ISM3"), + (r"-ism\s?[-+]?4", "ISM4"), + (r"-masa\s?1", "MASA1"), + (r"-masa\s?2", "MASA2"), + (r"-ism_masa\s?1\s?1", "OMASA_1_1"), + (r"-ism_masa\s?2\s?1", "OMASA_2_1"), + (r"-ism_masa\s?3\s?1", "OMASA_3_1"), + (r"-ism_masa\s?4\s?1", "OMASA_4_1"), + (r"-ism_masa\s?1\s?2", "OMASA_1_2"), + (r"-ism_masa\s?2\s?2", "OMASA_2_2"), + (r"-ism_masa\s?3\s?2", "OMASA_3_2"), + (r"-ism_masa\s?4\s?2", "OMASA_4_2"), + (r"-ism_sba\s?1\s?1", "OSBA_1_1"), + (r"-ism_sba\s?2\s?1", "OSBA_2_1"), + (r"-ism_sba\s?3\s?1", "OSBA_3_1"), + (r"-ism_sba\s?4\s?1", "OSBA_4_1"), + (r"-ism_sba\s?1\s?2", "OSBA_1_2"), + (r"-ism_sba\s?2\s?2", "OSBA_2_2"), + (r"-ism_sba\s?3\s?2", "OSBA_3_2"), + (r"-ism_sba\s?4\s?2", "OSBA_4_2"), +] + +TWO_CHAN_FMT = [ + "STEREO", + "BINAURAL", + "BINAURAL_ROOM_IR", + "BINAURAL_ROOM_REVERB", +] + PARAM_FILE_ID = ( "stv" if PARAM_FILE.stem == "self_test" @@ -152,6 +194,7 @@ def convert_test_string_to_tag(test_string): # hack to have stv/ltv/evs in the test name @pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID]) def test_param_file_tests( + test_info, props_to_record, encoder_only, decoder_only, @@ -174,6 +217,7 @@ def test_param_file_tests( get_enc_stats, get_odg, compare_to_input, + peaq_binaural, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -393,9 +437,113 @@ def test_param_file_tests( decoder_only, ) + # Reconstruct output file names for comparison + dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" + ref_output_file = f"{reference_path}/param_file/dec/{output_file}" + + # Check if we need to render to stereo or binaural for PEAQ comparison + if peaq_binaural: + + dut_output_file_binaural = dut_output_file[0:-4]+".BINAURAL.wav" + ref_output_file_binaural = ref_output_file[0:-4]+".BINAURAL.wav" + + # Disable tracefile supported comparison -- no tracefile available for input + tracefile_dec = "" + + record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property + in_meta_files = [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.csv\b', enc_opts)] # All .csv files in enc_opts are metadata files. + n_obj = len(in_meta_files) + if n_obj > 0 and output_config == "EXT": + out_meta_files = [f"{dut_output_file}.{i}.csv" for i in range(0,n_obj)] + else: + out_meta_files = None + metadata_input = None # Todo: This can be used for OMASA/OSBA rendering + trj_file = findstr(r'-t\s+(\S+)', dec_opts) + non_diegetic_pan = findstr(r'-non_diegetic_pan\s+(\S+)', dec_opts) + name_extension = None + refrot_file = findstr(r'-rf\s+(\S+)', dec_opts) + rot_tmp_file = findstr(r'-rvf\s+(\S+)', dec_opts) + refveclev_file = None + refvec_file = None + if "-otr ref_vec".upper() in dec_opts.upper(): + refvec_file = rot_tmp_file + if "-otr ref_vec_lev".upper() in dec_opts.upper(): + refveclev_file = rot_tmp_file + config_file = findstr(r'-render_config\s+(\S+)', dec_opts) + binary_suffix = "" + frame_size = findstr(r'-fr\s+(\S+)', dec_opts) + hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) + aeid = findstr(r'-aeid\s+(\S+)', dec_opts) + + # Find input format + in_fmt = [ + (a, b) + for (a, b) in INPUT_FMT + if re.search(a, enc_opts) + ][0][1] + + if not output_config in TWO_CHAN_FMT: + # Render output + output_reformat = "BINAURAL" + + rend.run_renderer( + record_property, + props_to_record, + test_info, + output_config, + output_reformat, + metadata_input, + out_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=dut_output_file, + out_file=dut_output_file_binaural, + ) + # Use binauralized file for comparison + dut_output_file = dut_output_file_binaural + + if not in_fmt in TWO_CHAN_FMT: + # Render input + out_fmt = output_config + if not out_fmt in TWO_CHAN_FMT: + out_fmt = "BINAURAL" + + rend.run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + metadata_input, + in_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=testv_file, + out_file=ref_output_file_binaural, + ) + # Use binauralized file for comparison + ref_output_file = ref_output_file_binaural + + if update_ref in [0, 2]: - dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" - ref_output_file = f"{reference_path}/param_file/dec/{output_file}" # set to false per default even if this is no JBM case - makes later check for failure easier tracefile_last_rtp_numbers_differ = False @@ -786,3 +934,11 @@ def get_expected_md_files(ref_output_file, enc_opts, output_config): pass return md_files + +def findstr(exp, s, one_element=False): + result = re.findall(exp, s) + if len(result) == 0: + return None + if one_element: + return result[0] + return result \ No newline at end of file diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index e8ae17f55b..9e52a8119d 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -114,6 +114,7 @@ def test_pca_enc( get_odg, get_enc_stats, compare_to_input, + peaq_binaural, ): pca = True bitrate = "256000" @@ -224,6 +225,7 @@ def test_sba_enc_system( get_odg, get_enc_stats, compare_to_input, + peaq_binaural, ): plc_pattern = None @@ -385,6 +387,7 @@ def test_spar_hoa2_enc_system( get_odg, get_enc_stats, compare_to_input, + peaq_binaural, ): sampling_rate = "48" pca = False @@ -520,6 +523,7 @@ def test_spar_hoa3_enc_system( get_odg, get_enc_stats, compare_to_input, + peaq_binaural, ): sampling_rate = "48" pca = False @@ -653,6 +657,7 @@ def test_sba_enc_BWforce_system( get_odg, get_enc_stats, compare_to_input, + peaq_binaural, ): sid = 0 plc_pattern = None @@ -806,6 +811,7 @@ def test_sba_plc_system( get_odg, get_enc_stats, compare_to_input, + peaq_binaural, ): sid = 0 pca = False diff --git a/tests/conftest.py b/tests/conftest.py index 989f080c90..8a882bf75d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -273,6 +273,13 @@ def pytest_addoption(parser): default=False, ) + parser.addoption( + "--peaq_binaural", + action="store_true", + help="Compare output to the input file using PEAQ. If input or output is not STEREO or BINAURAL, input and/or output is binauralized using the external renderer", + default=False, + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): """ @@ -383,6 +390,10 @@ def dut_encoder_path(request) -> str: return path +# fixture returns test information, enabling per-testcase SNR +@pytest.fixture +def test_info(request): + return request class EncoderFrontend: def __init__(self, path, enc_type, record_property, timeout=None) -> None: @@ -1007,6 +1018,13 @@ def compare_to_input(request) -> bool: """ return request.config.getoption("--compare_to_input") +@pytest.fixture(scope="session", autouse=True) +def peaq_binaural(request) -> bool: + """ + Return value of cmdl param --peaq_binaural + """ + return request.config.getoption("--peaq_binaural") + def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial") if config.option.param_file: diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index f987fe4229..ed313f12d3 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -60,12 +60,6 @@ from ..cmp_pcm import cmp_pcm from ..conftest import parse_properties -# fixture returns test information, enabling per-testcase SNR -@pytest.fixture -def test_info(request): - return request - - def run_cmd(cmd, env=None): logging.info(f"\nRunning command\n{' '.join(cmd)}\n") try: @@ -171,6 +165,8 @@ def run_renderer( get_ssnr=False, get_odg=False, aeid: Optional[Union[Path, int]] = None, + in_file = None, + out_file = None, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -241,19 +237,20 @@ def run_renderer( else: FORMAT_TO_FILE = FORMAT_TO_FILE_SMOKETEST - if metadata_input is not None: - in_file = metadata_input - in_name = metadata_input.stem - elif not isinstance(in_fmt, str): - in_file = FORMAT_TO_FILE[in_fmt.stem] - in_name = in_fmt.stem - else: - in_file = FORMAT_TO_FILE[in_fmt] - in_name = in_fmt - - out_file_stem = f"{in_name}_to_{out_name}{trj_name}{non_diegetic_pan}{refrot_name}{refvec_name}{refveclev_name}{config_name}{framing_name}{hrtf_file_name}{name_extension}{aeid_name}.wav" + if in_file is None: + if metadata_input is not None: + in_file = metadata_input + in_name = metadata_input.stem + elif not isinstance(in_fmt, str): + in_file = FORMAT_TO_FILE[in_fmt.stem] + in_name = in_fmt.stem + else: + in_file = FORMAT_TO_FILE[in_fmt] + in_name = in_fmt - out_file = str(output_path_base.joinpath(out_file_stem)) + if out_file is None: + out_file_stem = f"{in_name}_to_{out_name}{trj_name}{non_diegetic_pan}{refrot_name}{refvec_name}{refveclev_name}{config_name}{framing_name}{hrtf_file_name}{name_extension}{aeid_name}.wav" + out_file = str(output_path_base.joinpath(out_file_stem)) cmd = RENDERER_CMD[:] cmd[2] = str(in_file) -- GitLab From e05ed06ae77beeaddba98afaa5142d33e744f1ce Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 4 Dec 2024 15:20:24 +0100 Subject: [PATCH 02/40] Use _ref for external renderer --- tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index ae3afa6d27..18555e292b 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -470,7 +470,7 @@ def test_param_file_tests( if "-otr ref_vec_lev".upper() in dec_opts.upper(): refveclev_file = rot_tmp_file config_file = findstr(r'-render_config\s+(\S+)', dec_opts) - binary_suffix = "" + binary_suffix = "_ref" frame_size = findstr(r'-fr\s+(\S+)', dec_opts) hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) aeid = findstr(r'-aeid\s+(\S+)', dec_opts) -- GitLab From 28307d4c07540e584b64b6655c698976619a33f0 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 09:10:33 +0100 Subject: [PATCH 03/40] Add check_and_makedir for binauralization --- tests/codec_be_on_mr_nonselection/test_param_file.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 18555e292b..3f4794be86 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -486,6 +486,8 @@ def test_param_file_tests( # Render output output_reformat = "BINAURAL" + check_and_makedir(str(Path(dut_output_file_binaural).parent)) + rend.run_renderer( record_property, props_to_record, @@ -517,6 +519,8 @@ def test_param_file_tests( if not out_fmt in TWO_CHAN_FMT: out_fmt = "BINAURAL" + check_and_makedir(str(Path(ref_output_file_binaural).parent)) + rend.run_renderer( record_property, props_to_record, -- GitLab From e4a45fe171fadb3129cd5fadbb60be39beb99b34 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 09:26:14 +0100 Subject: [PATCH 04/40] Allow ref not to exist for peaq_binaural, since comparing to input --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8a882bf75d..abc55a7f5f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -970,7 +970,7 @@ def reference_path(request) -> str: path = str(path.resolve()) - if request.config.option.update_ref == "0" and not request.config.option.compare_to_input: + if request.config.option.update_ref == "0" and not request.config.option.compare_to_input and not request.config.option.peaq_binaural: if not os.path.isdir(path): raise FileNotFoundError( f"REF path {path} not found!\nPlease generate the references, first!\n!" -- GitLab From 31ae0bc994167b5093cf7eb9f32519b954e3b830 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 09:40:00 +0100 Subject: [PATCH 05/40] Fix for when input is already 2-channels --- tests/codec_be_on_mr_nonselection/test_param_file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 3f4794be86..89a111c148 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -545,6 +545,8 @@ def test_param_file_tests( ) # Use binauralized file for comparison ref_output_file = ref_output_file_binaural + else: + ref_output_file = testv_file if update_ref in [0, 2]: -- GitLab From a9cb8a289f9c44fc08435201aa7997e4db0dc0c0 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 13:17:08 +0100 Subject: [PATCH 06/40] Add mono to PEAQ supported outputs --- tests/codec_be_on_mr_nonselection/test_param_file.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 89a111c148..2fb703fd61 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -117,7 +117,8 @@ INPUT_FMT = [ (r"-ism_sba\s?4\s?2", "OSBA_4_2"), ] -TWO_CHAN_FMT = [ +PEAQ_SUPPORTED_FMT = [ + "MONO", "STEREO", "BINAURAL", "BINAURAL_ROOM_IR", @@ -482,7 +483,7 @@ def test_param_file_tests( if re.search(a, enc_opts) ][0][1] - if not output_config in TWO_CHAN_FMT: + if not output_config in PEAQ_SUPPORTED_FMT: # Render output output_reformat = "BINAURAL" @@ -513,10 +514,10 @@ def test_param_file_tests( # Use binauralized file for comparison dut_output_file = dut_output_file_binaural - if not in_fmt in TWO_CHAN_FMT: + if not in_fmt in PEAQ_SUPPORTED_FMT: # Render input out_fmt = output_config - if not out_fmt in TWO_CHAN_FMT: + if not out_fmt in PEAQ_SUPPORTED_FMT: out_fmt = "BINAURAL" check_and_makedir(str(Path(ref_output_file_binaural).parent)) -- GitLab From 628fe5f7f86056bef74e17ce6efad79e09b0128a Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 16:26:49 +0100 Subject: [PATCH 07/40] Add fix for EXT output and IVAS_rend -fs --- .../test_param_file.py | 18 ++++++++++++------ tests/renderer/utils.py | 2 ++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 2fb703fd61..4a825e1d22 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -251,6 +251,7 @@ def test_param_file_tests( fs = enc_split.pop() sampling_rate = int(fs) bitrate = enc_split.pop() + in_sr = sampling_rate # bitrate can be a filename: remove leading "../" if bitrate.startswith("../"): @@ -392,6 +393,7 @@ def test_param_file_tests( output_file = dec_split.pop() bitstream_file_dec = dec_split.pop() sampling_rate = int(dec_split.pop()) + out_sr = sampling_rate if len(dec_split) > 0: output_config = dec_split.pop() if output_config.upper() not in VALID_DEC_OUTPUT_CONF: @@ -451,11 +453,19 @@ def test_param_file_tests( # Disable tracefile supported comparison -- no tracefile available for input tracefile_dec = "" + # Find input format + in_fmt = [ + (a, b) + for (a, b) in INPUT_FMT + if re.search(a, enc_opts) + ][0][1] + record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property in_meta_files = [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.csv\b', enc_opts)] # All .csv files in enc_opts are metadata files. n_obj = len(in_meta_files) if n_obj > 0 and output_config == "EXT": out_meta_files = [f"{dut_output_file}.{i}.csv" for i in range(0,n_obj)] + output_config = in_fmt # For EXT, output format is the same as the input format else: out_meta_files = None metadata_input = None # Todo: This can be used for OMASA/OSBA rendering @@ -476,12 +486,6 @@ def test_param_file_tests( hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) aeid = findstr(r'-aeid\s+(\S+)', dec_opts) - # Find input format - in_fmt = [ - (a, b) - for (a, b) in INPUT_FMT - if re.search(a, enc_opts) - ][0][1] if not output_config in PEAQ_SUPPORTED_FMT: # Render output @@ -510,6 +514,7 @@ def test_param_file_tests( aeid, in_file=dut_output_file, out_file=dut_output_file_binaural, + sr=in_sr, ) # Use binauralized file for comparison dut_output_file = dut_output_file_binaural @@ -543,6 +548,7 @@ def test_param_file_tests( aeid, in_file=testv_file, out_file=ref_output_file_binaural, + sr=out_sr, ) # Use binauralized file for comparison ref_output_file = ref_output_file_binaural diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index ed313f12d3..65b35359d4 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -167,6 +167,7 @@ def run_renderer( aeid: Optional[Union[Path, int]] = None, in_file = None, out_file = None, + sr = 48, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -257,6 +258,7 @@ def run_renderer( cmd[4] = str(in_fmt) cmd[6] = str(out_file) cmd[8] = str(out_fmt) + cmd[10] = str(sr) if test_info.config.option.create_ref: cmd[0] += BIN_SUFFIX_MERGETARGET -- GitLab From 4f90b5615569c21980b924b179eb31903c354bd6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 16:53:50 +0100 Subject: [PATCH 08/40] Fix for when input/output is supported by PEAQ but not the same --- tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 4a825e1d22..6d2f6faf65 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -519,7 +519,7 @@ def test_param_file_tests( # Use binauralized file for comparison dut_output_file = dut_output_file_binaural - if not in_fmt in PEAQ_SUPPORTED_FMT: + if not in_fmt in PEAQ_SUPPORTED_FMT or in_fmt != output_config: # Render input out_fmt = output_config if not out_fmt in PEAQ_SUPPORTED_FMT: -- GitLab From 4aa2bc0e3602e3061e7588002cf7292988030902 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 17:00:15 +0100 Subject: [PATCH 09/40] Add manual job for testing peaq-binaural --- .gitlab-ci.yml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 546c729556..adfea1dbeb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,7 @@ variables: - 'check-float-reference' - 'check-clipping' - 'test-branch-vs-input-passthrough' + - 'peaq-binaural' GIT_CLEAN_FLAGS: -ffdxq TESTCASE_TIMEOUT_STV_SANITIZERS: 180 @@ -79,6 +80,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' variables: IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-binaural' + variables: + IVAS_PIPELINE_NAME: 'PEAQ comparison vs input on binauralized or mono signal: $CI_COMMIT_BRANCH' stages: - .pre @@ -280,8 +284,11 @@ stages: - if: $MANUAL_PIPELINE_TYPE == 'check-float-reference' when: never - if: $MANUAL_PIPELINE_TYPE == 'check-clipping' + when: never - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' when: never + - if: $MANUAL_PIPELINE_TYPE == 'peaq-binaural' + when: never - when: on_success .rules-merge-request: @@ -1722,6 +1729,42 @@ test-branch-vs-input-passthrough: - report-junit.xml +peaq-binaural: + tags: + - ivas-linux + stage: compare + timeout: "30 minutes" # TBD + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-binaural' + allow_failure: + exit_codes: + - 123 + script: + - *print-common-info + - make -j + + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --peaq-binaural --odg || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py report-junit.xml report.csv + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered."; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + artifacts: + name: "peaq-binaural--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 4 mos + paths: + - report-junit.xml + - report.html + - report.csv + expose_as: "peaq-binaural results" + reports: + junit: + - report-junit.xml + + # --------------------------------------------------------------- # Scheduled jobs on main # --------------------------------------------------------------- -- GitLab From b3805147dd99b2a12c192fe1b28b26a3df752f9e Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 17:08:58 +0100 Subject: [PATCH 10/40] Fix mixup of sampling rates for input/output --- tests/codec_be_on_mr_nonselection/test_param_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 6d2f6faf65..ab4cc73950 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -514,7 +514,7 @@ def test_param_file_tests( aeid, in_file=dut_output_file, out_file=dut_output_file_binaural, - sr=in_sr, + sr=out_sr, ) # Use binauralized file for comparison dut_output_file = dut_output_file_binaural @@ -548,7 +548,7 @@ def test_param_file_tests( aeid, in_file=testv_file, out_file=ref_output_file_binaural, - sr=out_sr, + sr=in_sr, ) # Use binauralized file for comparison ref_output_file = ref_output_file_binaural -- GitLab From 1db09bed4db7c1aa775db6d05a4a8f123e24d36c Mon Sep 17 00:00:00 2001 From: norvell Date: Thu, 5 Dec 2024 16:13:09 +0000 Subject: [PATCH 11/40] Fix syntax error: --peaq_binaural --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index adfea1dbeb..aaa9484cb2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1744,7 +1744,7 @@ peaq-binaural: - make -j - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --peaq-binaural --odg || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --peaq_binaural --odg || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit.xml report.csv - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi -- GitLab From 0b0300f89731518b8b2f46176fb9fd63832ff0dd Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 22:00:14 +0100 Subject: [PATCH 12/40] Fixes for EVS mono --- .../test_param_file.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index ab4cc73950..53e351b06e 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -465,9 +465,12 @@ def test_param_file_tests( n_obj = len(in_meta_files) if n_obj > 0 and output_config == "EXT": out_meta_files = [f"{dut_output_file}.{i}.csv" for i in range(0,n_obj)] - output_config = in_fmt # For EXT, output format is the same as the input format else: out_meta_files = None + if output_config == "EXT": + output_config = in_fmt + if output_config == "": + output_config = "MONO" # EVS mono metadata_input = None # Todo: This can be used for OMASA/OSBA rendering trj_file = findstr(r'-t\s+(\S+)', dec_opts) non_diegetic_pan = findstr(r'-non_diegetic_pan\s+(\S+)', dec_opts) @@ -488,7 +491,7 @@ def test_param_file_tests( if not output_config in PEAQ_SUPPORTED_FMT: - # Render output + # Render output to BINAURAL output_reformat = "BINAURAL" check_and_makedir(str(Path(dut_output_file_binaural).parent)) @@ -516,14 +519,14 @@ def test_param_file_tests( out_file=dut_output_file_binaural, sr=out_sr, ) + # Update output_config to rendered format + output_config = output_reformat # Use binauralized file for comparison dut_output_file = dut_output_file_binaural - if not in_fmt in PEAQ_SUPPORTED_FMT or in_fmt != output_config: - # Render input + if in_fmt != output_config: + # Render input to match output_config out_fmt = output_config - if not out_fmt in PEAQ_SUPPORTED_FMT: - out_fmt = "BINAURAL" check_and_makedir(str(Path(ref_output_file_binaural).parent)) -- GitLab From cc4bb425aec6e8e9beab72c72351d0ca73a72d02 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 5 Dec 2024 22:42:19 +0100 Subject: [PATCH 13/40] Handle non-diegetic panning --- tests/codec_be_on_mr_nonselection/test_param_file.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 53e351b06e..55a9ff1351 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -115,6 +115,7 @@ INPUT_FMT = [ (r"-ism_sba\s?2\s?2", "OSBA_2_2"), (r"-ism_sba\s?3\s?2", "OSBA_3_2"), (r"-ism_sba\s?4\s?2", "OSBA_4_2"), + (r".*", "MONO"), ] PEAQ_SUPPORTED_FMT = [ @@ -474,6 +475,8 @@ def test_param_file_tests( metadata_input = None # Todo: This can be used for OMASA/OSBA rendering trj_file = findstr(r'-t\s+(\S+)', dec_opts) non_diegetic_pan = findstr(r'-non_diegetic_pan\s+(\S+)', dec_opts) + if non_diegetic_pan is not None: + output_config = "STEREO" name_extension = None refrot_file = findstr(r'-rf\s+(\S+)', dec_opts) rot_tmp_file = findstr(r'-rvf\s+(\S+)', dec_opts) @@ -951,8 +954,8 @@ def get_expected_md_files(ref_output_file, enc_opts, output_config): return md_files -def findstr(exp, s, one_element=False): - result = re.findall(exp, s) +def findstr(exp, s, one_element=True): + result = [SCRIPTS_DIR.joinpath(x) for x in re.findall(exp, s)] if len(result) == 0: return None if one_element: -- GitLab From 41b67025575dbf23dfb4990f559aa21f167d545f Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 07:42:08 +0100 Subject: [PATCH 14/40] Fix for MASA re-rendering --- .../test_param_file.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 55a9ff1351..b9288c5b4c 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -396,8 +396,8 @@ def test_param_file_tests( sampling_rate = int(dec_split.pop()) out_sr = sampling_rate if len(dec_split) > 0: - output_config = dec_split.pop() - if output_config.upper() not in VALID_DEC_OUTPUT_CONF: + output_config = dec_split.pop().upper() + if output_config not in VALID_DEC_OUTPUT_CONF: if not output_config.endswith(".txt"): # must be EVS tests with additional parameters - put param back dec_split.append(output_config) @@ -462,12 +462,15 @@ def test_param_file_tests( ][0][1] record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property - in_meta_files = [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.csv\b', enc_opts)] # All .csv files in enc_opts are metadata files. + in_meta_files = [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.csv\b', enc_opts)] # All .csv files in enc_opts are ISM metadata files. + in_meta_files = in_meta_files + [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.met\b', enc_opts)] # All .met files in enc_opts are MASA metadata files. n_obj = len(in_meta_files) - if n_obj > 0 and output_config == "EXT": - out_meta_files = [f"{dut_output_file}.{i}.csv" for i in range(0,n_obj)] - else: - out_meta_files = None + out_meta_files = None + if output_config == "EXT": + if "ISM" in in_fmt and n_obj > 0: + out_meta_files = [f"{dut_output_file}.{i}.csv" for i in range(0,n_obj)] + if "MASA" in in_fmt: + out_meta_files = [f"{dut_output_file}.met"] if output_config == "EXT": output_config = in_fmt if output_config == "": @@ -489,7 +492,9 @@ def test_param_file_tests( config_file = findstr(r'-render_config\s+(\S+)', dec_opts) binary_suffix = "_ref" frame_size = findstr(r'-fr\s+(\S+)', dec_opts) - hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) + # hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) + hrtf_file = None # Default HRTFs used for binaural rendering of output + aeid = findstr(r'-aeid\s+(\S+)', dec_opts) -- GitLab From e2e47c5ad734ccf7a185221e98d6770428f00a4b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 08:10:39 +0100 Subject: [PATCH 15/40] Fixes for ref_vec_lev tracking and custom ls --- .../codec_be_on_mr_nonselection/test_param_file.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index b9288c5b4c..803fa18c86 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -396,8 +396,8 @@ def test_param_file_tests( sampling_rate = int(dec_split.pop()) out_sr = sampling_rate if len(dec_split) > 0: - output_config = dec_split.pop().upper() - if output_config not in VALID_DEC_OUTPUT_CONF: + output_config = dec_split.pop() + if output_config.upper() not in VALID_DEC_OUTPUT_CONF: if not output_config.endswith(".txt"): # must be EVS tests with additional parameters - put param back dec_split.append(output_config) @@ -485,10 +485,11 @@ def test_param_file_tests( rot_tmp_file = findstr(r'-rvf\s+(\S+)', dec_opts) refveclev_file = None refvec_file = None - if "-otr ref_vec".upper() in dec_opts.upper(): - refvec_file = rot_tmp_file if "-otr ref_vec_lev".upper() in dec_opts.upper(): refveclev_file = rot_tmp_file + else: + if "-otr ref_vec".upper() in dec_opts.upper(): + refvec_file = rot_tmp_file config_file = findstr(r'-render_config\s+(\S+)', dec_opts) binary_suffix = "_ref" frame_size = findstr(r'-fr\s+(\S+)', dec_opts) @@ -498,7 +499,7 @@ def test_param_file_tests( aeid = findstr(r'-aeid\s+(\S+)', dec_opts) - if not output_config in PEAQ_SUPPORTED_FMT: + if not output_config.upper() in PEAQ_SUPPORTED_FMT: # Render output to BINAURAL output_reformat = "BINAURAL" @@ -532,7 +533,7 @@ def test_param_file_tests( # Use binauralized file for comparison dut_output_file = dut_output_file_binaural - if in_fmt != output_config: + if in_fmt.upper() != output_config.upper(): # Render input to match output_config out_fmt = output_config -- GitLab From 399fabe9b91c67376d08ffc8cb200ae634cb6390 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 11:07:03 +0100 Subject: [PATCH 16/40] Refactoring/cleanup and add support for test_sba.py --- .../test_param_file.py | 121 ++------------- tests/codec_be_on_mr_nonselection/test_sba.py | 76 ++++++++- tests/renderer/constants.py | 8 + tests/renderer/utils.py | 146 ++++++++++++++++++ 4 files changed, 239 insertions(+), 112 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 803fa18c86..201bd55784 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -118,14 +118,6 @@ INPUT_FMT = [ (r".*", "MONO"), ] -PEAQ_SUPPORTED_FMT = [ - "MONO", - "STEREO", - "BINAURAL", - "BINAURAL_ROOM_IR", - "BINAURAL_ROOM_REVERB", -] - PARAM_FILE_ID = ( "stv" if PARAM_FILE.stem == "self_test" @@ -448,9 +440,6 @@ def test_param_file_tests( # Check if we need to render to stereo or binaural for PEAQ comparison if peaq_binaural: - dut_output_file_binaural = dut_output_file[0:-4]+".BINAURAL.wav" - ref_output_file_binaural = ref_output_file[0:-4]+".BINAURAL.wav" - # Disable tracefile supported comparison -- no tracefile available for input tracefile_dec = "" @@ -462,111 +451,21 @@ def test_param_file_tests( ][0][1] record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property - in_meta_files = [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.csv\b', enc_opts)] # All .csv files in enc_opts are ISM metadata files. - in_meta_files = in_meta_files + [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.met\b', enc_opts)] # All .met files in enc_opts are MASA metadata files. - n_obj = len(in_meta_files) - out_meta_files = None - if output_config == "EXT": - if "ISM" in in_fmt and n_obj > 0: - out_meta_files = [f"{dut_output_file}.{i}.csv" for i in range(0,n_obj)] - if "MASA" in in_fmt: - out_meta_files = [f"{dut_output_file}.met"] - if output_config == "EXT": - output_config = in_fmt - if output_config == "": - output_config = "MONO" # EVS mono - metadata_input = None # Todo: This can be used for OMASA/OSBA rendering - trj_file = findstr(r'-t\s+(\S+)', dec_opts) - non_diegetic_pan = findstr(r'-non_diegetic_pan\s+(\S+)', dec_opts) - if non_diegetic_pan is not None: - output_config = "STEREO" - name_extension = None - refrot_file = findstr(r'-rf\s+(\S+)', dec_opts) - rot_tmp_file = findstr(r'-rvf\s+(\S+)', dec_opts) - refveclev_file = None - refvec_file = None - if "-otr ref_vec_lev".upper() in dec_opts.upper(): - refveclev_file = rot_tmp_file - else: - if "-otr ref_vec".upper() in dec_opts.upper(): - refvec_file = rot_tmp_file - config_file = findstr(r'-render_config\s+(\S+)', dec_opts) - binary_suffix = "_ref" - frame_size = findstr(r'-fr\s+(\S+)', dec_opts) - # hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) - hrtf_file = None # Default HRTFs used for binaural rendering of output - - aeid = findstr(r'-aeid\s+(\S+)', dec_opts) - - - if not output_config.upper() in PEAQ_SUPPORTED_FMT: - # Render output to BINAURAL - output_reformat = "BINAURAL" - check_and_makedir(str(Path(dut_output_file_binaural).parent)) - - rend.run_renderer( - record_property, - props_to_record, - test_info, - output_config, - output_reformat, - metadata_input, - out_meta_files, - trj_file, - non_diegetic_pan, - name_extension, - refrot_file, - refvec_file, - refveclev_file, - config_file, - binary_suffix, - frame_size, - hrtf_file, - aeid, - in_file=dut_output_file, - out_file=dut_output_file_binaural, - sr=out_sr, - ) - # Update output_config to rendered format - output_config = output_reformat - # Use binauralized file for comparison - dut_output_file = dut_output_file_binaural - - if in_fmt.upper() != output_config.upper(): - # Render input to match output_config - out_fmt = output_config - - check_and_makedir(str(Path(ref_output_file_binaural).parent)) - - rend.run_renderer( + (ref_output_file, dut_output_file) = rend.binauralize_input_and_output( record_property, props_to_record, test_info, + testv_file, + dut_output_file, + ref_output_file, in_fmt, - out_fmt, - metadata_input, - in_meta_files, - trj_file, - non_diegetic_pan, - name_extension, - refrot_file, - refvec_file, - refveclev_file, - config_file, - binary_suffix, - frame_size, - hrtf_file, - aeid, - in_file=testv_file, - out_file=ref_output_file_binaural, - sr=in_sr, - ) - # Use binauralized file for comparison - ref_output_file = ref_output_file_binaural - else: - ref_output_file = testv_file - + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, + ) if update_ref in [0, 2]: diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 9e52a8119d..998e578291 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -44,7 +44,7 @@ from ..conftest import parse_properties from ..cmp_stats_files import cmp_stats_files from ..constants import TESTV_DIR, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF from tests.testconfig import use_ltv - +import tests.renderer.utils as rend tag_list = ["ltvFOA" if use_ltv else "stvFOA"] tag_list_HOA2 = ["ltvHOA2" if use_ltv else "stv2OA"] @@ -79,6 +79,8 @@ tag_list_plc = tag_list plc_patterns = ["PLperc12mblen5", "PLperc40mblen50", "PLperc42mblen2"] ivas_br_plc = ["13200", "16400", "32000", "64000", "96000", "256000"] +# SBA order to IVAS_rend format table +SBA_FORMAT={1: "FOA", 2: "HOA2", 3: "HOA3"} def check_and_makedir(dir_path): if not os.path.exists(dir_path): @@ -92,6 +94,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("sampling_rate", sample_rate_list) def test_pca_enc( + test_info, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -137,6 +140,8 @@ def test_pca_enc( else: assert 0 + input_config = SBA_FORMAT(abs(int(sba_order))) + if not decoder_only: sba_enc( dut_encoder_frontend, @@ -164,6 +169,7 @@ def test_pca_enc( if not encoder_only: sba_dec( + test_info, props_to_record, test_vector_path, dut_decoder_frontend, @@ -176,6 +182,7 @@ def test_pca_enc( dtx, sid, max_bw, + input_config, output_config, update_ref, gain_flag, @@ -189,6 +196,7 @@ def test_pca_enc( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, + peaq_binaural=peaq_binaural, ) @@ -199,6 +207,7 @@ def test_pca_enc( @pytest.mark.parametrize("gain_flag", gain_list) @pytest.mark.parametrize("sid", SID_list) def test_sba_enc_system( + test_info, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -272,6 +281,8 @@ def test_sba_enc_system( else: cut_gain = "1.0" + input_config = SBA_FORMAT(abs(int(sba_order))) + if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, @@ -334,6 +345,7 @@ def test_sba_enc_system( if not encoder_only: sba_dec( + test_info, props_to_record, test_vector_path, dut_decoder_frontend, @@ -346,6 +358,7 @@ def test_sba_enc_system( dtx, sid, max_bw, + input_config, output_config, update_ref, gain_flag, @@ -359,12 +372,14 @@ def test_sba_enc_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, + peaq_binaural=peaq_binaural, ) @pytest.mark.parametrize("bitrate", ivas_br_HOA2) @pytest.mark.parametrize("tag", tag_list_HOA2) def test_spar_hoa2_enc_system( + test_info, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -408,6 +423,8 @@ def test_spar_hoa2_enc_system( else: assert 0 + input_config = SBA_FORMAT(abs(int(sba_order))) + if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, @@ -470,6 +487,7 @@ def test_spar_hoa2_enc_system( if not encoder_only: sba_dec( + test_info, props_to_record, test_vector_path, dut_decoder_frontend, @@ -482,6 +500,7 @@ def test_spar_hoa2_enc_system( dtx, sid, max_bw, + input_config, output_config, update_ref, gain_flag, @@ -495,12 +514,14 @@ def test_spar_hoa2_enc_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, + peaq_binaural=peaq_binaural, ) @pytest.mark.parametrize("bitrate", ivas_br_HOA3) @pytest.mark.parametrize("tag", tag_list_HOA3) def test_spar_hoa3_enc_system( + test_info, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -544,6 +565,8 @@ def test_spar_hoa3_enc_system( else: assert 0 + input_config = SBA_FORMAT(abs(int(sba_order))) + if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, @@ -600,6 +623,7 @@ def test_spar_hoa3_enc_system( if not encoder_only: sba_dec( + test_info, props_to_record, test_vector_path, dut_decoder_frontend, @@ -612,6 +636,7 @@ def test_spar_hoa3_enc_system( dtx, sid, max_bw, + input_config, output_config, update_ref, gain_flag, @@ -625,6 +650,7 @@ def test_spar_hoa3_enc_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, + peaq_binaural=peaq_binaural, ) @@ -633,6 +659,7 @@ def test_spar_hoa3_enc_system( @pytest.mark.parametrize("tag", tag_list_bw_force) @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) def test_sba_enc_BWforce_system( + test_info, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -688,6 +715,8 @@ def test_sba_enc_BWforce_system( else: assert 0 + input_config = SBA_FORMAT(abs(int(sba_order))) + if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( dut_encoder_frontend, @@ -750,6 +779,7 @@ def test_sba_enc_BWforce_system( if not encoder_only: sba_dec( + test_info, props_to_record, test_vector_path, dut_decoder_frontend, @@ -762,6 +792,7 @@ def test_sba_enc_BWforce_system( dtx, sid, max_bw, + input_config, output_config, update_ref, gain_flag, @@ -775,6 +806,7 @@ def test_sba_enc_BWforce_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, + peaq_binaural=peaq_binaural, ) @@ -785,6 +817,7 @@ def test_sba_enc_BWforce_system( @pytest.mark.parametrize("sampling_rate", sample_rate_list) @pytest.mark.parametrize("gain_flag", gain_list) def test_sba_plc_system( + test_info, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -854,6 +887,8 @@ def test_sba_plc_system( else: cut_gain = "1.0" + input_config = SBA_FORMAT(abs(int(sba_order))) + if not decoder_only: sba_enc( dut_encoder_frontend, @@ -881,6 +916,7 @@ def test_sba_plc_system( if not encoder_only: sba_dec( + test_info, props_to_record, test_vector_path, dut_decoder_frontend, @@ -893,6 +929,7 @@ def test_sba_plc_system( dtx, sid, max_bw, + input_config, output_config, update_ref, gain_flag, @@ -906,6 +943,7 @@ def test_sba_plc_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, + peaq_binaural=peaq_binaural, ) @@ -1036,6 +1074,7 @@ def sba_enc( def sba_dec( + test_info, props_to_record, test_vector_path, dut_decoder_frontend, @@ -1048,6 +1087,7 @@ def sba_dec( dtx, sid, max_bw, + input_config, output_config, update_ref, gain_flag, @@ -1061,6 +1101,7 @@ def sba_dec( get_ssnr=False, get_odg=False, compare_to_input=False, + peaq_binaural=False, ): dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" ref_pkt_dir = f"{reference_path}/sba_bs/pkt" @@ -1132,8 +1173,41 @@ def sba_dec( if compare_to_input: # If comparing to input, set input as reference and allow different length ref_out_file = f"{test_vector_path}/{tag}.wav" + if gain_flag != -1: + ref_out_file = f"{test_vector_path}/{tag}_Gain{gain_flag}.wav" + else: + ref_out_file = f"{test_vector_path}/{tag}.wav" + allow_differing_lengths = True + + if peaq_binaural: + + enc_opts = "" + dec_opts = "" + if gain_flag != -1: + input_file = f"{test_vector_path}/{tag}_Gain{gain_flag}.wav" + else: + input_file = f"{test_vector_path}/{tag}.wav" + in_fmt = input_config + in_sr = sampling_rate + out_sr = sampling_rate + + (ref_out_file, dut_out_file) = rend.binauralize_input_and_output( + dut_decoder_frontend.record_property, + props_to_record, + test_info, + input_file, + dut_out_file, + ref_out_file, + in_fmt, + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, + ) + sampling_rate_Hz = int(sampling_rate) * 1000 cmp_result, reason = cmp_pcm( dut_out_file, diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index b7326a0a78..4800cd3655 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -340,3 +340,11 @@ HR_TRAJECTORIES_TO_TEST = [ """ Frame Size """ FRAMING_TO_TEST = ["5ms", "20ms"] + +PEAQ_SUPPORTED_FMT = [ + "MONO", + "STEREO", + "BINAURAL", + "BINAURAL_ROOM_IR", + "BINAURAL_ROOM_REVERB", +] \ No newline at end of file diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 65b35359d4..fe1a8d1413 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -40,6 +40,8 @@ from typing import Dict, Optional, Union import numpy as np import pytest +import re +import errno from .compare_audio import compare_audio_arrays from .constants import ( @@ -52,6 +54,7 @@ from .constants import ( FORMAT_TO_FILE_SMOKETEST, RENDERER_CMD, BIN_SUFFIX_MERGETARGET, + PEAQ_SUPPORTED_FMT, ) sys.path.append(SCRIPTS_DIR) @@ -373,3 +376,146 @@ def compare_renderer_args( pytest.fail( f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" ) + +def binauralize_input_and_output( + record_property, + props_to_record, + test_info, + input_file, + dut_output_file, + ref_output_file, + in_fmt, + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, +): + + # File names for binauralized input, if needed + dut_output_file_binaural = dut_output_file[0:-4]+".BINAURAL.wav" + ref_output_file_binaural = ref_output_file[0:-4]+".BINAURAL.wav" + + # Identify metadata + in_meta_files = [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.csv\b', enc_opts)] # All .csv files in enc_opts are ISM metadata files. + in_meta_files = in_meta_files + [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.met\b', enc_opts)] # All .met files in enc_opts are MASA metadata files. + n_obj = len(in_meta_files) + out_meta_files = None + if output_config == "EXT": + if "ISM" in in_fmt and n_obj > 0: + out_meta_files = [f"{dut_output_file}.{i}.csv" for i in range(0,n_obj)] + if "MASA" in in_fmt: + out_meta_files = [f"{dut_output_file}.met"] + if output_config == "EXT": + output_config = in_fmt + if output_config == "": + output_config = "MONO" # EVS mono + metadata_input = None # Todo: This can be used for OMASA/OSBA rendering + + # Identify headtracking and orientation trajectories + trj_file = findstr(r'-t\s+(\S+)', dec_opts) + non_diegetic_pan = findstr(r'-non_diegetic_pan\s+(\S+)', dec_opts) + if non_diegetic_pan is not None: + output_config = "STEREO" + name_extension = None + refrot_file = findstr(r'-rf\s+(\S+)', dec_opts) + rot_tmp_file = findstr(r'-rvf\s+(\S+)', dec_opts) + refveclev_file = None + refvec_file = None + if "-otr ref_vec_lev".upper() in dec_opts.upper(): + refveclev_file = rot_tmp_file + else: + if "-otr ref_vec".upper() in dec_opts.upper(): + refvec_file = rot_tmp_file + + # Rendering configuration + config_file = findstr(r'-render_config\s+(\S+)', dec_opts) + binary_suffix = "_ref" + frame_size = findstr(r'-fr\s+(\S+)', dec_opts) + # hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) + hrtf_file = None # Default HRTFs used for binaural rendering of output + + aeid = findstr(r'-aeid\s+(\S+)', dec_opts) + + if not output_config.upper() in PEAQ_SUPPORTED_FMT: + # Render output to BINAURAL + output_reformat = "BINAURAL" + + check_and_makedir(str(Path(dut_output_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + output_config, + output_reformat, + metadata_input, + out_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=dut_output_file, + out_file=dut_output_file_binaural, + sr=out_sr, + ) + # Update output_config to rendered format + output_config = output_reformat + else: + dut_output_file_binaural = dut_output_file + + if in_fmt.upper() != output_config.upper(): + # Render input to match output_config + out_fmt = output_config + + check_and_makedir(str(Path(ref_output_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + metadata_input, + in_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=input_file, + out_file=ref_output_file_binaural, + sr=in_sr, + ) + else: + ref_output_file_binaural = ref_output_file + return (ref_output_file_binaural, dut_output_file_binaural) + +def findstr(exp, s, one_element=True): + result = [SCRIPTS_DIR.joinpath(x) for x in re.findall(exp, s)] + if len(result) == 0: + return None + if one_element: + return result[0] + return result + +def check_and_makedir(dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise # raises the error again -- GitLab From ffeff9e27ebd2348ba6938ba7a3d5bb05ef3ae11 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 11:23:59 +0100 Subject: [PATCH 17/40] Fix for the case of non-rendered input file --- tests/renderer/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index fe1a8d1413..0b3f5110c9 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -501,7 +501,7 @@ def binauralize_input_and_output( sr=in_sr, ) else: - ref_output_file_binaural = ref_output_file + ref_output_file_binaural = input_file return (ref_output_file_binaural, dut_output_file_binaural) def findstr(exp, s, one_element=True): -- GitLab From ddea277e6731eac65bc264bc16240ec96a8d62ac Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 11:56:07 +0100 Subject: [PATCH 18/40] Fix syntax error and small cleanup --- .../test_param_file.py | 69 ++++++++++--------- tests/codec_be_on_mr_nonselection/test_sba.py | 14 ++-- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 201bd55784..3508caa37c 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -433,42 +433,12 @@ def test_param_file_tests( decoder_only, ) - # Reconstruct output file names for comparison - dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" - ref_output_file = f"{reference_path}/param_file/dec/{output_file}" - - # Check if we need to render to stereo or binaural for PEAQ comparison - if peaq_binaural: - - # Disable tracefile supported comparison -- no tracefile available for input - tracefile_dec = "" - - # Find input format - in_fmt = [ - (a, b) - for (a, b) in INPUT_FMT - if re.search(a, enc_opts) - ][0][1] - - record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property - - (ref_output_file, dut_output_file) = rend.binauralize_input_and_output( - record_property, - props_to_record, - test_info, - testv_file, - dut_output_file, - ref_output_file, - in_fmt, - output_config, - enc_opts, - dec_opts, - in_sr, - out_sr, - ) - if update_ref in [0, 2]: + # Output file names for comparison + dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" + ref_output_file = f"{reference_path}/param_file/dec/{output_file}" + # set to false per default even if this is no JBM case - makes later check for failure easier tracefile_last_rtp_numbers_differ = False @@ -501,6 +471,37 @@ def test_param_file_tests( else: ref_file = ref_output_file + # Check if we need to render to stereo or binaural for PEAQ comparison + if peaq_binaural: + + allow_differing_lengths = True + # Disable tracefile supported comparison -- no tracefile available for input + tracefile_dec = "" + + # Find input format + in_fmt = [ + (a, b) + for (a, b) in INPUT_FMT + if re.search(a, enc_opts) + ][0][1] + + record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property + + (ref_file, dut_output_file) = rend.binauralize_input_and_output( + record_property, + props_to_record, + test_info, + testv_file, + dut_output_file, + ref_output_file, + in_fmt, + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, + ) + fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( ref_file, diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 998e578291..2c0791ea78 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -140,7 +140,7 @@ def test_pca_enc( else: assert 0 - input_config = SBA_FORMAT(abs(int(sba_order))) + input_config = SBA_FORMAT[abs(int(sba_order))] if not decoder_only: sba_enc( @@ -281,7 +281,7 @@ def test_sba_enc_system( else: cut_gain = "1.0" - input_config = SBA_FORMAT(abs(int(sba_order))) + input_config = SBA_FORMAT[abs(int(sba_order))] if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -423,7 +423,7 @@ def test_spar_hoa2_enc_system( else: assert 0 - input_config = SBA_FORMAT(abs(int(sba_order))) + input_config = SBA_FORMAT[abs(int(sba_order))] if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -565,7 +565,7 @@ def test_spar_hoa3_enc_system( else: assert 0 - input_config = SBA_FORMAT(abs(int(sba_order))) + input_config = SBA_FORMAT[abs(int(sba_order))] if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -715,7 +715,7 @@ def test_sba_enc_BWforce_system( else: assert 0 - input_config = SBA_FORMAT(abs(int(sba_order))) + input_config = SBA_FORMAT[abs(int(sba_order))] if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( @@ -887,7 +887,7 @@ def test_sba_plc_system( else: cut_gain = "1.0" - input_config = SBA_FORMAT(abs(int(sba_order))) + input_config = SBA_FORMAT[abs(int(sba_order))] if not decoder_only: sba_enc( @@ -1182,7 +1182,7 @@ def sba_dec( if peaq_binaural: - + allow_differing_lengths = True enc_opts = "" dec_opts = "" if gain_flag != -1: -- GitLab From e05675af012148580e9719005be663f60f5f76d8 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 14:52:58 +0100 Subject: [PATCH 19/40] Fix SBA gain scaling for input signal --- .../test_param_file.py | 62 +++++++++---------- tests/codec_be_on_mr_nonselection/test_sba.py | 47 +++++++++----- 2 files changed, 62 insertions(+), 47 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 3508caa37c..5f0dedced7 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -439,6 +439,37 @@ def test_param_file_tests( dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" ref_output_file = f"{reference_path}/param_file/dec/{output_file}" + # Check if we need to render to stereo or binaural for PEAQ comparison + if peaq_binaural: + + allow_differing_lengths = True + # Disable tracefile supported comparison -- no tracefile available for input + tracefile_dec = "" + + # Find input format + in_fmt = [ + (a, b) + for (a, b) in INPUT_FMT + if re.search(a, enc_opts) + ][0][1] + + record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property + + (ref_output_file, dut_output_file) = rend.binauralize_input_and_output( + record_property, + props_to_record, + test_info, + testv_file, + dut_output_file, + ref_output_file, + in_fmt, + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, + ) + # set to false per default even if this is no JBM case - makes later check for failure easier tracefile_last_rtp_numbers_differ = False @@ -471,37 +502,6 @@ def test_param_file_tests( else: ref_file = ref_output_file - # Check if we need to render to stereo or binaural for PEAQ comparison - if peaq_binaural: - - allow_differing_lengths = True - # Disable tracefile supported comparison -- no tracefile available for input - tracefile_dec = "" - - # Find input format - in_fmt = [ - (a, b) - for (a, b) in INPUT_FMT - if re.search(a, enc_opts) - ][0][1] - - record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property - - (ref_file, dut_output_file) = rend.binauralize_input_and_output( - record_property, - props_to_record, - test_info, - testv_file, - dut_output_file, - ref_output_file, - in_fmt, - output_config, - enc_opts, - dec_opts, - in_sr, - out_sr, - ) - fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( ref_file, diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 2c0791ea78..ec0c58d1a8 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -188,6 +188,8 @@ def test_pca_enc( gain_flag, keep_files, decoder_only, + cut_gain=cut_gain, + cut_testv=cut_testv, plc_pattern=plc_pattern, get_mld=get_mld, get_mld_lim=get_mld_lim, @@ -364,6 +366,8 @@ def test_sba_enc_system( gain_flag, keep_files, decoder_only, + cut_gain=cut_gain, + cut_testv=cut_testv, plc_pattern=plc_pattern, get_mld=get_mld, get_mld_lim=get_mld_lim, @@ -506,6 +510,8 @@ def test_spar_hoa2_enc_system( gain_flag, keep_files, decoder_only, + cut_gain=cut_gain, + cut_testv=cut_testv, plc_pattern=plc_pattern, get_mld=get_mld, get_mld_lim=get_mld_lim, @@ -642,6 +648,8 @@ def test_spar_hoa3_enc_system( gain_flag, keep_files, decoder_only, + cut_gain=cut_gain, + cut_testv=cut_testv, plc_pattern=plc_pattern, get_mld=get_mld, get_mld_lim=get_mld_lim, @@ -798,6 +806,8 @@ def test_sba_enc_BWforce_system( gain_flag, keep_files, decoder_only, + cut_gain=cut_gain, + cut_testv=cut_testv, plc_pattern=plc_pattern, get_mld=get_mld, get_mld_lim=get_mld_lim, @@ -935,6 +945,8 @@ def test_sba_plc_system( gain_flag, keep_files, decoder_only, + cut_gain=cut_gain, + cut_testv=cut_testv, plc_pattern=plc_pattern, get_mld=get_mld, get_mld_lim=get_mld_lim, @@ -1093,6 +1105,8 @@ def sba_dec( gain_flag, keep_files, decoder_only, + cut_gain="1.0", + cut_testv=False, plc_pattern=None, get_mld=False, get_mld_lim=0, @@ -1169,26 +1183,16 @@ def sba_dec( plc_file=plc_file, ) - allow_differing_lengths = False - if compare_to_input: - # If comparing to input, set input as reference and allow different length - ref_out_file = f"{test_vector_path}/{tag}.wav" - if gain_flag != -1: - ref_out_file = f"{test_vector_path}/{tag}_Gain{gain_flag}.wav" - else: - ref_out_file = f"{test_vector_path}/{tag}.wav" - - allow_differing_lengths = True - - if peaq_binaural: allow_differing_lengths = True enc_opts = "" dec_opts = "" - if gain_flag != -1: - input_file = f"{test_vector_path}/{tag}_Gain{gain_flag}.wav" - else: - input_file = f"{test_vector_path}/{tag}.wav" + input_file = f"{test_vector_path}/{tag}.wav" + if cut_testv: + if cut_gain == "1.0": + input_file = f"{test_vector_path}/{tag}_cut.wav" + else: + input_file = f"{test_vector_path}/{tag}_cut_{cut_gain}.wav" in_fmt = input_config in_sr = sampling_rate out_sr = sampling_rate @@ -1207,6 +1211,17 @@ def sba_dec( in_sr, out_sr, ) + else: + allow_differing_lengths = False + if compare_to_input: + # If comparing to input, set input as reference and allow different length + ref_out_file = f"{test_vector_path}/{tag}.wav" + if gain_flag != -1: + ref_out_file = f"{test_vector_path}/{tag}_Gain{gain_flag}.wav" + else: + ref_out_file = f"{test_vector_path}/{tag}.wav" + + allow_differing_lengths = True sampling_rate_Hz = int(sampling_rate) * 1000 cmp_result, reason = cmp_pcm( -- GitLab From 1fd832273712beee2c451e0459bea65ab19fc043 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 15:09:01 +0100 Subject: [PATCH 20/40] Make PEAQ run on 1-2 channel input --- tests/cmp_pcm.py | 26 +++++++++++++------ .../test_param_file.py | 1 + tests/codec_be_on_mr_nonselection/test_sba.py | 1 + 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 32defbdd05..4885ccd7e1 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -33,6 +33,7 @@ def cmp_pcm( abs_tol=0, get_ssnr=False, get_odg=False, + peaq_binaural=False, ref_jbm_tf: Optional[Path] = None, cut_jbm_tf: Optional[Path] = None, ) -> (int, str): @@ -129,14 +130,23 @@ def cmp_pcm( reason += msg + " - " if get_odg: - for n in range(nchannels): - pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) - - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - odg = float(match_odg.groups()[0]) - msg = f"Channel {n} ODG: {odg}" - reason += " - " + msg - print(msg) + if peaq_binaural: + pqeval_output = pqevalaudio_wrapper(s1, s2, fs) + + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg = float(match_odg.groups()[0]) + msg = f"ODG: {odg}" + reason += " - " + msg + print(msg) + else: + for n in range(nchannels): + pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg = float(match_odg.groups()[0]) + msg = f"Channel {n} ODG: {odg}" + reason += " - " + msg + print(msg) return output_differs, reason diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 5f0dedced7..b38297bc3a 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -514,6 +514,7 @@ def test_param_file_tests( allow_differing_lengths=allow_differing_lengths, get_ssnr=get_ssnr, get_odg=get_odg, + peaq_binaural=peaq_binaural, ref_jbm_tf=ref_tracefile_dec, cut_jbm_tf=dut_tracefile_dec, ) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index ec0c58d1a8..7fcca02841 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1235,6 +1235,7 @@ def sba_dec( allow_differing_lengths=allow_differing_lengths, get_ssnr=get_ssnr, get_odg=get_odg, + peaq_binaural=peaq_binaural, ) text_to_parse = reason -- GitLab From a43cafd0bf179e50ff2731ad54a7e3e0cbb5b063 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 15:18:08 +0100 Subject: [PATCH 21/40] Fix test job --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aaa9484cb2..d6538f3f97 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1742,6 +1742,7 @@ peaq-binaural: script: - *print-common-info - make -j + - cp IVAS_rend IVAS_rend_ref - exit_code=0 - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --peaq_binaural --odg || exit_code=$? -- GitLab From f5715693c9711b5280c7730725c61bb46831e529 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 19:04:19 +0100 Subject: [PATCH 22/40] Remove duplicated function check_and_makedir --- .../codec_be_on_mr_nonselection/test_param_file.py | 13 ++----------- tests/codec_be_on_mr_nonselection/test_sba.py | 14 +++----------- tests/hrtf_binary_loading/utils.py | 2 +- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index fa145dcc07..198ecccbf9 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -52,7 +52,7 @@ from tests.constants import ( SCRIPTS_DIR, MAX_ENC_DIFF, ) -import tests.renderer.utils as rend +from tests.renderer.utils import check_and_makedir, binauralize_input_and_output PASSTHROUGH_CONF = [ (r"[\s\S]*", "EXT"), # Any input with EXT output is pass-through @@ -154,15 +154,6 @@ with open(PARAM_FILE, "r", encoding="UTF-8") as fp: param_file_test_dict[tag] = (enc_opts, dec_opts, sim_opts, eid_opts) -def check_and_makedir(dir_path): - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise # raises the error again - - def convert_test_string_to_tag(test_string): """ Convert a test string (i.e. the test tag from the parameter file) to a tag string. @@ -457,7 +448,7 @@ def test_param_file_tests( record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property - (ref_output_file, dut_output_file) = rend.binauralize_input_and_output( + (ref_output_file, dut_output_file) = binauralize_input_and_output( record_property, props_to_record, test_info, diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 70b8d903cf..7a3274b506 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -44,7 +44,8 @@ from ..conftest import parse_properties from ..cmp_stats_files import cmp_stats_files from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF from tests.testconfig import use_ltv -import tests.renderer.utils as rend + +from tests.renderer.utils import check_and_makedir, binauralize_input_and_output tag_list = ["ltvFOA" if use_ltv else "stvFOA"] tag_list_HOA2 = ["ltvHOA2" if use_ltv else "stv2OA"] @@ -82,15 +83,6 @@ ivas_br_plc = ["13200", "16400", "32000", "64000", "96000", "256000"] # SBA order to IVAS_rend format table SBA_FORMAT={1: "FOA", 2: "HOA2", 3: "HOA3"} -def check_and_makedir(dir_path): - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise # raises the error again - - @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("sampling_rate", sample_rate_list) def test_pca_enc( @@ -1197,7 +1189,7 @@ def sba_dec( in_sr = sampling_rate out_sr = sampling_rate - (ref_out_file, dut_out_file) = rend.binauralize_input_and_output( + (ref_out_file, dut_out_file) = binauralize_input_and_output( dut_decoder_frontend.record_property, props_to_record, test_info, diff --git a/tests/hrtf_binary_loading/utils.py b/tests/hrtf_binary_loading/utils.py index b0077edf08..4e74a32f7f 100644 --- a/tests/hrtf_binary_loading/utils.py +++ b/tests/hrtf_binary_loading/utils.py @@ -37,7 +37,7 @@ from typing import Optional import pytest -from tests.renderer.utils import check_BE, run_cmd, run_renderer, test_info +from tests.renderer.utils import check_BE, run_cmd, run_renderer from .constants import * -- GitLab From 70a89368834d0b07ecc2a1c04c99b17b6a943b5c Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 6 Dec 2024 19:09:56 +0100 Subject: [PATCH 23/40] Resolve import of test_info --- tests/renderer/test_renderer.py | 2 +- tests/split_rendering/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 80c935c4f6..bb999ba639 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -50,7 +50,7 @@ from .constants import ( CUSTOM_LAYOUT_DIR, METADATA_SCENES_TO_TEST, ) -from .utils import run_renderer, compare_renderer_args, test_info +from .utils import run_renderer, compare_renderer_args from ..conftest import props_to_record ############################################################################## diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index 1ea1e50aba..bb826752ad 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -39,7 +39,7 @@ from typing import Tuple import numpy as np import pytest -from tests.renderer.utils import check_BE, run_cmd, test_info, run_ivas_isar_enc_cmd, run_ivas_isar_dec_cmd, run_isar_post_rend_cmd, run_isar_ext_rend_cmd +from tests.renderer.utils import check_BE, run_cmd, run_ivas_isar_enc_cmd, run_ivas_isar_dec_cmd, run_isar_post_rend_cmd, run_isar_ext_rend_cmd from tests.split_rendering.constants import * sys.path.append(SCRIPTS_DIR) -- GitLab From dc08602924089a9e2b30f2706d96abee2e020bf6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Dec 2024 08:52:58 +0100 Subject: [PATCH 24/40] Add OMASA/OSBA rendering using scene description file --- tests/renderer/utils.py | 277 ++++++++++++++++++++++++---------------- 1 file changed, 164 insertions(+), 113 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 0b3f5110c9..e7d71c7d2b 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -42,6 +42,7 @@ import numpy as np import pytest import re import errno +import tempfile from .compare_audio import compare_audio_arrays from .constants import ( @@ -168,9 +169,9 @@ def run_renderer( get_ssnr=False, get_odg=False, aeid: Optional[Union[Path, int]] = None, - in_file = None, - out_file = None, - sr = 48, + in_file=None, + out_file=None, + sr=48, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -377,6 +378,7 @@ def compare_renderer_args( f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" ) + def binauralize_input_and_output( record_property, props_to_record, @@ -392,117 +394,145 @@ def binauralize_input_and_output( out_sr, ): - # File names for binauralized input, if needed - dut_output_file_binaural = dut_output_file[0:-4]+".BINAURAL.wav" - ref_output_file_binaural = ref_output_file[0:-4]+".BINAURAL.wav" - - # Identify metadata - in_meta_files = [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.csv\b', enc_opts)] # All .csv files in enc_opts are ISM metadata files. - in_meta_files = in_meta_files + [str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r'\b\S+\.met\b', enc_opts)] # All .met files in enc_opts are MASA metadata files. - n_obj = len(in_meta_files) - out_meta_files = None - if output_config == "EXT": - if "ISM" in in_fmt and n_obj > 0: - out_meta_files = [f"{dut_output_file}.{i}.csv" for i in range(0,n_obj)] - if "MASA" in in_fmt: - out_meta_files = [f"{dut_output_file}.met"] - if output_config == "EXT": - output_config = in_fmt - if output_config == "": - output_config = "MONO" # EVS mono - metadata_input = None # Todo: This can be used for OMASA/OSBA rendering - - # Identify headtracking and orientation trajectories - trj_file = findstr(r'-t\s+(\S+)', dec_opts) - non_diegetic_pan = findstr(r'-non_diegetic_pan\s+(\S+)', dec_opts) - if non_diegetic_pan is not None: - output_config = "STEREO" - name_extension = None - refrot_file = findstr(r'-rf\s+(\S+)', dec_opts) - rot_tmp_file = findstr(r'-rvf\s+(\S+)', dec_opts) - refveclev_file = None - refvec_file = None - if "-otr ref_vec_lev".upper() in dec_opts.upper(): - refveclev_file = rot_tmp_file - else: - if "-otr ref_vec".upper() in dec_opts.upper(): - refvec_file = rot_tmp_file - - # Rendering configuration - config_file = findstr(r'-render_config\s+(\S+)', dec_opts) - binary_suffix = "_ref" - frame_size = findstr(r'-fr\s+(\S+)', dec_opts) - # hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) - hrtf_file = None # Default HRTFs used for binaural rendering of output - - aeid = findstr(r'-aeid\s+(\S+)', dec_opts) - - if not output_config.upper() in PEAQ_SUPPORTED_FMT: - # Render output to BINAURAL - output_reformat = "BINAURAL" - - check_and_makedir(str(Path(dut_output_file_binaural).parent)) - - run_renderer( - record_property, - props_to_record, - test_info, - output_config, - output_reformat, - metadata_input, - out_meta_files, - trj_file, - non_diegetic_pan, - name_extension, - refrot_file, - refvec_file, - refveclev_file, - config_file, - binary_suffix, - frame_size, - hrtf_file, - aeid, - in_file=dut_output_file, - out_file=dut_output_file_binaural, - sr=out_sr, - ) - # Update output_config to rendered format - output_config = output_reformat - else: - dut_output_file_binaural = dut_output_file - - if in_fmt.upper() != output_config.upper(): - # Render input to match output_config - out_fmt = output_config - - check_and_makedir(str(Path(ref_output_file_binaural).parent)) + # Use current folder as location for temporary directory, since scene description does not handle spaces in path + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: + tmp_dir = Path(tmp_dir) + scene_out = str(tmp_dir.joinpath("scene_out.txt")) + scene_in = str(tmp_dir.joinpath("scene_in.txt")) + + # File names for binauralized input, if needed + dut_output_file_binaural = dut_output_file[0:-4] + ".BINAURAL.wav" + ref_output_file_binaural = ref_output_file[0:-4] + ".BINAURAL.wav" + + # Identify metadata + in_meta_files = [ + str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r"\b\S+\.csv\b", enc_opts) + ] # All .csv files in enc_opts are ISM metadata files. + in_meta_files = in_meta_files + [ + str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r"\b\S+\.met\b", enc_opts) + ] # All .met files in enc_opts are MASA metadata files. + n_obj = len(in_meta_files) + out_meta_files = None + if output_config == "EXT": + out_meta_files = [] + if n_obj > 0: + out_meta_files = out_meta_files + [ + f"{dut_output_file}.{i}.csv" for i in range(0, n_obj) + ] + if "MASA" in in_fmt: + out_meta_files = out_meta_files + [f"{dut_output_file}.met"] + if output_config == "EXT": + output_config = in_fmt + if output_config == "": + output_config = "MONO" # EVS mono + metadata_input = None + + if "OSBA" in in_fmt or "OMASA" in in_fmt: + scene_description_file(in_fmt, scene_in, n_obj, input_file, in_meta_files) + input_file = scene_in + in_meta_files = None + in_fmt = "META" + + if "OSBA" in output_config or "OMASA" in output_config: + scene_description_file( + output_config, scene_out, n_obj, dut_output_file, out_meta_files + ) + dut_output_file = scene_out + out_meta_files = None + output_config = "META" + + # Identify headtracking and orientation trajectories + trj_file = findstr(r"-t\s+(\S+)", dec_opts) + non_diegetic_pan = findstr(r"-non_diegetic_pan\s+(\S+)", dec_opts) + if non_diegetic_pan is not None: + output_config = "STEREO" + name_extension = None + refrot_file = findstr(r"-rf\s+(\S+)", dec_opts) + rot_tmp_file = findstr(r"-rvf\s+(\S+)", dec_opts) + refveclev_file = None + refvec_file = None + if "-otr ref_vec_lev".upper() in dec_opts.upper(): + refveclev_file = rot_tmp_file + else: + if "-otr ref_vec".upper() in dec_opts.upper(): + refvec_file = rot_tmp_file + + # Rendering configuration + config_file = findstr(r"-render_config\s+(\S+)", dec_opts) + binary_suffix = "_ref" + frame_size = findstr(r"-fr\s+(\S+)", dec_opts) + # hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) + hrtf_file = None # Default HRTFs used for binaural rendering of output + + aeid = findstr(r"-aeid\s+(\S+)", dec_opts) + + if not output_config.upper() in PEAQ_SUPPORTED_FMT: + # Render output to BINAURAL + output_reformat = "BINAURAL" + + check_and_makedir(str(Path(dut_output_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + output_config, + output_reformat, + metadata_input, + out_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=dut_output_file, + out_file=dut_output_file_binaural, + sr=out_sr, + ) + # Update output_config to rendered format + output_config = output_reformat + else: + dut_output_file_binaural = dut_output_file + + if in_fmt.upper() != output_config.upper(): + # Render input to match output_config + out_fmt = output_config + + check_and_makedir(str(Path(ref_output_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + metadata_input, + in_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=input_file, + out_file=ref_output_file_binaural, + sr=in_sr, + ) + else: + ref_output_file_binaural = input_file + return (ref_output_file_binaural, dut_output_file_binaural) - run_renderer( - record_property, - props_to_record, - test_info, - in_fmt, - out_fmt, - metadata_input, - in_meta_files, - trj_file, - non_diegetic_pan, - name_extension, - refrot_file, - refvec_file, - refveclev_file, - config_file, - binary_suffix, - frame_size, - hrtf_file, - aeid, - in_file=input_file, - out_file=ref_output_file_binaural, - sr=in_sr, - ) - else: - ref_output_file_binaural = input_file - return (ref_output_file_binaural, dut_output_file_binaural) def findstr(exp, s, one_element=True): result = [SCRIPTS_DIR.joinpath(x) for x in re.findall(exp, s)] @@ -512,6 +542,7 @@ def findstr(exp, s, one_element=True): return result[0] return result + def check_and_makedir(dir_path): if not os.path.exists(dir_path): try: @@ -519,3 +550,23 @@ def check_and_makedir(dir_path): except OSError as e: if e.errno != errno.EEXIST: raise # raises the error again + + +def scene_description_file(in_fmt, metadata_tmp, n_obj, input_file, in_meta_files): + with open(metadata_tmp, "w") as fp_meta: + currdir = Path( + metadata_tmp + ).parent # File names must be relative to config file location + fp_meta.write(f"{os.path.relpath(input_file, currdir)}\n") # Input file + fp_meta.write(f"{n_obj+1}\n") # Number of sources + for n in range(0, n_obj): + fp_meta.write( + f"ISM\n{n+1}\n{os.path.relpath(in_meta_files[n], currdir)}\n" + ) # ISM metadata + fp_meta.write(f"{in_fmt.split('_')[0][1:]}\n") # SBA or MASA + fp_meta.write(f"{n_obj+1}\n") + fp_meta.write(f"{in_fmt.split('_')[-1]}\n") # SBA or MASA parameter + if "MASA" in in_fmt: + fp_meta.write( + f"{os.path.relpath(in_meta_files[n_obj], currdir)}\n" + ) # MASA metadata -- GitLab From 4b54fb3a15353689731779dfc98dc1874a7b421b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Dec 2024 09:10:13 +0100 Subject: [PATCH 25/40] Add missing format OSBA for HOA3 --- tests/codec_be_on_mr_nonselection/test_param_file.py | 4 ++++ tests/renderer/utils.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 198ecccbf9..7964f6e90c 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -115,6 +115,10 @@ INPUT_FMT = [ (r"-ism_sba\s?2\s?2", "OSBA_2_2"), (r"-ism_sba\s?3\s?2", "OSBA_3_2"), (r"-ism_sba\s?4\s?2", "OSBA_4_2"), + (r"-ism_sba\s?1\s?3", "OSBA_1_3"), + (r"-ism_sba\s?2\s?3", "OSBA_2_3"), + (r"-ism_sba\s?3\s?3", "OSBA_3_3"), + (r"-ism_sba\s?4\s?3", "OSBA_4_3"), (r".*", "MONO"), ] diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index e7d71c7d2b..efb4e4f4b9 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -408,10 +408,10 @@ def binauralize_input_and_output( in_meta_files = [ str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r"\b\S+\.csv\b", enc_opts) ] # All .csv files in enc_opts are ISM metadata files. + n_obj = len(in_meta_files) in_meta_files = in_meta_files + [ str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r"\b\S+\.met\b", enc_opts) ] # All .met files in enc_opts are MASA metadata files. - n_obj = len(in_meta_files) out_meta_files = None if output_config == "EXT": out_meta_files = [] -- GitLab From 4e45ba23bfae5f643929d4673013411736928520 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Dec 2024 09:25:21 +0100 Subject: [PATCH 26/40] Fix for planar OSBA --- .../test_param_file.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 7964f6e90c..b042da8183 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -107,18 +107,18 @@ INPUT_FMT = [ (r"-ism_masa\s?2\s?2", "OMASA_2_2"), (r"-ism_masa\s?3\s?2", "OMASA_3_2"), (r"-ism_masa\s?4\s?2", "OMASA_4_2"), - (r"-ism_sba\s?1\s?1", "OSBA_1_1"), - (r"-ism_sba\s?2\s?1", "OSBA_2_1"), - (r"-ism_sba\s?3\s?1", "OSBA_3_1"), - (r"-ism_sba\s?4\s?1", "OSBA_4_1"), - (r"-ism_sba\s?1\s?2", "OSBA_1_2"), - (r"-ism_sba\s?2\s?2", "OSBA_2_2"), - (r"-ism_sba\s?3\s?2", "OSBA_3_2"), - (r"-ism_sba\s?4\s?2", "OSBA_4_2"), - (r"-ism_sba\s?1\s?3", "OSBA_1_3"), - (r"-ism_sba\s?2\s?3", "OSBA_2_3"), - (r"-ism_sba\s?3\s?3", "OSBA_3_3"), - (r"-ism_sba\s?4\s?3", "OSBA_4_3"), + (r"-ism_sba\s?1\s?[-+]?1", "OSBA_1_1"), + (r"-ism_sba\s?2\s?[-+]?1", "OSBA_2_1"), + (r"-ism_sba\s?3\s?[-+]?1", "OSBA_3_1"), + (r"-ism_sba\s?4\s?[-+]?1", "OSBA_4_1"), + (r"-ism_sba\s?1\s?[-+]?2", "OSBA_1_2"), + (r"-ism_sba\s?2\s?[-+]?2", "OSBA_2_2"), + (r"-ism_sba\s?3\s?[-+]?2", "OSBA_3_2"), + (r"-ism_sba\s?4\s?[-+]?2", "OSBA_4_2"), + (r"-ism_sba\s?1\s?[-+]?3", "OSBA_1_3"), + (r"-ism_sba\s?2\s?[-+]?3", "OSBA_2_3"), + (r"-ism_sba\s?3\s?[-+]?3", "OSBA_3_3"), + (r"-ism_sba\s?4\s?[-+]?3", "OSBA_4_3"), (r".*", "MONO"), ] -- GitLab From e96d99858c971c2c69c403a93eb7146c6964c1a9 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Dec 2024 10:35:16 +0100 Subject: [PATCH 27/40] Fix for NULL metadata input --- tests/renderer/utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index efb4e4f4b9..e817123511 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -406,8 +406,8 @@ def binauralize_input_and_output( # Identify metadata in_meta_files = [ - str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r"\b\S+\.csv\b", enc_opts) - ] # All .csv files in enc_opts are ISM metadata files. + str(SCRIPTS_DIR.joinpath(m)) if m != "NULL" else m for m in re.findall(r"\b\S+\.csv|NULL\b", enc_opts) + ] # All .csv or NULL files in enc_opts are ISM metadata files. n_obj = len(in_meta_files) in_meta_files = in_meta_files + [ str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r"\b\S+\.met\b", enc_opts) @@ -560,8 +560,12 @@ def scene_description_file(in_fmt, metadata_tmp, n_obj, input_file, in_meta_file fp_meta.write(f"{os.path.relpath(input_file, currdir)}\n") # Input file fp_meta.write(f"{n_obj+1}\n") # Number of sources for n in range(0, n_obj): + if in_meta_files[n] == "NULL": + md_file = "NULL" + else: + md_file = os.path.relpath(in_meta_files[n], currdir) fp_meta.write( - f"ISM\n{n+1}\n{os.path.relpath(in_meta_files[n], currdir)}\n" + f"ISM\n{n+1}\n{md_file}\n" ) # ISM metadata fp_meta.write(f"{in_fmt.split('_')[0][1:]}\n") # SBA or MASA fp_meta.write(f"{n_obj+1}\n") -- GitLab From f49a32e7b41067cdf0735f200fa01d7d00b14170 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Dec 2024 14:36:39 +0100 Subject: [PATCH 28/40] Fix for NULL metadata in scene description --- tests/renderer/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index e817123511..9c940a6816 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -561,7 +561,7 @@ def scene_description_file(in_fmt, metadata_tmp, n_obj, input_file, in_meta_file fp_meta.write(f"{n_obj+1}\n") # Number of sources for n in range(0, n_obj): if in_meta_files[n] == "NULL": - md_file = "NULL" + md_file = "1\n1,0,0" # NULL metadata position: azim=0,elev=0 for 1 frame, looped throughout all frames. else: md_file = os.path.relpath(in_meta_files[n], currdir) fp_meta.write( -- GitLab From ee36749d68a82cf1e22494caa5fa852f2d86af9d Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 12 Dec 2024 08:24:04 +0100 Subject: [PATCH 29/40] Add fix for differing sampling rates in output --- tests/cmp_pcm.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 4885ccd7e1..86e986f6a4 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -54,15 +54,33 @@ def cmp_pcm( else: nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] - s1, _ = pyaudio3dtools.audiofile.readfile( + s1, fs1 = pyaudio3dtools.audiofile.readfile( ref_file, nchannels, fs, outdtype=np.int16 ) - s2, _ = pyaudio3dtools.audiofile.readfile( + s2, fs2 = pyaudio3dtools.audiofile.readfile( cmp_file, nchannels, fs, outdtype=np.int16 ) # In case of wav input, override the nchannels with the one from the wav header nchannels = s1.shape[1] + fs = fs1 + + if fs1 != fs2: + if peaq_binaural: + fs = max(fs1,fs2) + s1 = np.clip( + pyaudio3dtools.audioarray.resample(s1.astype(float), fs, 48000), + -32768, + 32767, + ).astype(np.int16) + s2 = np.clip( + pyaudio3dtools.audioarray.resample(s2.astype(float), fs, 48000), + -32768, + 32767, + ).astype(np.int16) + else: + reason = "FAIL: Sampling rate differs." + return 1, reason # In case number of channels do not match, fail already now. Could happen in case of # comparison to input with for a non-passthrough mode. -- GitLab From cdeedc9b3a1ca8f9efbb726cf433fb4c3ce221d2 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 12 Dec 2024 09:48:19 +0100 Subject: [PATCH 30/40] Fix for PEAQ resampling --- tests/cmp_pcm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 86e986f6a4..32cd8709ab 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -67,14 +67,14 @@ def cmp_pcm( if fs1 != fs2: if peaq_binaural: - fs = max(fs1,fs2) + fs = 48000 s1 = np.clip( - pyaudio3dtools.audioarray.resample(s1.astype(float), fs, 48000), + pyaudio3dtools.audioarray.resample(s1.astype(float), fs1, fs), -32768, 32767, ).astype(np.int16) s2 = np.clip( - pyaudio3dtools.audioarray.resample(s2.astype(float), fs, 48000), + pyaudio3dtools.audioarray.resample(s2.astype(float), fs2, fs), -32768, 32767, ).astype(np.int16) -- GitLab From 4bfe20d6ee0cc1b99e087780ea00569d2ec84634 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 12 Dec 2024 10:04:43 +0100 Subject: [PATCH 31/40] Fix indentation error --- tests/cmp_pcm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 32cd8709ab..41c23a1490 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -78,9 +78,9 @@ def cmp_pcm( -32768, 32767, ).astype(np.int16) - else: - reason = "FAIL: Sampling rate differs." - return 1, reason + else: + reason = "FAIL: Sampling rate differs." + return 1, reason # In case number of channels do not match, fail already now. Could happen in case of # comparison to input with for a non-passthrough mode. -- GitLab From 6256c1da94c07bb0578fc2a2383adc182436a623 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 13 Dec 2024 10:43:55 +0100 Subject: [PATCH 32/40] Fix in diff_report.py --- scripts/diff_report.py | 2 +- .../test_param_file.py | 47 ++++++++----------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/scripts/diff_report.py b/scripts/diff_report.py index 0b9cec591a..df32b03a44 100644 --- a/scripts/diff_report.py +++ b/scripts/diff_report.py @@ -48,7 +48,7 @@ def main(args): df_test = pd.read_csv(args.csv_test, sep=";") for col in COLUMNS_TO_COMPARE: - df_ref[col] = df_ref[col] - df_test[col] + df_ref[col] = df_test[col] - df_ref[col] df_ref.to_csv(args.csv_diff, index=False, sep=";") return 0 diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index b042da8183..37095a19e8 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -32,7 +32,6 @@ __doc__ = """ Execute tests specified via a parameter file. """ -import errno import filecmp import os import platform @@ -90,7 +89,7 @@ INPUT_FMT = [ (r"-sba\s?[-+]?3", "HOA3"), (r"-mc\s5_1\s", "5_1"), (r"-mc\s7_1\s", "7_1"), - (r"-mc\s5_1_2", "5_1_2"), + (r"-mc\s5_1_2", "5_1_2"), (r"-mc\s5_1_4", "5_1_4"), (r"-mc\s7_1_4", "7_1_4"), (r"-ism\s?[-+]?1", "ISM1"), @@ -444,27 +443,27 @@ def test_param_file_tests( tracefile_dec = "" # Find input format - in_fmt = [ - (a, b) - for (a, b) in INPUT_FMT - if re.search(a, enc_opts) - ][0][1] + in_fmt = [(a, b) for (a, b) in INPUT_FMT if re.search(a, enc_opts)][0][1] - record_property = dut_decoder_frontend.record_property if dut_decoder_frontend else ref_decoder_frontend.record_property + record_property = ( + dut_decoder_frontend.record_property + if dut_decoder_frontend + else ref_decoder_frontend.record_property + ) (ref_output_file, dut_output_file) = binauralize_input_and_output( - record_property, - props_to_record, - test_info, - testv_file, - dut_output_file, - ref_output_file, - in_fmt, - output_config, - enc_opts, - dec_opts, - in_sr, - out_sr, + record_property, + props_to_record, + test_info, + testv_file, + dut_output_file, + ref_output_file, + in_fmt, + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, ) # set to false per default even if this is no JBM case - makes later check for failure easier @@ -857,11 +856,3 @@ def get_expected_md_files(ref_output_file, enc_opts, output_config): pass return md_files - -def findstr(exp, s, one_element=True): - result = [SCRIPTS_DIR.joinpath(x) for x in re.findall(exp, s)] - if len(result) == 0: - return None - if one_element: - return result[0] - return result \ No newline at end of file -- GitLab From 83e6d3f6911d143ddd32681e13c9556fee57a7d0 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 13 Dec 2024 14:10:47 +0100 Subject: [PATCH 33/40] Truncate IVAS_rend metadata in PEAQ tests, if IVAS_dec does not include extended metadata --- tests/renderer/utils.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 9c940a6816..cbb2d99950 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -406,9 +406,29 @@ def binauralize_input_and_output( # Identify metadata in_meta_files = [ - str(SCRIPTS_DIR.joinpath(m)) if m != "NULL" else m for m in re.findall(r"\b\S+\.csv|NULL\b", enc_opts) + str(SCRIPTS_DIR.joinpath(m)) if m != "NULL" else m + for m in re.findall(r"\b\S+\.csv|NULL\b", enc_opts) ] # All .csv or NULL files in enc_opts are ISM metadata files. n_obj = len(in_meta_files) + + # If extended metadata is not used, strip the metadata for the external renderer + extended_md_used = ( + re.search(r"-ism\s?\+[1-4]", enc_opts) + and not "OMASA" in in_fmt + and not "OSBA" in in_fmt + ) + if not extended_md_used and n_obj > 0: + truncated_meta_files = [] + for md in in_meta_files: + md_out_file = str(tmp_dir.joinpath(os.path.basename(md))) + with open(md_out_file, "w") as fp_out, open(md, "r") as fp_in: + for line in fp_in: + fp_out.write( + ",".join(line.split(",")[:2]) + "\n" + ) # Keep only first two elements: azim, elev + truncated_meta_files.append(md_out_file) + in_meta_files = truncated_meta_files + in_meta_files = in_meta_files + [ str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r"\b\S+\.met\b", enc_opts) ] # All .met files in enc_opts are MASA metadata files. @@ -561,16 +581,22 @@ def scene_description_file(in_fmt, metadata_tmp, n_obj, input_file, in_meta_file fp_meta.write(f"{n_obj+1}\n") # Number of sources for n in range(0, n_obj): if in_meta_files[n] == "NULL": - md_file = "1\n1,0,0" # NULL metadata position: azim=0,elev=0 for 1 frame, looped throughout all frames. + md_file = "1\n1,0,0" # NULL metadata position: azim=0,elev=0 for 1 frame, looped throughout all frames. else: md_file = os.path.relpath(in_meta_files[n], currdir) - fp_meta.write( - f"ISM\n{n+1}\n{md_file}\n" - ) # ISM metadata + fp_meta.write(f"ISM\n{n+1}\n{md_file}\n") # ISM metadata + if "OSBA" in in_fmt: + fp_meta.write( + "gain_dB:-6\n" + ) # Set -6 dB on all components for OSBA to match IVAS_dec fp_meta.write(f"{in_fmt.split('_')[0][1:]}\n") # SBA or MASA fp_meta.write(f"{n_obj+1}\n") fp_meta.write(f"{in_fmt.split('_')[-1]}\n") # SBA or MASA parameter - if "MASA" in in_fmt: + if "OMASA" in in_fmt: fp_meta.write( f"{os.path.relpath(in_meta_files[n_obj], currdir)}\n" ) # MASA metadata + if "OSBA" in in_fmt: + fp_meta.write( + "gain_dB:-6\n" + ) # Set -6 dB on all components for OSBA to match IVAS_dec -- GitLab From ae6c43f546d43d66be7124cbc8de97aedf898829 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 18 Dec 2024 16:59:58 +0100 Subject: [PATCH 34/40] Added test_enc_passthrough.py --- .gitlab-ci.yml | 2 +- .../test_param_file.py | 62 ++++++ tests/test_enc_passthrough.py | 181 ++++++++++++++++++ 3 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 tests/test_enc_passthrough.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index feaff439ac..7ef6e4d3e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1774,7 +1774,7 @@ peaq-binaural: - cp IVAS_rend IVAS_rend_ref - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --peaq_binaural --odg || exit_code=$? + - python3 -m pytest tests/test_enc_passthrough.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --peaq_binaural --odg || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit.xml report.csv - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 37095a19e8..937c88ad98 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -207,8 +207,70 @@ def test_param_file_tests( compare_to_input, peaq_binaural, ): + enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] + run_test( + test_info, + props_to_record, + encoder_only, + decoder_only, + dut_encoder_frontend, + dut_decoder_frontend, + ref_encoder_frontend, + ref_decoder_frontend, + enc_opts, + dec_opts, + sim_opts, + eid_opts, + reference_path, + dut_base_path, + test_vector_path, + update_ref, + rootdir, + keep_files, + test_tag, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_enc_stats, + get_odg, + compare_to_input, + peaq_binaural, + ) + + +def run_test( + test_info, + props_to_record, + encoder_only, + decoder_only, + dut_encoder_frontend: EncoderFrontend, + dut_decoder_frontend: DecoderFrontend, + ref_encoder_frontend: EncoderFrontend, + ref_decoder_frontend: DecoderFrontend, + enc_opts, + dec_opts, + sim_opts, + eid_opts, + reference_path, + dut_base_path, + test_vector_path, + update_ref, + rootdir, + keep_files, + test_tag, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_enc_stats, + get_odg, + compare_to_input, + peaq_binaural, +): + # If compare_to_input is set, only run pass-through test cases if compare_to_input: passthrough = [ diff --git a/tests/test_enc_passthrough.py b/tests/test_enc_passthrough.py new file mode 100644 index 0000000000..4ccd928323 --- /dev/null +++ b/tests/test_enc_passthrough.py @@ -0,0 +1,181 @@ +__copyright__ = """ +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +__doc__ = """ +Execute tests specified via a parameter file. +""" +import pytest + +from tests.codec_be_on_mr_nonselection.test_param_file import run_test +from tests.conftest import DecoderFrontend, EncoderFrontend +from tests.constants import SCRIPTS_DIR +import os.path + +BITRATES = [13200, 16400, 24400, 32000, 48000, 64000, 80000, 96000, 128000, 160000, 192000, 256000, 384000, 512000] +BW = [16, 32, 48] + +FORMAT = [ + ("STEREO","-stereo","stvST{bw}c.wav","STEREO"), + ("FOA","-sba 1", "stvFOA{bw}c.wav","FOA"), + ("HOA2","-sba 2", "stv2OA{bw}c.wav","HOA2"), + ("HOA3","-sba 3", "stv3OA{bw}c.wav","HOA3"), + ("5_1","-mc 5_1", "stv51MC{bw}c.wav","5_1"), + ("5_1_2","-mc 5_1_2", "stv512MC{bw}c.wav","5_1_2"), + ("5_1_4","-mc 5_1_4", "stv514MC{bw}c.wav","5_1_4"), + ("7_1","-mc 7_1", "stv71MC{bw}c.wav","7_1"), + ("7_1_4","-mc 7_1_4", "stv714MC{bw}c.wav","7_1_4"), + ("ISM1","-ism 1 testv/stvISM1.csv", "stv1ISM{bw}s.wav","EXT"), + ("ISM2","-ism 2 testv/stvISM1.csv testv/stvISM2.csv", "stv2ISM{bw}s.wav","EXT"), + ("ISM3","-ism 3 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv", "stv3ISM{bw}s.wav","EXT"), + ("ISM4","-ism 4 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv", "stv4ISM{bw}s.wav","EXT"), + ("MASA1-1Dir","-masa 1 testv/stv1MASA1TC48c.met","stv1MASA1TC{bw}c.wav","EXT"), + ("MASA1-2Dir","-masa 1 testv/stv2MASA1TC48c.met","stv2MASA1TC{bw}c.wav","EXT"), + ("MASA2-1Dir","-masa 2 testv/stv1MASA2TC48c.met","stv1MASA2TC{bw}c.wav","EXT"), + ("MASA2-2Dir","-masa 2 testv/stv2MASA2TC48c.met","stv2MASA2TC{bw}c.wav","EXT"), + ("OMASA-1-1-1Dir","-ism_masa 1 1 testv/stvISM1.csv testv/stv1MASA1TC48c.met","stvOMASA_1ISM_1MASA1TC{bw}c.wav","BINAURAL"), + ("OMASA-2-1-1Dir","-ism_masa 2 1 testv/stvISM1.csv testv/stvISM2.csv testv/stv1MASA1TC48c.met","stvOMASA_2ISM_1MASA1TC{bw}c.wav","BINAURAL"), + ("OMASA-3-1-1Dir","-ism_masa 3 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stv1MASA1TC48c.met","stvOMASA_3ISM_1MASA1TC{bw}c.wav","BINAURAL"), + ("OMASA-4-1-1Dir","-ism_masa 4 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv testv/stv1MASA1TC48c.met","stvOMASA_4ISM_1MASA1TC{bw}c.wav","BINAURAL"), + ("OMASA-1-2-1Dir","-ism_masa 1 2 testv/stvISM1.csv testv/stv1MASA2TC48c.met","stvOMASA_1ISM_1MASA2TC{bw}c.wav","BINAURAL"), + ("OMASA-2-2-1Dir","-ism_masa 2 2 testv/stvISM1.csv testv/stvISM2.csv testv/stv1MASA2TC48c.met","stvOMASA_2ISM_1MASA2TC{bw}c.wav","BINAURAL"), + ("OMASA-3-2-1Dir","-ism_masa 3 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stv1MASA2TC48c.met","stvOMASA_3ISM_1MASA2TC{bw}c.wav","BINAURAL"), + ("OMASA-4-2-1Dir","-ism_masa 4 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv testv/stv1MASA2TC48c.met","stvOMASA_4ISM_1MASA2TC{bw}c.wav","BINAURAL"), + ("OMASA-1-1-2Dir","-ism_masa 1 1 testv/stvISM1.csv testv/stv2MASA1TC48c.met","stvOMASA_1ISM_2MASA1TC{bw}c.wav","BINAURAL"), + ("OMASA-2-1-2Dir","-ism_masa 2 1 testv/stvISM1.csv testv/stvISM2.csv testv/stv2MASA1TC48c.met","stvOMASA_2ISM_2MASA1TC{bw}c.wav","BINAURAL"), + ("OMASA-3-1-2Dir","-ism_masa 3 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stv2MASA1TC48c.met","stvOMASA_3ISM_2MASA1TC{bw}c.wav","BINAURAL"), + ("OMASA-4-1-2Dir","-ism_masa 4 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv testv/stv2MASA1TC48c.met","stvOMASA_4ISM_2MASA1TC{bw}c.wav","BINAURAL"), + ("OMASA-1-2-2Dir","-ism_masa 1 2 testv/stvISM1.csv testv/stv2MASA2TC48c.met","stvOMASA_1ISM_2MASA2TC{bw}c.wav","BINAURAL"), + ("OMASA-2-2-2Dir","-ism_masa 2 2 testv/stvISM1.csv testv/stvISM2.csv testv/stv2MASA2TC48c.met","stvOMASA_2ISM_2MASA2TC{bw}c.wav","BINAURAL"), + ("OMASA-3-2-2Dir","-ism_masa 3 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stv2MASA2TC48c.met","stvOMASA_3ISM_2MASA2TC{bw}c.wav","BINAURAL"), + ("OMASA-4-2-2Dir","-ism_masa 4 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv testv/stv2MASA2TC48c.met","stvOMASA_4ISM_2MASA2TC{bw}c.wav","BINAURAL"), + ("OSBA-1-1","-ism_sba 1 1 testv/stvISM1.csv","stvOSBA_1ISM_FOA{bw}c.wav", "EXT"), + ("OSBA-2-1","-ism_sba 2 1 testv/stvISM1.csv testv/stvISM2.csv","stvOSBA_2ISM_FOA{bw}c.wav", "EXT"), + ("OSBA-3-1","-ism_sba 3 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv","stvOSBA_3ISM_FOA{bw}c.wav", "EXT"), + ("OSBA-4-1","-ism_sba 4 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv", "stvOSBA_4ISM_FOA{bw}c.wav", "EXT"), + ("OSBA-1-2","-ism_sba 1 2 testv/stvISM1.csv","stvOSBA_1ISM_2OA{bw}c.wav", "EXT"), + ("OSBA-2-2","-ism_sba 2 2 testv/stvISM1.csv testv/stvISM2.csv","stvOSBA_2ISM_2OA{bw}c.wav", "EXT"), + ("OSBA-3-2","-ism_sba 3 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv","stvOSBA_3ISM_2OA{bw}c.wav", "EXT"), + ("OSBA-4-2","-ism_sba 4 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv", "stvOSBA_4ISM_2OA{bw}c.wav", "EXT"), + ("OSBA-1-3","-ism_sba 1 3 testv/stvISM1.csv","stvOSBA_1ISM_3OA{bw}c.wav", "EXT"), + ("OSBA-2-3","-ism_sba 2 3 testv/stvISM1.csv testv/stvISM2.csv","stvOSBA_2ISM_3OA{bw}c.wav", "EXT"), + ("OSBA-3-3","-ism_sba 3 3 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv","stvOSBA_3ISM_3OA{bw}c.wav", "EXT"), + ("OSBA-4-3","-ism_sba 4 3 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv", "stvOSBA_4ISM_3OA{bw}c.wav", "EXT"), +] + +test_dict = {} +for fmt, in_fmt, in_file, out_fmt in FORMAT: + for br in BITRATES: + for bw in BW: + + # Filter out unsupported bitrate combinations + if fmt == "STEREO" and br > 256000: + continue + if fmt == "ISM1" and br > 128000: + continue + if fmt == "ISM2" and (br < 16400 or br > 256000): + continue + if fmt == "ISM3" and (br < 24400 or br > 384000): + continue + if fmt == "ISM4" and br < 24400: + continue + + # Filter out cases where the input file does not exist for this bandwidth + if not os.path.isfile(os.path.join(SCRIPTS_DIR, f"testv/{in_file.format(bw=bw)}")): + continue + + tag = f"{fmt}-{br}-{bw}" + enc_opts = f"{in_fmt} {br} {bw} testv/{in_file.format(bw=bw)} bit" + dec_opts = f"{out_fmt} {bw} bit out.wav" + sim_opts = "" + eid_opts = "" + test_dict[tag] = (enc_opts, dec_opts, sim_opts, eid_opts) + + + +@pytest.mark.parametrize("test_tag", list(test_dict.keys())) +def test_enc( + test_info, + props_to_record, + encoder_only, + decoder_only, + dut_encoder_frontend: EncoderFrontend, + dut_decoder_frontend: DecoderFrontend, + ref_encoder_frontend: EncoderFrontend, + ref_decoder_frontend: DecoderFrontend, + reference_path, + dut_base_path, + test_vector_path, + update_ref, + rootdir, + keep_files, + test_tag, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_enc_stats, + get_odg, + compare_to_input, + peaq_binaural, +): + + enc_opts, dec_opts, sim_opts, eid_opts = test_dict[test_tag] + + run_test( + test_info, + props_to_record, + encoder_only, + decoder_only, + dut_encoder_frontend, + dut_decoder_frontend, + ref_encoder_frontend, + ref_decoder_frontend, + enc_opts, + dec_opts, + sim_opts, + eid_opts, + reference_path, + dut_base_path, + test_vector_path, + update_ref, + rootdir, + keep_files, + test_tag, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_enc_stats, + get_odg, + compare_to_input, + peaq_binaural, + ) + + \ No newline at end of file -- GitLab From 32ba6f44791fb2ea48be100495a8fb368ed20389 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 19 Dec 2024 08:13:35 +0100 Subject: [PATCH 35/40] Fix test names to match parse_xml_report.py --- scripts/parse_xml_report.py | 2 +- tests/test_enc_passthrough.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index d1c5f30e47..d68637c649 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -15,7 +15,7 @@ PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"] IVAS_FORMATS = { "Stereo": r"stereo", "ISM": r"ISM", - "Multichannel": r"Multi-channel", + "Multichannel": r"Multi-channel|MC", "MASA": r"(? Date: Tue, 7 Jan 2025 16:18:52 +0100 Subject: [PATCH 36/40] Disable FOA and HOA2 for OSBA, since it is not supported in ivas-float-update yet --- tests/test_enc_passthrough.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_enc_passthrough.py b/tests/test_enc_passthrough.py index dd2e41f69f..ba115f23ba 100644 --- a/tests/test_enc_passthrough.py +++ b/tests/test_enc_passthrough.py @@ -75,14 +75,15 @@ FORMAT = [ ("OMASA-2-2-2Dir","-ism_masa 2 2 testv/stvISM1.csv testv/stvISM2.csv testv/stv2MASA2TC48c.met","stvOMASA_2ISM_2MASA2TC{bw}c.wav","BINAURAL"), ("OMASA-3-2-2Dir","-ism_masa 3 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stv2MASA2TC48c.met","stvOMASA_3ISM_2MASA2TC{bw}c.wav","BINAURAL"), ("OMASA-4-2-2Dir","-ism_masa 4 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv testv/stv2MASA2TC48c.met","stvOMASA_4ISM_2MASA2TC{bw}c.wav","BINAURAL"), - ("OSBA-1-1","-ism_sba 1 1 testv/stvISM1.csv","stvOSBA_1ISM_FOA{bw}c.wav", "EXT"), - ("OSBA-2-1","-ism_sba 2 1 testv/stvISM1.csv testv/stvISM2.csv","stvOSBA_2ISM_FOA{bw}c.wav", "EXT"), - ("OSBA-3-1","-ism_sba 3 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv","stvOSBA_3ISM_FOA{bw}c.wav", "EXT"), - ("OSBA-4-1","-ism_sba 4 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv", "stvOSBA_4ISM_FOA{bw}c.wav", "EXT"), - ("OSBA-1-2","-ism_sba 1 2 testv/stvISM1.csv","stvOSBA_1ISM_2OA{bw}c.wav", "EXT"), - ("OSBA-2-2","-ism_sba 2 2 testv/stvISM1.csv testv/stvISM2.csv","stvOSBA_2ISM_2OA{bw}c.wav", "EXT"), - ("OSBA-3-2","-ism_sba 3 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv","stvOSBA_3ISM_2OA{bw}c.wav", "EXT"), - ("OSBA-4-2","-ism_sba 4 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv", "stvOSBA_4ISM_2OA{bw}c.wav", "EXT"), + # OSBA only supported for HOA3 in ivas-float-update for now + # ("OSBA-1-1","-ism_sba 1 1 testv/stvISM1.csv","stvOSBA_1ISM_FOA{bw}c.wav", "EXT"), + # ("OSBA-2-1","-ism_sba 2 1 testv/stvISM1.csv testv/stvISM2.csv","stvOSBA_2ISM_FOA{bw}c.wav", "EXT"), + # ("OSBA-3-1","-ism_sba 3 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv","stvOSBA_3ISM_FOA{bw}c.wav", "EXT"), + # ("OSBA-4-1","-ism_sba 4 1 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv", "stvOSBA_4ISM_FOA{bw}c.wav", "EXT"), + # ("OSBA-1-2","-ism_sba 1 2 testv/stvISM1.csv","stvOSBA_1ISM_2OA{bw}c.wav", "EXT"), + # ("OSBA-2-2","-ism_sba 2 2 testv/stvISM1.csv testv/stvISM2.csv","stvOSBA_2ISM_2OA{bw}c.wav", "EXT"), + # ("OSBA-3-2","-ism_sba 3 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv","stvOSBA_3ISM_2OA{bw}c.wav", "EXT"), + # ("OSBA-4-2","-ism_sba 4 2 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv", "stvOSBA_4ISM_2OA{bw}c.wav", "EXT"), ("OSBA-1-3","-ism_sba 1 3 testv/stvISM1.csv","stvOSBA_1ISM_3OA{bw}c.wav", "EXT"), ("OSBA-2-3","-ism_sba 2 3 testv/stvISM1.csv testv/stvISM2.csv","stvOSBA_2ISM_3OA{bw}c.wav", "EXT"), ("OSBA-3-3","-ism_sba 3 3 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv","stvOSBA_3ISM_3OA{bw}c.wav", "EXT"), -- GitLab From c14765d8154a74df1a0c1253957197d6d277fa5e Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 8 Jan 2025 14:58:02 +0100 Subject: [PATCH 37/40] Remove --peaq_binaural argument and use binauralization for --odg option --- tests/cmp_pcm.py | 63 +++++++++---------- .../test_param_file.py | 36 +++++++---- tests/codec_be_on_mr_nonselection/test_sba.py | 48 ++++++++++---- tests/conftest.py | 17 +---- tests/renderer/utils.py | 3 +- tests/test_enc_passthrough.py | 2 - 6 files changed, 93 insertions(+), 76 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 41c23a1490..9b837156bb 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -33,7 +33,9 @@ def cmp_pcm( abs_tol=0, get_ssnr=False, get_odg=False, - peaq_binaural=False, + odg_input=None, + odg_test=None, + odg_ref=None, ref_jbm_tf: Optional[Path] = None, cut_jbm_tf: Optional[Path] = None, ) -> (int, str): @@ -66,21 +68,8 @@ def cmp_pcm( fs = fs1 if fs1 != fs2: - if peaq_binaural: - fs = 48000 - s1 = np.clip( - pyaudio3dtools.audioarray.resample(s1.astype(float), fs1, fs), - -32768, - 32767, - ).astype(np.int16) - s2 = np.clip( - pyaudio3dtools.audioarray.resample(s2.astype(float), fs2, fs), - -32768, - 32767, - ).astype(np.int16) - else: - reason = "FAIL: Sampling rate differs." - return 1, reason + reason = "FAIL: Sampling rate differs." + return 1, reason # In case number of channels do not match, fail already now. Could happen in case of # comparison to input with for a non-passthrough mode. @@ -148,23 +137,31 @@ def cmp_pcm( reason += msg + " - " if get_odg: - if peaq_binaural: - pqeval_output = pqevalaudio_wrapper(s1, s2, fs) - - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - odg = float(match_odg.groups()[0]) - msg = f"ODG: {odg}" - reason += " - " + msg - print(msg) - else: - for n in range(nchannels): - pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) - - match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) - odg = float(match_odg.groups()[0]) - msg = f"Channel {n} ODG: {odg}" - reason += " - " + msg - print(msg) + odg_files = {} + for f in [odg_input, odg_test, odg_ref]: + # Load PEAQ test files and ensure 48 kHz sampling rate + s, fs = pyaudio3dtools.audiofile.readfile( + f, nchannels, fs, outdtype=np.int16 + ) + odg_files[f] = np.clip( + pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000), + -32768, + 32767, + ).astype(np.int16) + + pqeval_output = pqevalaudio_wrapper(odg_files[odg_input], odg_files[odg_ref], 48000) + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg_ref = float(match_odg.groups()[0]) + + pqeval_output = pqevalaudio_wrapper(odg_files[odg_input], odg_files[odg_test], 48000) + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg_test = float(match_odg.groups()[0]) + + odg = odg_test - odg_ref # Todo: store both rather than difference? + + msg = f"ODG: {odg}" + reason += " - " + msg + print(msg) return output_differs, reason diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 937c88ad98..905d5a6aad 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -205,7 +205,6 @@ def test_param_file_tests( get_enc_stats, get_odg, compare_to_input, - peaq_binaural, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -237,7 +236,6 @@ def test_param_file_tests( get_enc_stats, get_odg, compare_to_input, - peaq_binaural, ) @@ -268,7 +266,6 @@ def run_test( get_enc_stats, get_odg, compare_to_input, - peaq_binaural, ): # If compare_to_input is set, only run pass-through test cases @@ -497,12 +494,11 @@ def run_test( dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}" ref_output_file = f"{reference_path}/param_file/dec/{output_file}" - # Check if we need to render to stereo or binaural for PEAQ comparison - if peaq_binaural: - - allow_differing_lengths = True - # Disable tracefile supported comparison -- no tracefile available for input - tracefile_dec = "" + # Check if we need to render to mono, stereo or binaural for PEAQ comparison + odg_input = None + odg_test = None + odg_ref = None + if get_odg: # Find input format in_fmt = [(a, b) for (a, b) in INPUT_FMT if re.search(a, enc_opts)][0][1] @@ -513,7 +509,7 @@ def run_test( else ref_decoder_frontend.record_property ) - (ref_output_file, dut_output_file) = binauralize_input_and_output( + (odg_input, odg_test) = binauralize_input_and_output( record_property, props_to_record, test_info, @@ -528,6 +524,22 @@ def run_test( out_sr, ) + (_ , odg_ref) = binauralize_input_and_output( + record_property, + props_to_record, + test_info, + testv_file, + ref_output_file, + ref_output_file, + in_fmt, + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, + skip_input_rendering=True, + ) + # set to false per default even if this is no JBM case - makes later check for failure easier tracefile_last_rtp_numbers_differ = False @@ -572,7 +584,9 @@ def run_test( allow_differing_lengths=allow_differing_lengths, get_ssnr=get_ssnr, get_odg=get_odg, - peaq_binaural=peaq_binaural, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, ref_jbm_tf=ref_tracefile_dec, cut_jbm_tf=dut_tracefile_dec, ) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 7a3274b506..4afcd4a57c 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -1175,8 +1175,11 @@ def sba_dec( plc_file=plc_file, ) - if peaq_binaural: - allow_differing_lengths = True + # Check if we need to render to mono, stereo or binaural for PEAQ comparison + odg_input = None + odg_test = None + odg_ref = None + if get_odg: enc_opts = "" dec_opts = "" input_file = f"{test_vector_path}/{tag}.wav" @@ -1189,7 +1192,7 @@ def sba_dec( in_sr = sampling_rate out_sr = sampling_rate - (ref_out_file, dut_out_file) = binauralize_input_and_output( + (odg_input, odg_test) = binauralize_input_and_output( dut_decoder_frontend.record_property, props_to_record, test_info, @@ -1203,17 +1206,34 @@ def sba_dec( in_sr, out_sr, ) - else: - allow_differing_lengths = False - if compare_to_input: - # If comparing to input, set input as reference and allow different length + + (_, odg_ref) = binauralize_input_and_output( + dut_decoder_frontend.record_property, + props_to_record, + test_info, + input_file, + ref_out_file, + ref_out_file, + in_fmt, + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, + skip_input_rendering=True, + ) + + + allow_differing_lengths = False + if compare_to_input: + # If comparing to input, set input as reference and allow different length + ref_out_file = f"{test_vector_path}/{tag}.wav" + if gain_flag != -1: + ref_out_file = f"{test_vector_path}/{tag}_Gain{gain_flag}.wav" + else: ref_out_file = f"{test_vector_path}/{tag}.wav" - if gain_flag != -1: - ref_out_file = f"{test_vector_path}/{tag}_Gain{gain_flag}.wav" - else: - ref_out_file = f"{test_vector_path}/{tag}.wav" - allow_differing_lengths = True + allow_differing_lengths = True sampling_rate_Hz = int(sampling_rate) * 1000 cmp_result, reason = cmp_pcm( @@ -1227,7 +1247,9 @@ def sba_dec( allow_differing_lengths=allow_differing_lengths, get_ssnr=get_ssnr, get_odg=get_odg, - peaq_binaural=peaq_binaural, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, ) text_to_parse = reason diff --git a/tests/conftest.py b/tests/conftest.py index 26a1705ce3..d290c7dbc6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -274,14 +274,7 @@ def pytest_addoption(parser): help="Compare output to the input file instead of reference output. N.B. Only applicable to pass-through tests.", default=False, ) - - - parser.addoption( - "--peaq_binaural", - action="store_true", - help="Compare output to the input file using PEAQ. If input or output is not STEREO or BINAURAL, input and/or output is binauralized using the external renderer", - default=False, - ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -1003,7 +996,6 @@ def reference_path(request) -> str: if ( request.config.option.update_ref == "0" and not request.config.option.compare_to_input - and not request.config.option.peaq_binaural ): if not os.path.isdir(path): raise FileNotFoundError( @@ -1054,13 +1046,6 @@ def compare_to_input(request) -> bool: return request.config.getoption("--compare_to_input") -@pytest.fixture(scope="session", autouse=True) -def peaq_binaural(request) -> bool: - """ - Return value of cmdl param --peaq_binaural - """ - return request.config.getoption("--peaq_binaural") - def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial") diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index cbb2d99950..44348ba808 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -392,6 +392,7 @@ def binauralize_input_and_output( dec_opts, in_sr, out_sr, + skip_input_rendering=False, ): # Use current folder as location for temporary directory, since scene description does not handle spaces in path @@ -520,7 +521,7 @@ def binauralize_input_and_output( else: dut_output_file_binaural = dut_output_file - if in_fmt.upper() != output_config.upper(): + if in_fmt.upper() != output_config.upper() and not skip_input_rendering: # Render input to match output_config out_fmt = output_config diff --git a/tests/test_enc_passthrough.py b/tests/test_enc_passthrough.py index ba115f23ba..be9b7909fd 100644 --- a/tests/test_enc_passthrough.py +++ b/tests/test_enc_passthrough.py @@ -144,7 +144,6 @@ def test_enc( get_enc_stats, get_odg, compare_to_input, - peaq_binaural, ): enc_opts, dec_opts, sim_opts, eid_opts = test_dict[test_tag] @@ -176,7 +175,6 @@ def test_enc( get_enc_stats, get_odg, compare_to_input, - peaq_binaural, ) \ No newline at end of file -- GitLab From c39b017a8f6c4bb3b0c2f938e81d83cecb2f767a Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 9 Jan 2025 10:25:04 +0100 Subject: [PATCH 38/40] Add fix for rendering for PEAQ to avoid conflict with renderer tests --- tests/renderer/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 44348ba808..9a149bd7ea 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -172,6 +172,7 @@ def run_renderer( in_file=None, out_file=None, sr=48, + render_for_peaq=False, ) -> str: # prepare arguments and filepaths if trj_file is not None: @@ -310,7 +311,7 @@ def run_renderer( # run the renderer run_cmd(cmd, env) - if test_info.config.option.create_cut: + if test_info.config.option.create_cut and not render_for_peaq: # CUT creation mode will run a comparison with REF out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) @@ -515,6 +516,7 @@ def binauralize_input_and_output( in_file=dut_output_file, out_file=dut_output_file_binaural, sr=out_sr, + render_for_peaq=True, ) # Update output_config to rendered format output_config = output_reformat @@ -549,6 +551,7 @@ def binauralize_input_and_output( in_file=input_file, out_file=ref_output_file_binaural, sr=in_sr, + render_for_peaq=True, ) else: ref_output_file_binaural = input_file -- GitLab From 4ab254e02ed897c600dd1d29ae3b1ff6c60144a2 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 9 Jan 2025 12:05:00 +0100 Subject: [PATCH 39/40] Remove obsolete peaq-binaural job --- .gitlab-ci.yml | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ef6e4d3e9..55f66e4188 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,6 @@ variables: - 'ivas-conformance-linux' - 'check-clipping' - 'test-branch-vs-input-passthrough' - - 'peaq-binaural' GIT_CLEAN_FLAGS: -ffdxq TESTCASE_TIMEOUT_STV_SANITIZERS: 180 @@ -76,9 +75,6 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' variables: IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-binaural' - variables: - IVAS_PIPELINE_NAME: 'PEAQ comparison vs input on binauralized or mono signal: $CI_COMMIT_BRANCH' stages: - .pre @@ -281,8 +277,6 @@ stages: when: never - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' when: never - - if: $MANUAL_PIPELINE_TYPE == 'peaq-binaural' - when: never - when: on_success .rules-merge-request: @@ -1757,44 +1751,6 @@ test-branch-vs-input-passthrough: junit: - report-junit.xml - -peaq-binaural: - tags: - - ivas-linux - stage: compare - timeout: "30 minutes" # TBD - rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-binaural' - allow_failure: - exit_codes: - - 123 - script: - - *print-common-info - - make -j - - cp IVAS_rend IVAS_rend_ref - - - exit_code=0 - - python3 -m pytest tests/test_enc_passthrough.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --peaq_binaural --odg || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_xml_report.py report-junit.xml report.csv - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered."; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - artifacts: - name: "peaq-binaural--sha-$CI_COMMIT_SHORT_SHA--results" - when: always - expire_in: 4 mos - paths: - - report-junit.xml - - report.html - - report.csv - expose_as: "peaq-binaural results" - reports: - junit: - - report-junit.xml - - # --------------------------------------------------------------- # Scheduled jobs on main # --------------------------------------------------------------- -- GitLab From fbdadac83286a43735e5b45c994579c6bd3b2c89 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 9 Jan 2025 12:29:09 +0100 Subject: [PATCH 40/40] Remove peaq_binaural parameter from test_sba.py --- tests/codec_be_on_mr_nonselection/test_sba.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 4afcd4a57c..576ffd8a2f 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -109,7 +109,6 @@ def test_pca_enc( get_odg, get_enc_stats, compare_to_input, - peaq_binaural, ): pca = True bitrate = "256000" @@ -190,7 +189,6 @@ def test_pca_enc( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, - peaq_binaural=peaq_binaural, ) @@ -228,7 +226,6 @@ def test_sba_enc_system( get_odg, get_enc_stats, compare_to_input, - peaq_binaural, ): plc_pattern = None @@ -368,7 +365,6 @@ def test_sba_enc_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, - peaq_binaural=peaq_binaural, ) @@ -398,7 +394,6 @@ def test_spar_hoa2_enc_system( get_odg, get_enc_stats, compare_to_input, - peaq_binaural, ): sampling_rate = "48" pca = False @@ -512,7 +507,6 @@ def test_spar_hoa2_enc_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, - peaq_binaural=peaq_binaural, ) @@ -542,7 +536,6 @@ def test_spar_hoa3_enc_system( get_odg, get_enc_stats, compare_to_input, - peaq_binaural, ): sampling_rate = "48" pca = False @@ -650,7 +643,6 @@ def test_spar_hoa3_enc_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, - peaq_binaural=peaq_binaural, ) @@ -684,7 +676,6 @@ def test_sba_enc_BWforce_system( get_odg, get_enc_stats, compare_to_input, - peaq_binaural, ): sid = 0 plc_pattern = None @@ -808,7 +799,6 @@ def test_sba_enc_BWforce_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, - peaq_binaural=peaq_binaural, ) @@ -846,7 +836,6 @@ def test_sba_plc_system( get_odg, get_enc_stats, compare_to_input, - peaq_binaural, ): sid = 0 pca = False @@ -947,7 +936,6 @@ def test_sba_plc_system( get_ssnr=get_ssnr, get_odg=get_odg, compare_to_input=compare_to_input, - peaq_binaural=peaq_binaural, ) @@ -1107,7 +1095,6 @@ def sba_dec( get_ssnr=False, get_odg=False, compare_to_input=False, - peaq_binaural=False, ): dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" ref_pkt_dir = f"{reference_path}/sba_bs/pkt" -- GitLab