diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f532bfb1d8faad169958fd87c2babe6e86773e69..4c830957a91e64735ed024a9f461f50c1afc9cba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1307,11 +1307,8 @@ ivas-conformance: # Reference creation - python scripts/prepare_combined_format_inputs.py - - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" - - python -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files - - # Output creation - - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer_short/test_renderer.py", "tests/split_rendering/test_split_rendering.py" + - python -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1325,7 +1322,7 @@ ivas-conformance: - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - - mkdir testvec/testv/renderer + - mkdir testvec/testv/renderer_short - mkdir testvec/testv/split_rendering - mkdir testvec/bin - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv @@ -1335,9 +1332,8 @@ ivas-conformance: - cp -r -force -ErrorAction Ignore scripts/trajectories testvec - cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref - - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r -force -ErrorAction Ignore tests/renderer_short/ref testvec/testv/renderer_short/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/ref testvec/testv/split_rendering/ref - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns @@ -1407,11 +1403,8 @@ ivas-conformance-linux: # Reference creation - python3 scripts/prepare_combined_format_inputs.py - - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files - - # Output creation - - python3 -m pytest -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py" + - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1419,7 +1412,7 @@ ivas-conformance-linux: - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - - mkdir testvec/testv/renderer + - mkdir testvec/testv/renderer_short - mkdir testvec/testv/split_rendering - mkdir testvec/bin - cp -r scripts/testv/* testvec/testv @@ -1429,9 +1422,8 @@ ivas-conformance-linux: - cp -r scripts/trajectories testvec - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - cp -r tests/ref testvec/testv/ref - - cp -r tests/dut/* testvec/testv/ref - - cp -r tests/renderer/cut testvec/testv/renderer/ref - - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref + - cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns diff --git a/scripts/dec_header.txt b/scripts/dec_header.txt index 599c7e73ed9171ebb930dd06662ec195ff611448..68aff3635581bc8586825f3ceeb9ac7823cafa11 100644 --- a/scripts/dec_header.txt +++ b/scripts/dec_header.txt @@ -16,7 +16,7 @@ LOG_FILE=Readme_IVAS_dec_log.txt rm -rf tmp rm -rf $CUT_PATH mkdir -p $CUT_PATH -mkdir -p $CUT_PATH/dut/masa_test/dec_output -mkdir -p $CUT_PATH/dut/param_file/dec -mkdir -p $CUT_PATH/dut/sba_bs/raw +mkdir -p $CUT_PATH/ref/masa_test/dec_output +mkdir -p $CUT_PATH/ref/param_file/dec +mkdir -p $CUT_PATH/ref/sba_bs/raw diff --git a/scripts/dec_isar_header.txt b/scripts/dec_isar_header.txt index 454383382f916f091bebd25166b1dba9df32b651..0f449b21c2a70530fc6dbaac718cf2208d95a1af 100644 --- a/scripts/dec_isar_header.txt +++ b/scripts/dec_isar_header.txt @@ -16,6 +16,6 @@ LOG_FILE=Readme_IVAS_ISAR_dec_log.txt rm -rf tmp rm -rf $CUT_PATH mkdir -p $CUT_PATH -mkdir -p $CUT_PATH/split_rendering/cut +mkdir -p $CUT_PATH/split_rendering/ref diff --git a/scripts/enc_header.txt b/scripts/enc_header.txt index 18387a9326fc3a7342ad272283641a78263e70f3..76ebeee3e1b1ed66a64e8548364248019a2f1587 100644 --- a/scripts/enc_header.txt +++ b/scripts/enc_header.txt @@ -16,7 +16,7 @@ LOG_FILE=Readme_IVAS_enc_log.txt rm -rf tmp rm -rf $CUT_PATH mkdir -p $CUT_PATH -mkdir -p $CUT_PATH/dut/masa_test/bitstreams -mkdir -p $CUT_PATH/dut/param_file/enc -mkdir -p $CUT_PATH/dut/sba_bs/pkt +mkdir -p $CUT_PATH/ref/masa_test/bitstreams +mkdir -p $CUT_PATH/ref/param_file/enc +mkdir -p $CUT_PATH/ref/sba_bs/pkt diff --git a/scripts/isar_post_rend_header.txt b/scripts/isar_post_rend_header.txt index ed7eb2098866cbcbdcd8e3c4fc7c507ec3ba8415..99a84b6c36a0a476b9617c5b8befc5c449433a95 100644 --- a/scripts/isar_post_rend_header.txt +++ b/scripts/isar_post_rend_header.txt @@ -15,7 +15,7 @@ LOG_FILE=Readme_IVAS_isar_post_rend_log.txt rm -rf tmp rm -rf $CUT_PATH -mkdir -p $CUT_PATH/split_rendering/cut +mkdir -p $CUT_PATH/split_rendering/ref diff --git a/scripts/jbm_header.txt b/scripts/jbm_header.txt index d244c83b3f63757139e057810c66b467f4941957..57f59809f9bd0cd076daa8f447803e9a8859daf4 100644 --- a/scripts/jbm_header.txt +++ b/scripts/jbm_header.txt @@ -16,7 +16,7 @@ LOG_FILE=Readme_IVAS_jbm_log.txt rm -rf tmp rm -rf $CUT_PATH mkdir -p $CUT_PATH -mkdir -p $CUT_PATH/dut/masa_test/dec_output -mkdir -p $CUT_PATH/dut/param_file/dec -mkdir -p $CUT_PATH/dut/sba_bs/raw +mkdir -p $CUT_PATH/ref/masa_test/dec_output +mkdir -p $CUT_PATH/ref/param_file/dec +mkdir -p $CUT_PATH/ref/sba_bs/raw diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 3a5fb6c1afbda1a07593b1c5a3e1c7867490fe00..c5f4cd71fdaa6982313b270b40938489cece5edb 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -13,7 +13,7 @@ if __name__ == '__main__': parser.add_argument('txt_file',type=str,help='Output txt file, e.g. output.txt') args = parser.parse_args() input = args.input - txt_file = args.txt_file + txt_file = args.txt_file TESTV_PATH='$TESTV_PATH' REF_PATH='$REF_PATH' @@ -26,6 +26,7 @@ if __name__ == '__main__': cmds_rend=[] cmds_isar_post_rend=[] + all_args = set() if path.isdir(input): input = Path(input).rglob('*.html') @@ -35,8 +36,8 @@ if __name__ == '__main__': with open(html_report,'r') as infile: for line in infile.readlines(): - cmds_enc.extend(re.findall(r"DUT encoder command:\\n\\t(.*?)\\n", line)) - cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line)) + cmds_enc.extend(re.findall(r"REF encoder command:\\n\\t(.*?)\\n", line)) + cmds_dec.extend(re.findall(r"REF decoder command:\\n\\t(.*?)\\n", line)) cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line)) cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line)) @@ -63,9 +64,9 @@ if __name__ == '__main__': cmds_isar_post_rend.append(line) isar_post_rend_cmd = False else: - if "DUT encoder command" in line: + if "REF encoder command" in line: enc_cmd = True - elif "DUT decoder command" in line: + elif "REF decoder command" in line: dec_cmd = True elif "Running command" in line: rend_cmd = True @@ -78,6 +79,18 @@ if __name__ == '__main__': cmds_rend.sort() cmds_isar_post_rend.sort() + # Remove duplicates from cmds_enc -- some decoder tests use same encoder options + # Relies on the list being sorted + i = 0 + while i + 1 < len(cmds_enc): + if " ".join(cmds_enc[i].split()[:-1]) == " ".join(cmds_enc[i+1].split()[:-1]): + del cmds_enc[i+1] + else: + i = i + 1 + + # Filter out networkSimulator_g192 commands from cmds_rend + cmds_rend = [cmd for cmd in cmds_rend if not "networkSimulator_g192" in cmd] + with open(txt_file.replace('.','_enc.'),'w', newline='\n') as outfile: with open('scripts/enc_header.txt','r') as header: outfile.write(header.read()) @@ -87,7 +100,7 @@ if __name__ == '__main__': # Adjust file arguments, pass other arguments as they are if path.exists(arg): arg = path.relpath(arg).replace('\\','/') - arg = re.sub('IVAS_cod(.exe)?', '$CUT_ENC_BIN', arg) + arg = re.sub('IVAS_cod_ref(.exe)?', '$CUT_ENC_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) arg = re.sub('tests', CUT_PATH, arg) args.append(arg) @@ -95,7 +108,7 @@ if __name__ == '__main__': outfile.write(cmd+'\n') bts = re.search(r"\s(([\S]+)(.bts|.192|.pkt|.fer))$", cmd) if bts: - outfile.write('$DIFF_BIN '+bts.group(1).replace(CUT_PATH + r'/dut',REF_PATH + r'/ref')+' '+bts.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('$DIFF_BIN '+bts.group(1).replace(CUT_PATH, REF_PATH)+' '+bts.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) @@ -111,22 +124,25 @@ if __name__ == '__main__': absolute_out = re.search(r"\s(([\S]+)(.wav))$", cmd) args = [] + input_set = False for arg in cmd.split(): # Adjust file arguments, pass other arguments as they are if path.exists(arg): arg = path.relpath(arg).replace('\\','/') - arg = re.sub('IVAS_dec(.exe)?', '$CUT_DEC_BIN', arg) + arg = re.sub('IVAS_dec_ref(.exe)?', '$CUT_DEC_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) - arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # For .fer cases the bitstream is in ref - arg = re.sub('tests/split_rendering/renderer_configs', REF_PATH + r'/split_rendering/renderer_configs', arg) - if re.search("^tests.*192$",arg): - arg = re.sub('tests/split_rendering/cut', REF_PATH + r'/split_rendering/ref', arg) - if re.search("\.wav$",arg): - arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav - else: - arg = re.sub('tests/dut', REF_PATH + r'/ref', arg) # Input argument - if re.search("^tests.*bit$",arg): - arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav + # Identify input files + if '.txt' in arg: + arg = re.sub('tests', REF_PATH, arg) + # Identify special cases of output files: dectrace, spltmd.bit + if 'spltmd.bit' in arg or 'dectrace' in arg: + arg = re.sub('tests', CUT_PATH, arg) + if 'tests' in arg: + if not input_set: + arg = re.sub('tests', REF_PATH, arg) # First occurence of tests/* is the input file + input_set = True + else: + arg = re.sub('tests', CUT_PATH, arg) # Remaining occurences of tests/* are output files args.append(arg) cmd = ' '.join(args) @@ -146,12 +162,12 @@ if __name__ == '__main__': for output in glob.glob(absolute_out.group(1) + '*'): output = path.relpath(output).replace('\\','/') output = re.sub('tests', CUT_PATH, output) - diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/dut',REF_PATH + r'/ref')+' '+output+' >> $LOG_FILE 2>&1') + diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH, REF_PATH)+' '+output+' >> $LOG_FILE 2>&1') outfile.write(('; ').join(diff_cmds)) - if isar_out and "cut" in isar_out.group(1): - outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1') - if isar_md_out and "cut" in isar_md_out.group(1): - outfile.write('; $DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n') + if isar_out and "ref" in isar_out.group(1): + outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1') + if isar_md_out and "ref" in isar_md_out.group(1): + outfile.write('; $DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n') else: outfile.write('\n') outfile.write('\n\n') @@ -171,18 +187,18 @@ if __name__ == '__main__': # Adjust file arguments, pass other arguments as they are if path.exists(arg): arg = path.relpath(arg).replace('\\','/') - arg = re.sub('IVAS_rend(.exe)?', '$CUT_REND_BIN', arg) + arg = re.sub('IVAS_rend_ref(.exe)?', '$CUT_REND_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) - arg = re.sub('tests/renderer/data', TESTV_PATH + r'renderer/data/', arg) + arg = re.sub('tests/renderer_short/data', TESTV_PATH + r'renderer_short/data/', arg) arg = re.sub('tests', CUT_PATH, arg) args.append(arg) cmd = ' '.join(args) - if "cut" in cmd: + if "ref" in cmd: outfile.write(cmd+'\n') out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) - if out and "cut" in out.group(1): - outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/renderer/cut',REF_PATH + r'/renderer/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') + if out and "ref" in out.group(1): + outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/renderer_short/ref',REF_PATH + r'/renderer_short/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) @@ -196,19 +212,20 @@ if __name__ == '__main__': # Adjust file arguments, pass other arguments as they are if path.exists(arg): arg = path.relpath(arg).replace('\\','/') - arg = re.sub('ISAR_post_rend(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg) + arg = re.sub('ISAR_post_rend_ref(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) if re.search("^tests.*bit$",arg): - arg = re.sub('tests/split_rendering/cut', REF_PATH + r'/split_rendering/ref', arg) + arg = re.sub('tests', REF_PATH, arg) arg = re.sub('tests', CUT_PATH, arg) args.append(arg) cmd = ' '.join(args) - if "cut" in cmd: + if "ref" in cmd: outfile.write(cmd+'\n') out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) - if out and "cut" in out.group(1): - outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') + if out and "ref" in out.group(1): + outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH, REF_PATH)+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: - outfile.write(footer.read()) \ No newline at end of file + outfile.write(footer.read()) + diff --git a/scripts/rend_header.txt b/scripts/rend_header.txt index c56eb17cb0844a882521f83377f146b1169231c2..ce2e274c6698bccd11aef138606b7106fa7167fd 100644 --- a/scripts/rend_header.txt +++ b/scripts/rend_header.txt @@ -15,7 +15,7 @@ LOG_FILE=Readme_IVAS_rend_log.txt rm -rf tmp rm -rf $CUT_PATH -mkdir -p $CUT_PATH/renderer/cut -mkdir -p $CUT_PATH/renderer/data +mkdir -p $CUT_PATH/renderer_short/ref +mkdir -p $CUT_PATH/renderer_short/data diff --git a/scripts/testv/mixed_mc714_foa_masa2_ism4.txt b/scripts/testv/mixed_mc714_foa_masa2_ism4.txt new file mode 100644 index 0000000000000000000000000000000000000000..800e88ae7cf96a208680d099d74bd91299a08d5e --- /dev/null +++ b/scripts/testv/mixed_mc714_foa_masa2_ism4.txt @@ -0,0 +1,24 @@ +mixed_mc714_foa_masa2_ism4.wav +7 +MC +1 +7_1_4 +SBA +13 +1 +MASA +17 +2 +stv2MASA2TC48c.met +ISM +19 +stvISM1.csv +ISM +20 +stvISM2.csv +ISM +21 +stvISM3.csv +ISM +22 +stvISM4.csv diff --git a/scripts/testv/mixed_mc714_foa_masa2_ism4.wav b/scripts/testv/mixed_mc714_foa_masa2_ism4.wav new file mode 100644 index 0000000000000000000000000000000000000000..c680493c22891dcedc83b3da85572dbf7d1c353f --- /dev/null +++ b/scripts/testv/mixed_mc714_foa_masa2_ism4.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900feffb59cec3a831c23b07e6f2f305e74d382c4d64903e65cd7cc321bd7083 +size 6336044 diff --git a/scripts/testv/mixed_scene.txt b/scripts/testv/mixed_scene.txt new file mode 100644 index 0000000000000000000000000000000000000000..2501c75383abc8f7c27aa6b64beca353d721e33b --- /dev/null +++ b/scripts/testv/mixed_scene.txt @@ -0,0 +1,15 @@ +spectral_test_16ch_48kHz.wav +4 +ISM +1 +ism_0a_0e.csv +ISM +2 +1 +1,-30,0 +SBA +3 +1 +MC +7 +5_1_4 diff --git a/scripts/testv/mixed_scene_simple.txt b/scripts/testv/mixed_scene_simple.txt new file mode 100644 index 0000000000000000000000000000000000000000..85d38b8e939aac8cf9c06a5f1ef5dc780b8db2a1 --- /dev/null +++ b/scripts/testv/mixed_scene_simple.txt @@ -0,0 +1,12 @@ +spectral_test_4ch_48kHz.wav +3 +ISM +1 +ism_0a_0e.csv +ISM +2 +1 +1,-30,0 +MC +3 +STEREO diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 2a0d50568cf4e665b9d5e9569da76e61c5c91f39..9acb1b7f18101c802f3c9edb5b65af498ce85806 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -109,7 +109,10 @@ def run_ivas_isar_enc_cmd(cmd, env=None): def run_ivas_isar_dec_cmd(cmd, env=None): - logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") + if BIN_SUFFIX_MERGETARGET in cmd[0]: + logging.info(f"\nREF decoder command:\n\t{' '.join(cmd)}\n") + else: + logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") _run_cmd(cmd, env) diff --git a/tests/renderer_short/README.md b/tests/renderer_short/README.md new file mode 100644 index 0000000000000000000000000000000000000000..713821c5d389adaaf89de09b0ec60d4aee84dacf --- /dev/null +++ b/tests/renderer_short/README.md @@ -0,0 +1,44 @@ +======================================== +THIS FOLDER WILL NOT BE PART OF DELIVERY +======================================== + +# External Renderer Tests + +See also the [contribution page](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/Contributions/2-external-renderer) for related presentations. + +### Run tests with: + +## Smoke test: + +```bash +python3 -m pytest -q -n auto tests/renderer/test_renderer.py +``` + +## Comparison test: + +```bash +python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref # requires IVAS_rend_ref in root! +python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut +``` + +### Important flags (see [pytest docs](https://docs.pytest.org/en/7.2.x/) for more information): + +- `-k` flag can filter test cases, e.g. `-k "test_ism_binaural_static"` +- `-rA` reports ALL (pass, xpass, xfail, fail) instead of the default behaviour of reporting only failed tests\ + this option will also report captured logs, **required for obtaining the commandline of testcases that pass or xfail** +- `--last-failed` re-runs only the cases that failed in the last test run +- `--collect-only` is useful when adding new testcases to check if argument parametrization is working correctly + +### Directory tree + +``` +. +├── compare_audio.py -> Python implementation of CompAudio, used for comparisons in tests +├── conftest.py -> Pytest configuration (enable commandline argument ingestion) +├── constants.py -> Important paths, formats, metadata files and commandline templates +├── cut -> Default location for output files for test conditions +├── data -> Input test vectors +├── ref -> Default location for output files for reference conditions +├── test_renderer.py -> Runs the renderer for all modes +└── utils.py -> Wrapper functions for executables for use in testcases +``` diff --git a/tests/renderer_short/__init__.py b/tests/renderer_short/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..26555940430c214d3c47dda947bf5fa3207220ae --- /dev/null +++ b/tests/renderer_short/__init__.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +""" + (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" diff --git a/tests/renderer_short/compare_audio.py b/tests/renderer_short/compare_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..49bfa86c062b830cd791a5bcbc1aa48828e50eed --- /dev/null +++ b/tests/renderer_short/compare_audio.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 + +""" + (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import sys +import warnings +from typing import Tuple + +import numpy as np + +from .constants import SCRIPTS_DIR + +sys.path.append(str(SCRIPTS_DIR)) +from pyaudio3dtools.audioarray import getdelay + + +def compare_audio_arrays( + left: np.ndarray, left_fs: int, right: np.ndarray, right_fs: int +) -> Tuple[float, float]: + if left_fs != right_fs: + return ValueError(f"Differing samplerates: {left_fs} vs {right_fs}!") + + if left.shape[1] != right.shape[1]: + cmp_ch = min(left.shape[1], right.shape[1]) + warnings.warn( + f"Differing number of channels: {left.shape[1]} vs {right.shape[1]}! Comparing first {cmp_ch} channel(s)", + category=RuntimeWarning, + ) + left = left[:, :cmp_ch] + right = right[:, :cmp_ch] + + if left.shape[0] != right.shape[0]: + cmp_smp = min(left.shape[0], right.shape[0]) + warnings.warn( + f"Warning - different durations: {left.shape[0] / left_fs:.2f}s vs {right.shape[0] / right_fs:.2f}s! Comparing first {cmp_smp / left_fs : .2f} sample(s)", + category=RuntimeWarning, + ) + left = left[:cmp_smp, :] + right = right[:cmp_smp, :] + + if not np.array_equal(left, right): + delay = getdelay(left, right) + delay_abs = np.abs(delay) + # getdelay can return large values if signals are quite different + # limit any delay compensation to 20 ms + if delay != 0 and (delay_abs < left_fs / 50): + warnings.warn( + f"File B is delayed by {delay} samples ({delay*1000 / left_fs : .2f}ms)!", + category=RuntimeWarning, + ) + + # shift array + left = np.roll(left, delay, axis=0) + + # zero shifted out samples + if delay < 0: + left[-np.abs(delay) :, :] = 0 + elif delay > 0: + left[: np.abs(delay), :] = 0 + """ + http://www-mmsp.ece.mcgill.ca/Documents/Software/Packages/AFsp/AFsp/CompAudio.html + """ + num = np.sum(left * right) + den = np.sqrt(np.sum(left**2) * np.sum(right**2)) + if den > 0: + r = num / den + else: + r = np.inf + snr = 10 * np.log10(1 / (1 - (r**2))) + gain_b = num / np.sum(right**2) + max_diff = np.abs(np.max(left - right)) + else: + snr = np.inf + gain_b = 1 + max_diff = 0 + + return snr, gain_b, max_diff diff --git a/tests/renderer_short/constants.py b/tests/renderer_short/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..766e64722b80132b963c007880ac732222855dfc --- /dev/null +++ b/tests/renderer_short/constants.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 + +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +from pathlib import Path +import platform + +""" Set up paths """ +TESTS_DIR = Path(__file__).parent +SCRIPTS_DIR = TESTS_DIR.parents[1].joinpath("scripts").resolve() +TEST_VECTOR_DIR = SCRIPTS_DIR.joinpath("testv") + +OUTPUT_PATH_REF = TESTS_DIR.joinpath("ref") +OUTPUT_PATH_CUT = TESTS_DIR.joinpath("cut") + +CUSTOM_LAYOUT_DIR = SCRIPTS_DIR.joinpath("ls_layouts") +HR_TRAJECTORY_DIR = SCRIPTS_DIR.joinpath("trajectories") +TESTV_DIR = SCRIPTS_DIR.joinpath("testv") +LTV_DIR = TESTV_DIR + +BIN_SUFFIX_MERGETARGET = "_ref" + +if platform.system() == "Windows": + EXE_SUFFIX = ".exe" +elif platform.system() in ["Linux", "Darwin"]: + EXE_SUFFIX = "" +else: + assert False, f"Unsupported platform {platform.system()}" + +""" Renderer commandline template """ +RENDERER_CMD = [ + str(TESTS_DIR.parent.parent.joinpath("IVAS_rend")), + "-i", + "", # 2 -> input file + "-if", + "", # 4 -> input format + "-o", + "/dev/null", # 6 -> output file + "-of", + "", # 8 -> output format + "-fs", + "48", # 10 -> input fs + "-no_delay_cmp", + # "-ndl", + "-q", +] + + +""" Format to file mappings """ +NCHAN_TO_FILE = { + 1: TEST_VECTOR_DIR.joinpath("spectral_test_1ch_48kHz.wav"), + 2: TEST_VECTOR_DIR.joinpath("spectral_test_2ch_48kHz.wav"), + 3: TEST_VECTOR_DIR.joinpath("spectral_test_3ch_48kHz.wav"), + 4: TEST_VECTOR_DIR.joinpath("spectral_test_4ch_48kHz.wav"), + 5: TEST_VECTOR_DIR.joinpath("spectral_test_5ch_48kHz.wav"), + 6: TEST_VECTOR_DIR.joinpath("spectral_test_6ch_48kHz.wav"), + 7: TEST_VECTOR_DIR.joinpath("spectral_test_7ch_48kHz.wav"), + 8: TEST_VECTOR_DIR.joinpath("spectral_test_8ch_48kHz.wav"), + 9: TEST_VECTOR_DIR.joinpath("spectral_test_9ch_48kHz.wav"), + 10: TEST_VECTOR_DIR.joinpath("spectral_test_10ch_48kHz.wav"), + 11: TEST_VECTOR_DIR.joinpath("spectral_test_11ch_48kHz.wav"), + 12: TEST_VECTOR_DIR.joinpath("spectral_test_12ch_48kHz.wav"), + 13: TEST_VECTOR_DIR.joinpath("spectral_test_13ch_48kHz.wav"), + 15: TEST_VECTOR_DIR.joinpath("spectral_test_15ch_48kHz.wav"), + 16: TEST_VECTOR_DIR.joinpath("spectral_test_16ch_48kHz.wav"), + 17: TEST_VECTOR_DIR.joinpath("spectral_test_17ch_48kHz.wav"), + 18: TEST_VECTOR_DIR.joinpath("spectral_test_18ch_48kHz.wav"), + 19: TEST_VECTOR_DIR.joinpath("spectral_test_19ch_48kHz.wav"), + 20: TEST_VECTOR_DIR.joinpath("spectral_test_20ch_48kHz.wav"), +} + +FORMAT_TO_FILE_SMOKETEST = { + "MONO": NCHAN_TO_FILE[1], + "STEREO": NCHAN_TO_FILE[2], + "5_1": NCHAN_TO_FILE[6], + "7_1": NCHAN_TO_FILE[8], + "5_1_2": NCHAN_TO_FILE[8], + "5_1_4": NCHAN_TO_FILE[10], + "7_1_4": NCHAN_TO_FILE[12], + "FOA": NCHAN_TO_FILE[4], + "HOA2": NCHAN_TO_FILE[9], + "HOA3": NCHAN_TO_FILE[16], + "ISM1": NCHAN_TO_FILE[1], + "ISM2": NCHAN_TO_FILE[2], + "ISM3": NCHAN_TO_FILE[3], + "ISM4": NCHAN_TO_FILE[4], + "NDP_ISM4": NCHAN_TO_FILE[4], + "MASA1": NCHAN_TO_FILE[1], + "MASA2": NCHAN_TO_FILE[2], + "OMASA_1_1": NCHAN_TO_FILE[2], + "OMASA_1_2": NCHAN_TO_FILE[3], + "OMASA_1_3": NCHAN_TO_FILE[4], + "OMASA_1_4": NCHAN_TO_FILE[5], + "OMASA_2_1": NCHAN_TO_FILE[3], + "OMASA_2_2": NCHAN_TO_FILE[4], + "OMASA_2_3": NCHAN_TO_FILE[5], + "OMASA_2_4": NCHAN_TO_FILE[6], + "OSBA_1_1": NCHAN_TO_FILE[5], + "OSBA_2_1": NCHAN_TO_FILE[6], + "OSBA_3_1": NCHAN_TO_FILE[7], + "OSBA_4_1": NCHAN_TO_FILE[8], + "OSBA_1_2": NCHAN_TO_FILE[10], + "OSBA_2_2": NCHAN_TO_FILE[11], + "OSBA_3_2": NCHAN_TO_FILE[12], + "OSBA_4_2": NCHAN_TO_FILE[13], + "OSBA_1_3": NCHAN_TO_FILE[17], + "OSBA_2_3": NCHAN_TO_FILE[18], + "OSBA_3_3": NCHAN_TO_FILE[19], + "OSBA_4_3": NCHAN_TO_FILE[20], + "META": TEST_VECTOR_DIR.joinpath("mixed_scene.txt"), + "16ch_8+4+4": NCHAN_TO_FILE[16], + "4d4": NCHAN_TO_FILE[8], + "t_design_4": NCHAN_TO_FILE[12], +} + +FORMAT_TO_FILE_COMPARETEST = { + "MONO": TESTV_DIR.joinpath("stv48c.wav"), + "STEREO": TESTV_DIR.joinpath("stvST48c.wav"), + "5_1": TESTV_DIR.joinpath("stv51MC48c.wav"), + "7_1": TESTV_DIR.joinpath("stv71MC48c.wav"), + "5_1_2": TESTV_DIR.joinpath("stv512MC48c.wav"), + "5_1_4": TESTV_DIR.joinpath("stv514MC48c.wav"), + "7_1_4": TESTV_DIR.joinpath("stv714MC48c.wav"), + "FOA": TESTV_DIR.joinpath("stvFOA48c.wav"), + "HOA2": TESTV_DIR.joinpath("stv2OA48c.wav"), + "HOA3": TESTV_DIR.joinpath("stv3OA48c.wav"), + "ISM1": TESTV_DIR.joinpath("stv1ISM48s.wav"), + "ISM2": TESTV_DIR.joinpath("stv2ISM48s.wav"), + "ISM3": TESTV_DIR.joinpath("stv3ISM48s.wav"), + "ISM4": TESTV_DIR.joinpath("stv4ISM48s.wav"), + "MASA1": TESTV_DIR.joinpath("stv1MASA1TC48c.wav"), + "MASA2": TESTV_DIR.joinpath("stv2MASA2TC48c.wav"), + "OMASA_1_1": TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA1TC48c.wav"), + "OMASA_1_2": TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA1TC48c.wav"), + "OMASA_1_3": TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA1TC48c.wav"), + "OMASA_1_4": TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA1TC48c.wav"), + "OMASA_2_1": TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA2TC48c.wav"), + "OMASA_2_2": TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA2TC48c.wav"), + "OMASA_2_3": TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA2TC48c.wav"), + "OMASA_2_4": TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA2TC48c.wav"), + "OSBA_1_1": TESTV_DIR.joinpath("stvOSBA_1ISM_FOA48c.wav"), + "OSBA_1_2": TESTV_DIR.joinpath("stvOSBA_1ISM_2OA48c.wav"), + "OSBA_1_3": TESTV_DIR.joinpath("stvOSBA_1ISM_3OA48c.wav"), + "OSBA_2_1": TESTV_DIR.joinpath("stvOSBA_2ISM_FOA48c.wav"), + "OSBA_2_2": TESTV_DIR.joinpath("stvOSBA_2ISM_2OA48c.wav"), + "OSBA_2_3": TESTV_DIR.joinpath("stvOSBA_2ISM_3OA48c.wav"), + "OSBA_3_1": TESTV_DIR.joinpath("stvOSBA_3ISM_FOA48c.wav"), + "OSBA_3_2": TESTV_DIR.joinpath("stvOSBA_3ISM_2OA48c.wav"), + "OSBA_3_3": TESTV_DIR.joinpath("stvOSBA_3ISM_3OA48c.wav"), + "OSBA_4_1": TESTV_DIR.joinpath("stvOSBA_4ISM_FOA48c.wav"), + "OSBA_4_2": TESTV_DIR.joinpath("stvOSBA_4ISM_2OA48c.wav"), + "OSBA_4_3": TESTV_DIR.joinpath("stvOSBA_4ISM_3OA48c.wav"), + "META": TEST_VECTOR_DIR.joinpath("mixed_scene.txt"), + "16ch_8+4+4": TESTV_DIR.joinpath("stv3OA48c.wav"), + "4d4": TESTV_DIR.joinpath("stv71MC48c.wav"), + "t_design_4": TESTV_DIR.joinpath("stv714MC48c.wav"), +} + +FORMAT_TO_FILE_LTV = { + "MONO": LTV_DIR.joinpath("ltv48_MONO.wav"), + "STEREO": LTV_DIR.joinpath("ltv48_STEREO.wav"), + "5_1": LTV_DIR.joinpath("ltv48_MC51.wav"), + "7_1": LTV_DIR.joinpath("ltv48_MC71.wav"), + "5_1_2": LTV_DIR.joinpath("ltv48_MC512.wav"), + "5_1_4": LTV_DIR.joinpath("ltv48_MC514.wav"), + "7_1_4": LTV_DIR.joinpath("ltv48_MC714.wav"), + "FOA": LTV_DIR.joinpath("ltv48_FOA.wav"), + "HOA2": LTV_DIR.joinpath("ltv48_HOA2.wav"), + "HOA3": LTV_DIR.joinpath("ltv48_HOA3.wav"), + "ISM1": LTV_DIR.joinpath("ltv48_1ISM.wav"), + "ISM2": LTV_DIR.joinpath("ltv48_2ISM.wav"), + "ISM3": LTV_DIR.joinpath("ltv48_3ISM.wav"), + "ISM4": LTV_DIR.joinpath("ltv48_4ISM.wav"), + "MASA1": LTV_DIR.joinpath("ltv48_MASA1TC.wav"), + "MASA2": LTV_DIR.joinpath("ltv48_MASA2TC.wav"), + "OMASA_1_1": LTV_DIR.joinpath("ltv48_OMASA_1ISM_1TC.wav"), + "OMASA_1_2": LTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.wav"), + "OMASA_1_3": LTV_DIR.joinpath("ltv48_OMASA_3ISM_1TC.wav"), + "OMASA_1_4": LTV_DIR.joinpath("ltv48_OMASA_4ISM_1TC.wav"), + "OMASA_2_1": LTV_DIR.joinpath("ltv48_OMASA_1ISM_2TC.wav"), + "OMASA_2_2": LTV_DIR.joinpath("ltv48_OMASA_2ISM_2TC.wav"), + "OMASA_2_3": LTV_DIR.joinpath("ltv48_OMASA_3ISM_2TC.wav"), + "OMASA_2_4": LTV_DIR.joinpath("ltv48_OMASA_4ISM_2TC.wav"), + "OSBA_1_1": LTV_DIR.joinpath("ltv48_OSBA_1ISM_FOA.wav"), + "OSBA_1_2": LTV_DIR.joinpath("ltv48_OSBA_1ISM_HOA2.wav"), + "OSBA_1_3": LTV_DIR.joinpath("ltv48_OSBA_1ISM_HOA3.wav"), + "OSBA_2_1": LTV_DIR.joinpath("ltv48_OSBA_2ISM_FOA.wav"), + "OSBA_2_2": LTV_DIR.joinpath("ltv48_OSBA_2ISM_HOA2.wav"), + "OSBA_2_3": LTV_DIR.joinpath("ltv48_OSBA_2ISM_HOA3.wav"), + "OSBA_3_1": LTV_DIR.joinpath("ltv48_OSBA_3ISM_FOA.wav"), + "OSBA_3_2": LTV_DIR.joinpath("ltv48_OSBA_3ISM_HOA2.wav"), + "OSBA_3_3": LTV_DIR.joinpath("ltv48_OSBA_3ISM_HOA3.wav"), + "OSBA_4_1": LTV_DIR.joinpath("ltv48_OSBA_4ISM_FOA.wav"), + "OSBA_4_2": LTV_DIR.joinpath("ltv48_OSBA_4ISM_HOA2.wav"), + "OSBA_4_3": LTV_DIR.joinpath("ltv48_OSBA_4ISM_HOA3.wav"), + "META": TEST_VECTOR_DIR.joinpath("mixed_scene.txt"), + "16ch_8+4+4": LTV_DIR.joinpath("ltv48_HOA3.wav"), + "4d4": LTV_DIR.joinpath("ltv48_MC71.wav"), + "t_design_4": LTV_DIR.joinpath("ltv48_MC714.wav"), +} + +FORMAT_TO_METADATA_FILES = { + "ISM1": [str(TESTV_DIR.joinpath("stvISM1.csv"))], + "ISM2": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + ], + "ISM3": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + ], + "ISM4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + ], + "NDP_ISM4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2_non-diegetic-pan.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + ], + "MASA1": [str(TESTV_DIR.joinpath("stv1MASA1TC48c.met"))], + "MASA2": [str(TESTV_DIR.joinpath("stv2MASA2TC48c.met"))], + "OMASA_1_1": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA1TC48c.met")), + ], + "OMASA_1_2": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA1TC48c.met")), + ], + "OMASA_1_3": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA1TC48c.met")), + ], + "OMASA_1_4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + str(TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA1TC48c.met")), + ], + "OMASA_2_1": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA2TC48c.met")), + ], + "OMASA_2_2": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA2TC48c.met")), + ], + "OMASA_2_3": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA2TC48c.met")), + ], + "OMASA_2_4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + str(TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA2TC48c.met")), + ], +} + +FORMAT_TO_METADATA_FILES_LTV = { + "ISM1": [str(LTV_DIR.joinpath("ltvISM1.csv"))], + "ISM2": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + ], + "ISM3": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + ], + "ISM4": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltvISM4.csv")), + ], + "NDP_ISM4": [ # Should not be needed, because it is included in all ISM metadata files. + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltvISM4.csv")), + ], + "MASA1": [str(LTV_DIR.joinpath("ltv48_MASA1TC.met"))], + "MASA2": [str(LTV_DIR.joinpath("ltv48_MASA2TC.met"))], + "OMASA_1_1": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_1ISM_1TC.met")), + ], + "OMASA_1_2": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met")), + ], + "OMASA_1_3": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_3ISM_1TC.met")), + ], + "OMASA_1_4": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltvISM4.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_4ISM_1TC.met")), + ], + "OMASA_2_1": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_1ISM_2TC.met")), + ], + "OMASA_2_2": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_2ISM_2TC.met")), + ], + "OMASA_2_3": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_3ISM_2TC.met")), + ], + "OMASA_2_4": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltvISM4.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_4ISM_2TC.met")), + ], +} + +""" Input formats """ +INPUT_FORMATS_AMBI = ["FOA", "HOA2", "HOA3"] +INPUT_FORMATS_MC = ["MONO", "STEREO", "5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"] +INPUT_FORMATS_ISM = ["ISM1", "ISM2", "ISM3", "ISM4"] +INPUT_FORMATS_MASA = ["MASA1", "MASA2"] + +""" Non binaural / parametric output formats """ +OUTPUT_FORMATS = [ + "MONO", + "STEREO", + "5_1", + "5_1_2", + "5_1_4", + "7_1", + "7_1_4", + "FOA", + "HOA2", + "HOA3", +] + +""" Custom loudspeaker input/output """ +CUSTOM_LS_TO_TEST = [ + "t_design_4", + "4d4", + "16ch_8+4+4", +] + +""" Mixed scene ( metadata ) rendering """ +METADATA_SCENES_TO_TEST = ["mixed_scene", "mixed_scene_simple"] +METADATA_SCENES_TO_TEST_NO_BE = ["masa_scene"] +METADATA_SCENES_TO_TEST_MASA_PREREND = ["mixed_mc714_foa_masa2_ism4"] + +""" Binaural rendering """ +OUTPUT_FORMATS_BINAURAL = ["BINAURAL", "BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"] +HR_TRAJECTORIES_TO_TEST = [ + "full_circle_in_15s", + # "rotate_yaw_pitch_roll1", +] + +""" Frame Size """ +FRAMING_TO_TEST = ["5ms", "20ms"] + +PEAQ_SUPPORTED_FMT = [ + "MONO", + "STEREO", + "BINAURAL", + "BINAURAL_ROOM_IR", + "BINAURAL_ROOM_REVERB", +] + diff --git a/tests/renderer_short/cut/.gitignore b/tests/renderer_short/cut/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..f935021a8f8a7bd22f9d6703cafa5134bb6a57f8 --- /dev/null +++ b/tests/renderer_short/cut/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/tests/renderer_short/ref/.gitignore b/tests/renderer_short/ref/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..f935021a8f8a7bd22f9d6703cafa5134bb6a57f8 --- /dev/null +++ b/tests/renderer_short/ref/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/tests/renderer_short/test_renderer.py b/tests/renderer_short/test_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..8cfda2f6dbbed56e3c31f39b6ffd818408bc7e4d --- /dev/null +++ b/tests/renderer_short/test_renderer.py @@ -0,0 +1,1285 @@ +#!/usr/bin/env python3 + +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest + +from .constants import ( + FORMAT_TO_METADATA_FILES_LTV, + OUTPUT_FORMATS, + INPUT_FORMATS_AMBI, + FRAMING_TO_TEST, + EXE_SUFFIX, + OUTPUT_FORMATS_BINAURAL, + HR_TRAJECTORIES_TO_TEST, + HR_TRAJECTORY_DIR, + INPUT_FORMATS_MC, + INPUT_FORMATS_ISM, + INPUT_FORMATS_MASA, + METADATA_SCENES_TO_TEST_MASA_PREREND, + TEST_VECTOR_DIR, + CUSTOM_LS_TO_TEST, + CUSTOM_LAYOUT_DIR, + METADATA_SCENES_TO_TEST, +) +from .utils import run_renderer, compare_renderer_args +from ..conftest import props_to_record + +############################################################################## +# Bit-exactness tests +# +# These tests are run in REF and CUT mode and the outputs are compared for +# bit-exactness +############################################################################## +""" Ambisonics """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ambisonics( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ambisonics_binaural_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ambisonics_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.skip(reason="Not supported for BASOP code currently") +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +@pytest.mark.parametrize("aeid", ["1", "0"]) +def test_dynamic_acoustic_environment( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + aeid, + split_comparison, +): + rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") + rend_config_path.with_stem(f"rend_config") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + config_file=rend_config_path, + aeid=aeid, + split_comparison=split_comparison, + ) + + +@pytest.mark.skip("MSAN errors in BASOP need to be fixed") +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_dynamic_acoustic_environment_file( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") + rend_config_path.with_stem(f"rend_config") + + aeid = TEST_VECTOR_DIR.joinpath(f"aeid1.txt") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + config_file=rend_config_path, + aeid=aeid, + split_comparison=split_comparison, + ) + + +""" Multichannel """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_multichannel( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_multichannel_binaural_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_multichannel_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" ISM """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ism( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ism_binaural_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ism_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" MASA """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_masa( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_masa_binaural_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: + pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_masa_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: + pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) +def test_masa_prerend( + record_property, + props_to_record, + test_info, + in_fmt, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + "META", + "MASA2", + metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" Custom loudspeaker layouts """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_custom_ls_input( + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize( + "in_fmt", + [*INPUT_FORMATS_AMBI, *INPUT_FORMATS_MC, *INPUT_FORMATS_ISM, *INPUT_FORMATS_MASA], +) +def test_custom_ls_output( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + # TODO: revert once BASOP is brought up-to-date + if in_fmt in INPUT_FORMATS_MASA: + pytest.skip("MASA to custom LS not supported on ivas-float-update yet") + if in_fmt in INPUT_FORMATS_ISM and out_fmt == "t_design_4": + pytest.skip("ISMx + t_design_4 skipped until bug in BASOP is fixed") + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) +def test_custom_ls_input_output( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), + CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_custom_ls_input_binaural( + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_custom_ls_input_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" Metadata / scene description input """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_metadata( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + "META", + out_fmt, + metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" non diegetic pan """ + + +@pytest.mark.parametrize("out_fmt", ["STEREO"]) +@pytest.mark.parametrize("in_fmt", ["MONO"]) +@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) +def test_non_diegetic_pan_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan=non_diegetic_pan, + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", ["STEREO"]) +@pytest.mark.parametrize("in_fmt", ["ISM1"]) +@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) +def test_non_diegetic_pan_ism_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan=non_diegetic_pan, + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +############################################################################## +# Smoke tests +# +# These tests are run only for the smoke test and do not perform a +# bit-exactness comparison between REF and CUT, but between CUT1 and CUT2 +############################################################################## + + +# Test compares rendering with just a trajectory file against rendering with a trajectory file + a zero ref rotation. +# These should be binary equivalent. +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refrotzero( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refrotzero", + "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "refrot_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# Second test compares rendering with no head rotation against rendering with equal ref and head rotation. +# These should also be binary equivalent. +# Note that reference rotation is supplied per 4 subframes; head rotation per subframe. +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refrotequal( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refrotequal", + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "azi_plus_2-ele_plus_2-every-100-frames.csv" + ), + "refrot_file": HR_TRAJECTORY_DIR.joinpath( + "azi_plus_2-ele_plus_2-every-25-rows.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: head rotation trajectory file (OTR=NONE) +# cut: identical head rotation trajectory file as ref but in addition a constant +# reference vector in the looking direction of the coordinate system (OTR=REF_VEC) +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refveczero( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refveczero", + "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath("const000-Vector3.csv"), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: no head rotation (OTR=NONE) +# cut: rendering with head rotation and a ref vector which moves in the +# looking-direction of the head rotation and therefore compensates it (OTR=REF_VEC) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refvecequal( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + # TODO revert + if in_fmt == "HOA3" and out_fmt == "BINAURAL_ROOM_REVERB": + pytest.xfail("WIP : minor differences to be resolved") + else: + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvecequal", + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s.csv" + ), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a head rotation trajectory with elevation (OTR=NONE) +# cut: a static head rotation and a reference position trajectory which moves +# in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refvec_rotating( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + # TODO revert + if in_fmt == "HOA2" and out_fmt == "BINAURAL_ROOM_REVERB": + pytest.xfail("WIP : minor differences to be resolved") + else: + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvec_rotating", + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-ccw-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a head rotation trajectory with elevation (OTR=NONE) +# cut: a static head rotation and a reference position trajectory which moves +# in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) +# which also contains a fixed position offset between listener and reference position (which +# gets compensated in the REF_VEV OTR modes) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvec_rotating", + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-ccw.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-fixed-pos-offset-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a reference position trajectory with elevation and REF_VEC_LEV OTR mode (OTR=REF_VEC_LEV) +# cut: a reference position trajectory without the elevation and REF_VEC OTR mode (OTR=REF_VEC) +# Since the only difference between REF_VEC_LEV and REF_VEC is that *LEV ignores +# the height difference in positions, the output must be binary equivalent. +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refveclevel", + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refveclev_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-Vector3.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath("full-circle-4s-Vector3.csv"), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a head rotation trajectory with elevation (OTR=NONE) +# cut: a static head rotation and a reference position trajectory which moves +# in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +def test_multichannel_binaural_headrotation_refvec_rotating( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvec_rotating", + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-ccw-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a head rotation trajectory with elevation (OTR=NONE) +# cut: a static head rotation and a reference position trajectory which moves +# in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +def test_ism_binaural_headrotation_refvec_rotating( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvec_rotating", + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-ccw-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) diff --git a/tests/renderer_short/utils.py b/tests/renderer_short/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9afda68b940d90511cf4cb75485f0f526495de3f --- /dev/null +++ b/tests/renderer_short/utils.py @@ -0,0 +1,786 @@ +#!/usr/bin/env python3 + +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import filecmp +import logging +import os +from pathlib import Path +import subprocess as sp +import sys +from typing import Dict, Optional, Union + +import numpy as np +import pytest +import re +import errno +import tempfile + +from .compare_audio import compare_audio_arrays +from .constants import ( + LTV_DIR, + SCRIPTS_DIR, + OUTPUT_PATH_REF, + OUTPUT_PATH_CUT, + FORMAT_TO_FILE_COMPARETEST, + FORMAT_TO_FILE_LTV, + FORMAT_TO_METADATA_FILES, + FORMAT_TO_METADATA_FILES_LTV, + FORMAT_TO_FILE_SMOKETEST, + RENDERER_CMD, + BIN_SUFFIX_MERGETARGET, + PEAQ_SUPPORTED_FMT, +) +from ..constants import CAT_NORMAL + +sys.path.append(SCRIPTS_DIR) +from pyaudio3dtools.audiofile import readfile +from ..cmp_pcm import cmp_pcm +from ..conftest import parse_properties, get_split_idx + + +def _run_cmd(cmd, env, test_info=None): + """ + Helper function for running some command. + Raises a SystemError if either the return code is non-zero or a USAN printout is detected + """ + proc = sp.run(cmd, capture_output=True, text=True, env=env) + stdout = proc.stdout + proc.stderr + + # check for USAN error first + if "UndefinedBehaviorSanitizer" in stdout: + error = f"USAN error detected in stdout of command: {' '.join(cmd)}\n{stdout}" + if test_info is not None: + test_info.error = error + raise SystemError(error) + + # then handle possible crash + try: + proc.check_returncode() + except sp.CalledProcessError as e: + error = f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" + if test_info is not None: + test_info.error = error + raise SystemError(error) + + +def run_cmd(cmd, test_info, env=None): + logging.info(f"\nRunning command\n{' '.join(cmd)}\n") + _run_cmd(cmd, env, test_info) + + +def run_isar_ext_rend_cmd(cmd, env=None): + logging.info(f"\nRunning ISAR EXT REND command\n{' '.join(cmd)}\n") + _run_cmd(cmd, env) + + +def run_ivas_isar_enc_cmd(cmd, env=None): + logging.info(f"\nRunning IVAS ISAR encoder command\n{' '.join(cmd)}\n") + _run_cmd(cmd, env) + + +def run_ivas_isar_dec_cmd(cmd, env=None): + logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") + _run_cmd(cmd, env) + + +def run_isar_post_rend_cmd(cmd, env=None): + logging.info(f"\nRunning ISAR post renderer command\n{' '.join(cmd)}\n") + _run_cmd(cmd, env) + + +def check_BE( + test_info, + ref: np.ndarray, + ref_fs: int, + cut: np.ndarray, + cut_fs: int, + atol: int = 2, +) -> tuple: + if ref is None or np.array_equal(ref, np.zeros_like(ref)): + pytest.fail("REF signal does not exist or is zero!") + + if cut is None or np.array_equal(cut, np.zeros_like(cut)): + pytest.fail("CuT signal does not exist or is zero!") + + snr, gain_b, max_diff = compare_audio_arrays(ref, ref_fs, cut, cut_fs) + + if np.isnan(snr) or gain_b == 0: + pytest.fail("Invalid comparison result, check your signals!") + + if ref.shape[0] < cut.shape[0]: + ref = np.pad(ref, [(0, cut.shape[0] - ref.shape[0]), (0, 0)]) + elif ref.shape[0] > cut.shape[0]: + cut = np.pad(cut, [(0, ref.shape[0] - cut.shape[0]), (0, 0)]) + + # check max_diff as well, since compare_audio_arrays will try to adjust for small delay differences + diff_found = not np.allclose(ref, cut, rtol=0, atol=atol) and max_diff > atol + + return diff_found, snr, gain_b, max_diff + + +def run_renderer( + record_property, + props_to_record, + test_info, + in_fmt: str, + out_fmt: str, + metadata_input: Optional[str] = None, + in_meta_files: Optional[list] = None, + trj_file: Optional[str] = None, + non_diegetic_pan: Optional[str] = None, + name_extension: Optional[str] = None, + refrot_file: Optional[str] = None, + refvec_file: Optional[str] = None, + refveclev_file: Optional[str] = None, + config_file: Optional[str] = None, + binary_suffix: str = "", + frame_size: Optional[str] = "20ms", + hrtf_file: Optional[str] = None, + get_mld=False, + mld_lim=0, + get_mld_lim=0, + abs_tol=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, + aeid: Optional[Union[Path, int]] = None, + in_file=None, + out_file=None, + sr=48, + render_for_peaq=False, + split_comparison=False, +) -> str: + # prepare arguments and filepaths + if trj_file is not None: + trj_name = f"_{trj_file.stem}" + else: + trj_name = "" + + if refrot_file is not None: + refrot_name = f"_{refrot_file.stem}" + else: + refrot_name = "" + + if refvec_file is not None: + refvec_name = f"_{refvec_file.stem}" + else: + refvec_name = "" + + if refveclev_file is not None: + refveclev_name = f"_{refveclev_file.stem}" + else: + refveclev_name = "" + + if config_file is not None: + config_name = f"_{config_file.stem}" + else: + config_name = "" + + if frame_size: + framing_name = f"_{frame_size}" + else: + framing_name = "" + + if aeid is not None: + if isinstance(aeid, Path): + aeid_name = f"_{aeid.stem}" + else: + aeid_name = aeid + else: + aeid_name = "" + + if not isinstance(out_fmt, str): + out_name = f"{out_fmt.stem}" + else: + out_name = out_fmt + + if hrtf_file is not None: + hrtf_file_name = f"_{hrtf_file.stem}" + else: + hrtf_file_name = "" + + if test_info.config.option.create_ref: + output_path_base = OUTPUT_PATH_REF + else: + output_path_base = OUTPUT_PATH_CUT + + # if in REF or CUT creation mode use the comparetestv + if test_info.config.option.create_ref or test_info.config.option.create_cut: + format_to_file = FORMAT_TO_FILE_COMPARETEST + else: + format_to_file = FORMAT_TO_FILE_SMOKETEST + + format_to_metadata_files = FORMAT_TO_METADATA_FILES + + if test_info.config.option.use_ltv: + if test_info.config.option.ltv_dir: + format_to_file = dict() + format_to_metadata_files = dict() + for k, v in FORMAT_TO_FILE_LTV.items(): + format_to_file[k] = str(v).replace( + str(LTV_DIR), str(test_info.config.option.ltv_dir) + ) + for k, v in FORMAT_TO_METADATA_FILES_LTV.items(): + format_to_file[k] = str(v).replace( + str(LTV_DIR), str(test_info.config.option.ltv_dir) + ) + else: + format_to_file = FORMAT_TO_FILE_LTV + format_to_metadata_files = FORMAT_TO_METADATA_FILES_LTV + + if in_file is None: + if metadata_input is not None: + in_file = metadata_input + in_name = metadata_input.stem + elif not isinstance(in_fmt, str): + in_file = format_to_file[in_fmt.stem] + in_name = in_fmt.stem + else: + in_file = format_to_file[in_fmt] + in_name = in_fmt + + if in_meta_files is None and in_fmt in format_to_metadata_files: + in_meta_files = format_to_metadata_files[in_fmt] + + if out_file is None: + out_file_stem = f"{in_name}_to_{out_name}{trj_name}{non_diegetic_pan}{refrot_name}{refvec_name}{refveclev_name}{config_name}{framing_name}{hrtf_file_name}{name_extension}{aeid_name}.wav" + out_file = str(output_path_base.joinpath(out_file_stem)) + + cmd = RENDERER_CMD[:] + cmd[2] = str(in_file) + cmd[4] = str(in_fmt) + cmd[6] = str(out_file) + cmd[8] = str(out_fmt) + cmd[10] = str(sr) + + if test_info.config.option.create_ref: + cmd[0] += BIN_SUFFIX_MERGETARGET + cmd[0] += binary_suffix + + if in_meta_files is not None: + cmd.extend(["-im", *in_meta_files]) + + if trj_file is not None: + cmd.extend(["-T", str(trj_file)]) + + if hrtf_file is not None: + cmd.extend(["-hrtf", str(hrtf_file)]) + + if non_diegetic_pan is not None: + cmd.extend(["-non_diegetic_pan", str(non_diegetic_pan)]) + if refrot_file is not None: + cmd.extend(["-rf", str(refrot_file)]) + cmd.extend(["-otr", "ref"]) + + if refvec_file is not None: + cmd.extend(["-rvf", str(refvec_file)]) + cmd.extend(["-otr", "ref_vec"]) + + if refveclev_file is not None: + cmd.extend(["-rvf", str(refveclev_file)]) + cmd.extend(["-otr", "ref_vec_lev"]) + + if config_file is not None: + cmd.extend(["-render_config", str(config_file)]) + + if frame_size: + cmd.extend(["-fr", str(frame_size.replace("ms", ""))]) + + if aeid is not None: + cmd.extend(["-aeid", str(aeid)]) + + # Set env variables for UBSAN + env = os.environ.copy() + if test_info.node.name and "UBSAN_OPTIONS" in env.keys(): + env["UBSAN_OPTIONS"] = ( + env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_info.node.name}" + ) + + testcase_props = { + "format": "Renderer", + "category": CAT_NORMAL, + } + + if record_property is not None: + for k, v in testcase_props.items(): + record_property(k, v) + + # run the renderer + run_cmd(cmd, test_info, env) + + if test_info.config.option.create_cut and not render_for_peaq: + # CUT creation mode will run a comparison with REF + out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) + + # Check if we need to render to mono, stereo or binaural for PEAQ comparison + odg_input = None + odg_test = None + odg_ref = None + if get_odg_bin: + odg_input = out_file_ref[0:-4] + ".INPUT.BINAURAL.wav" + odg_test = str(out_file)[0:-4] + ".BINAURAL.wav" + odg_ref = out_file_ref[0:-4] + ".BINAURAL.wav" + + if out_fmt not in PEAQ_SUPPORTED_FMT: + if in_fmt in PEAQ_SUPPORTED_FMT: + new_fmt = in_fmt # MONO or STEREO + else: + # If input is META which contains stereo, new_fmt needs to be STEREO. + if in_fmt == "META": + with open(in_file, "r") as scene: + if "STEREO" in scene.read(): + new_fmt = "STEREO" + else: + new_fmt = "BINAURAL" + else: + new_fmt = "BINAURAL" + + # Render test to PEAQ supported format (MONO, STEREO or BINAURAL) + cmd2 = RENDERER_CMD[:] + cmd2[2] = str(out_file) # in_file + cmd2[4] = str(out_fmt) # in_fmt + cmd2[6] = odg_test # out_file + cmd2[8] = new_fmt # out_fmt + cmd2[10] = str(sr) + cmd2[0] += BIN_SUFFIX_MERGETARGET # Use IVAS_rend_ref for re-rendering + cmd2[0] += binary_suffix + if "MASA" in str(out_fmt): + cmd2.extend(["-im", out_file + ".met"]) + run_cmd(cmd2, test_info, env) + + # Render ref to BINAURAL with same settings as test + cmd2[2] = str(out_file_ref) # in_file + cmd2[6] = odg_ref # out_file + run_cmd(cmd2, test_info, env) + out_fmt_bin = new_fmt + else: + out_fmt_bin = out_fmt + odg_test = out_file + odg_ref = out_file_ref + + if out_fmt_bin != in_fmt: + # Render input to match out_fmt_bin using same config as input, but with IVAS_rend_ref + cmd[0] += BIN_SUFFIX_MERGETARGET + cmd[0] += binary_suffix + cmd[6] = odg_input # out_file + cmd[8] = out_fmt_bin # out_fmt + run_cmd(cmd, test_info, env) + else: + odg_input = in_file + + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison + ref_fs = int(cmd[10]) * 1000 + output_differs_parts, reason_parts = cmp_pcm( + out_file_ref, + out_file, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + + # 2. run comparison on split files if --split_comparison is given + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases + if split_comparison: + split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) + + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: + output_differs_splits, reason_splits = cmp_pcm( + out_file_ref, + out_file, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + if split_comparison: + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + for k, v in result_props.items(): + record_property(k, v) + + if output_differs_parts[0]: + logging.error(f"Command line was: {' '.join(cmd)}") + pytest.fail(f"Output differs: ({reason_parts[0]})") + + # compare metadata files in case of MASA prerendering + if "MASA" in str(out_fmt): + meta_file_ref = out_file_ref + ".met" + meta_file_cut = out_file + ".met" + if not filecmp.cmp(meta_file_cut, meta_file_ref): + pytest.fail("Metadata file differs from reference") + + return out_file + + +def compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs: Dict, + cut_kwargs: Dict, + split_comparison=False, +): + out_file_ref = run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + **ref_kwargs, + split_comparison=split_comparison, + ) + ref, ref_fs = readfile(out_file_ref) + out_file_cut = run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + **cut_kwargs, + split_comparison=split_comparison, + ) + cut, cut_fs = readfile(out_file_cut) + [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) + if diff_found: + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + + +def binauralize_input_and_output( + record_property, + props_to_record, + test_info, + input_file, + dut_output_file, + ref_output_file, + in_fmt, + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, +): + # Use current folder as location for temporary directory, since scene description does not handle spaces in path + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: + tmp_dir = Path(tmp_dir) + scene_dut = str(tmp_dir.joinpath("scene_dut.txt")) + scene_ref = str(tmp_dir.joinpath("scene_ref.txt")) + scene_in = str(tmp_dir.joinpath("scene_in.txt")) + + # File names for binauralized signals, if needed + ref_input_file_binaural = ref_output_file[0:-4] + ".INPUT.BINAURAL.wav" + dut_output_file_binaural = dut_output_file[0:-4] + ".BINAURAL.wav" + ref_output_file_binaural = ref_output_file[0:-4] + ".BINAURAL.wav" + + # Identify metadata + in_meta_files = [ + str(SCRIPTS_DIR.joinpath(m)) if m != "NULL" else m + for m in re.findall(r"\b\S+\.csv|NULL\b", enc_opts) + ] # All .csv or NULL files in enc_opts are ISM metadata files. + n_obj = len(in_meta_files) + + # If extended metadata is not used, strip the metadata for the external renderer + extended_md_used = ( + re.search(r"-ism\s?\+[1-4]", enc_opts) + and not "OMASA" in in_fmt + and not "OSBA" in in_fmt + ) + if not extended_md_used and n_obj > 0: + truncated_meta_files = [] + for md in in_meta_files: + if md != "NULL": + md_out_file = str(tmp_dir.joinpath(os.path.basename(md))) + with open(md_out_file, "w") as fp_out, open(md, "r") as fp_in: + for line in fp_in: + fp_out.write( + ",".join(line.split(",")[:2]) + "\n" + ) # Keep only first two elements: azim, elev + else: + md_out_file = "NULL" # Cannot truncate NULL, just insert it without modification + truncated_meta_files.append(md_out_file) + in_meta_files = truncated_meta_files + + in_meta_files = in_meta_files + [ + str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r"\b\S+\.met\b", enc_opts) + ] # All .met files in enc_opts are MASA metadata files. + out_meta_files = None + if output_config == "EXT": + out_meta_files = [] + if n_obj > 0: + out_meta_files = out_meta_files + [ + f"{dut_output_file}.{i}.csv" for i in range(0, n_obj) + ] + if "MASA" in in_fmt: + out_meta_files = out_meta_files + [f"{dut_output_file}.met"] + if output_config == "EXT": + output_config = in_fmt + if output_config == "": + output_config = "MONO" # EVS mono + metadata_input = None + + if "OSBA" in in_fmt or "OMASA" in in_fmt: + scene_description_file(in_fmt, scene_in, n_obj, input_file, in_meta_files) + input_file = scene_in + in_meta_files = None + in_fmt = "META" + + if "OSBA" in output_config or "OMASA" in output_config: + if "OSBA" in output_config: + output_config = ( + output_config[:-1] + "3" + ) # Temporary fix to handle than IVAS_dec produces HOA3 for all OSBA configs. Needs to be removed when this fix is ported to BASOP. + scene_description_file( + output_config, scene_dut, n_obj, dut_output_file, out_meta_files + ) + dut_output_file = scene_dut + scene_description_file( + output_config, scene_ref, n_obj, ref_output_file, out_meta_files + ) + ref_output_file = scene_ref + out_meta_files = None + output_config = "META" + + # Identify headtracking and orientation trajectories + trj_file = findstr(r"-t\s+(\S+)", dec_opts) + non_diegetic_pan = findstr(r"-non_diegetic_pan\s+(\S+)", dec_opts) + if non_diegetic_pan is not None: + output_config = "STEREO" + name_extension = None + refrot_file = findstr(r"-rf\s+(\S+)", dec_opts) + rot_tmp_file = findstr(r"-rvf\s+(\S+)", dec_opts) + refveclev_file = None + refvec_file = None + if "-otr ref_vec_lev".upper() in dec_opts.upper(): + refveclev_file = rot_tmp_file + else: + if "-otr ref_vec".upper() in dec_opts.upper(): + refvec_file = rot_tmp_file + + # Rendering configuration + config_file = findstr(r"-render_config\s+(\S+)", dec_opts) + binary_suffix = "_ref" + frame_size = findstr(r"-fr\s+(\S+)", dec_opts) + # hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) + hrtf_file = None # Default HRTFs used for binaural rendering of output + + aeid = findstr(r"-aeid\s+(\S+)", dec_opts) + + if not output_config.upper() in PEAQ_SUPPORTED_FMT: + # Render output to BINAURAL + output_reformat = "BINAURAL" + + check_and_makedir(str(Path(dut_output_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + output_config, + output_reformat, + metadata_input, + out_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=dut_output_file, + out_file=dut_output_file_binaural, + sr=out_sr, + render_for_peaq=True, + ) + + check_and_makedir(str(Path(ref_output_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + output_config, + output_reformat, + metadata_input, + out_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=ref_output_file, + out_file=ref_output_file_binaural, + sr=out_sr, + render_for_peaq=True, + ) + + # Update output_config to rendered format + output_config = output_reformat + else: + # Signal already mono, stereo or binaural + dut_output_file_binaural = dut_output_file + ref_output_file_binaural = ref_output_file + + if in_fmt.upper() != output_config.upper(): + # Render input to match output_config + out_fmt = output_config + + check_and_makedir(str(Path(ref_input_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + metadata_input, + in_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=input_file, + out_file=ref_input_file_binaural, + sr=in_sr, + render_for_peaq=True, + ) + else: + ref_input_file_binaural = input_file + return ( + ref_input_file_binaural, + dut_output_file_binaural, + ref_output_file_binaural, + ) + + +def findstr(exp, s, one_element=True): + result = [SCRIPTS_DIR.joinpath(x) for x in re.findall(exp, s)] + if len(result) == 0: + return None + if one_element: + return result[0] + return result + + +def check_and_makedir(dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise # raises the error again + + +def scene_description_file(in_fmt, metadata_tmp, n_obj, input_file, in_meta_files): + with open(metadata_tmp, "w") as fp_meta: + currdir = Path( + metadata_tmp + ).parent # File names must be relative to config file location + fp_meta.write(f"{os.path.relpath(input_file, currdir)}\n") # Input file + fp_meta.write(f"{n_obj+1}\n") # Number of sources + for n in range(0, n_obj): + if in_meta_files[n] == "NULL": + md_file = "1\n1,0,0" # NULL metadata position: azim=0,elev=0 for 1 frame, looped throughout all frames. + else: + md_file = os.path.relpath(in_meta_files[n], currdir) + fp_meta.write(f"ISM\n{n+1}\n{md_file}\n") # ISM metadata + if "OSBA" in in_fmt: + fp_meta.write( + "gain_dB:-6\n" + ) # Set -6 dB on all components for OSBA to match IVAS_dec + fp_meta.write(f"{in_fmt.split('_')[0][1:]}\n") # SBA or MASA + fp_meta.write(f"{n_obj+1}\n") + fp_meta.write(f"{in_fmt.split('_')[-1]}\n") # SBA or MASA parameter + if "OMASA" in in_fmt: + fp_meta.write( + f"{os.path.relpath(in_meta_files[n_obj], currdir)}\n" + ) # MASA metadata + if "OSBA" in in_fmt: + fp_meta.write( + "gain_dB:-6\n" + ) # Set -6 dB on all components for OSBA to match IVAS_dec