From 57d71e79a745f8f7f13c80e47154fa9bbb545b15 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Sun, 19 Oct 2025 16:18:59 +0200 Subject: [PATCH 01/23] Fix for ivas-conformance-linux --- .gitlab-ci.yml | 10 ++----- scripts/parse_commands.py | 58 ++++++--------------------------------- 2 files changed, 12 insertions(+), 56 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef4908c45b..ce484f4950 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1385,10 +1385,7 @@ ivas-conformance-linux: # Reference creation - python3 scripts/prepare_combined_format_inputs.py - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files - - # Output creation - - python3 -m pytest -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1406,9 +1403,8 @@ ivas-conformance-linux: - cp -r scripts/trajectories testvec - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - cp -r tests/ref testvec/testv/ref - - cp -r tests/dut/* testvec/testv/ref - - cp -r tests/renderer/cut testvec/testv/renderer/ref - - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r tests/renderer/ref testvec/testv/renderer/ref + - cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 3a5fb6c1af..50419e9490 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -21,56 +21,16 @@ if __name__ == '__main__': here = Path(__file__).parent.resolve() - cmds_enc=[] - cmds_dec=[] - cmds_rend=[] - cmds_isar_post_rend=[] + with open(input,'r') as infile: + report = infile.read() + cmds_enc = re.findall(r'REF encoder command:\s*\n\s*(\S.*)', report, re.MULTILINE) + cmds_dec = re.findall(r'REF decoder command:\s*\n\s*(\S.*)', report, re.MULTILINE) + cmds_rend = re.findall(r'Running command\s*\n\s*(\S.*)', report, re.MULTILINE) + cmds_isar_post_rend = re.findall(r'Running ISAR post renderer command\s*\n\s*(\S.*)', report, re.MULTILINE) - - if path.isdir(input): - input = Path(input).rglob('*.html') - else: - input = [input] - for html_report in input: - - with open(html_report,'r') as infile: - for line in infile.readlines(): - cmds_enc.extend(re.findall(r"DUT encoder command:\\n\\t(.*?)\\n", line)) - cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line)) - cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line)) - cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line)) - - # If pytest-html < v4 is used, the parsing will fail and render empty lists. This is a work-around in case that happens. - if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]): - for html_report in input: - with open(html_report,'r') as infile: - enc_cmd = False - dec_cmd = False - rend_cmd = False - isar_post_rend_cmd = False - for line in infile.readlines(): - line = line.split("
")[0] # Remove trailing html tags - if enc_cmd: - cmds_enc.append(line) - enc_cmd = False - elif dec_cmd: - cmds_dec.append(line) - dec_cmd = False - elif rend_cmd: - cmds_rend.append(line) - rend_cmd = False - elif isar_post_rend_cmd: - cmds_isar_post_rend.append(line) - isar_post_rend_cmd = False - else: - if "DUT encoder command" in line: - enc_cmd = True - elif "DUT decoder command" in line: - dec_cmd = True - elif "Running command" in line: - rend_cmd = True - elif "Running ISAR post renderer command" in line: - isar_post_rend_cmd = True + # Remove HTML tags + cmds_rend = [x.split('<')[0] for x in cmds_rend] + cmds_isar_post_rend = [x.split('<')[0] for x in cmds_isar_post_rend] # Sort lists to keep deterministic order between runs cmds_enc.sort() -- GitLab From b6d649fd2effeda3d8ce4a1d72ff3f2b57f81d68 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Sun, 19 Oct 2025 18:04:11 +0200 Subject: [PATCH 02/23] Adapt script to use _ref suffix in command lines --- scripts/parse_commands.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 50419e9490..86b584c981 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -47,7 +47,7 @@ if __name__ == '__main__': # Adjust file arguments, pass other arguments as they are if path.exists(arg): arg = path.relpath(arg).replace('\\','/') - arg = re.sub('IVAS_cod(.exe)?', '$CUT_ENC_BIN', arg) + arg = re.sub('IVAS_cod_ref(.exe)?', '$CUT_ENC_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) arg = re.sub('tests', CUT_PATH, arg) args.append(arg) @@ -75,7 +75,7 @@ if __name__ == '__main__': # Adjust file arguments, pass other arguments as they are if path.exists(arg): arg = path.relpath(arg).replace('\\','/') - arg = re.sub('IVAS_dec(.exe)?', '$CUT_DEC_BIN', arg) + arg = re.sub('IVAS_dec_ref(.exe)?', '$CUT_DEC_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # For .fer cases the bitstream is in ref arg = re.sub('tests/split_rendering/renderer_configs', REF_PATH + r'/split_rendering/renderer_configs', arg) @@ -131,7 +131,7 @@ if __name__ == '__main__': # Adjust file arguments, pass other arguments as they are if path.exists(arg): arg = path.relpath(arg).replace('\\','/') - arg = re.sub('IVAS_rend(.exe)?', '$CUT_REND_BIN', arg) + arg = re.sub('IVAS_rend_ref(.exe)?', '$CUT_REND_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) arg = re.sub('tests/renderer/data', TESTV_PATH + r'renderer/data/', arg) arg = re.sub('tests', CUT_PATH, arg) @@ -156,7 +156,7 @@ if __name__ == '__main__': # Adjust file arguments, pass other arguments as they are if path.exists(arg): arg = path.relpath(arg).replace('\\','/') - arg = re.sub('ISAR_post_rend(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg) + arg = re.sub('ISAR_post_rend_ref(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) if re.search("^tests.*bit$",arg): arg = re.sub('tests/split_rendering/cut', REF_PATH + r'/split_rendering/ref', arg) -- GitLab From 833b502f85992c52e211f0fcc01ad1fd12b62e25 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 20 Oct 2025 09:22:29 +0200 Subject: [PATCH 03/23] Updates for parse_commands.py and split renderer logging of REF decoder commands --- scripts/parse_commands.py | 40 +++++++++++++++++++++++++++++++++++---- tests/renderer/utils.py | 5 ++++- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 86b584c981..88ac9ca4a7 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -2,6 +2,7 @@ import argparse import re +import json from os import path from pathlib import Path import glob @@ -28,6 +29,37 @@ if __name__ == '__main__': cmds_rend = re.findall(r'Running command\s*\n\s*(\S.*)', report, re.MULTILINE) cmds_isar_post_rend = re.findall(r'Running ISAR post renderer command\s*\n\s*(\S.*)', report, re.MULTILINE) + + if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]): + lines = re.split(r'\\n', report) + enc_cmd = False + dec_cmd = False + rend_cmd = False + isar_post_rend_cmd = False + for line in lines: + line = line.split("
")[0].replace('\\t','') # Remove trailing html tags + if enc_cmd: + cmds_enc.append(line) + enc_cmd = False + elif dec_cmd: + cmds_dec.append(line) + dec_cmd = False + elif rend_cmd: + cmds_rend.append(line) + rend_cmd = False + elif isar_post_rend_cmd: + cmds_isar_post_rend.append(line) + isar_post_rend_cmd = False + else: + if "REF encoder command" in line: + enc_cmd = True + elif "REF decoder command" in line: + dec_cmd = True + elif "Running command" in line: + rend_cmd = True + elif "Running ISAR post renderer command" in line: + isar_post_rend_cmd = True + # Remove HTML tags cmds_rend = [x.split('<')[0] for x in cmds_rend] cmds_isar_post_rend = [x.split('<')[0] for x in cmds_isar_post_rend] @@ -138,10 +170,10 @@ if __name__ == '__main__': args.append(arg) cmd = ' '.join(args) - if "cut" in cmd: + if "ref" in cmd: outfile.write(cmd+'\n') out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) - if out and "cut" in out.group(1): + if out and "ref" in out.group(1): outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/renderer/cut',REF_PATH + r'/renderer/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: @@ -164,10 +196,10 @@ if __name__ == '__main__': args.append(arg) cmd = ' '.join(args) - if "cut" in cmd: + if "ref" in cmd: outfile.write(cmd+'\n') out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) - if out and "cut" in out.group(1): + if out and "ref" in out.group(1): outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 2a0d50568c..9acb1b7f18 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -109,7 +109,10 @@ def run_ivas_isar_enc_cmd(cmd, env=None): def run_ivas_isar_dec_cmd(cmd, env=None): - logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") + if BIN_SUFFIX_MERGETARGET in cmd[0]: + logging.info(f"\nREF decoder command:\n\t{' '.join(cmd)}\n") + else: + logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") _run_cmd(cmd, env) -- GitLab From 47fb79ad007262860f3eb92f885b588a4dd3ea49 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 20 Oct 2025 11:32:28 +0200 Subject: [PATCH 04/23] Correct dut->ref cut->ref in headers for Readme* scripts --- scripts/dec_header.txt | 6 +++--- scripts/dec_isar_header.txt | 2 +- scripts/enc_header.txt | 6 +++--- scripts/isar_post_rend_header.txt | 2 +- scripts/jbm_header.txt | 6 +++--- scripts/rend_header.txt | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/dec_header.txt b/scripts/dec_header.txt index 599c7e73ed..68aff36355 100644 --- a/scripts/dec_header.txt +++ b/scripts/dec_header.txt @@ -16,7 +16,7 @@ LOG_FILE=Readme_IVAS_dec_log.txt rm -rf tmp rm -rf $CUT_PATH mkdir -p $CUT_PATH -mkdir -p $CUT_PATH/dut/masa_test/dec_output -mkdir -p $CUT_PATH/dut/param_file/dec -mkdir -p $CUT_PATH/dut/sba_bs/raw +mkdir -p $CUT_PATH/ref/masa_test/dec_output +mkdir -p $CUT_PATH/ref/param_file/dec +mkdir -p $CUT_PATH/ref/sba_bs/raw diff --git a/scripts/dec_isar_header.txt b/scripts/dec_isar_header.txt index 454383382f..0f449b21c2 100644 --- a/scripts/dec_isar_header.txt +++ b/scripts/dec_isar_header.txt @@ -16,6 +16,6 @@ LOG_FILE=Readme_IVAS_ISAR_dec_log.txt rm -rf tmp rm -rf $CUT_PATH mkdir -p $CUT_PATH -mkdir -p $CUT_PATH/split_rendering/cut +mkdir -p $CUT_PATH/split_rendering/ref diff --git a/scripts/enc_header.txt b/scripts/enc_header.txt index 18387a9326..76ebeee3e1 100644 --- a/scripts/enc_header.txt +++ b/scripts/enc_header.txt @@ -16,7 +16,7 @@ LOG_FILE=Readme_IVAS_enc_log.txt rm -rf tmp rm -rf $CUT_PATH mkdir -p $CUT_PATH -mkdir -p $CUT_PATH/dut/masa_test/bitstreams -mkdir -p $CUT_PATH/dut/param_file/enc -mkdir -p $CUT_PATH/dut/sba_bs/pkt +mkdir -p $CUT_PATH/ref/masa_test/bitstreams +mkdir -p $CUT_PATH/ref/param_file/enc +mkdir -p $CUT_PATH/ref/sba_bs/pkt diff --git a/scripts/isar_post_rend_header.txt b/scripts/isar_post_rend_header.txt index ed7eb20988..99a84b6c36 100644 --- a/scripts/isar_post_rend_header.txt +++ b/scripts/isar_post_rend_header.txt @@ -15,7 +15,7 @@ LOG_FILE=Readme_IVAS_isar_post_rend_log.txt rm -rf tmp rm -rf $CUT_PATH -mkdir -p $CUT_PATH/split_rendering/cut +mkdir -p $CUT_PATH/split_rendering/ref diff --git a/scripts/jbm_header.txt b/scripts/jbm_header.txt index d244c83b3f..57f59809f9 100644 --- a/scripts/jbm_header.txt +++ b/scripts/jbm_header.txt @@ -16,7 +16,7 @@ LOG_FILE=Readme_IVAS_jbm_log.txt rm -rf tmp rm -rf $CUT_PATH mkdir -p $CUT_PATH -mkdir -p $CUT_PATH/dut/masa_test/dec_output -mkdir -p $CUT_PATH/dut/param_file/dec -mkdir -p $CUT_PATH/dut/sba_bs/raw +mkdir -p $CUT_PATH/ref/masa_test/dec_output +mkdir -p $CUT_PATH/ref/param_file/dec +mkdir -p $CUT_PATH/ref/sba_bs/raw diff --git a/scripts/rend_header.txt b/scripts/rend_header.txt index c56eb17cb0..a83b0a17bc 100644 --- a/scripts/rend_header.txt +++ b/scripts/rend_header.txt @@ -15,7 +15,7 @@ LOG_FILE=Readme_IVAS_rend_log.txt rm -rf tmp rm -rf $CUT_PATH -mkdir -p $CUT_PATH/renderer/cut +mkdir -p $CUT_PATH/renderer/ref mkdir -p $CUT_PATH/renderer/data -- GitLab From 9fcd9b038c1d4ce498d9f49ff93ba675edc83d34 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 20 Oct 2025 13:20:01 +0200 Subject: [PATCH 05/23] Correct dut->ref cut->ref in parse_commands.py --- scripts/parse_commands.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 88ac9ca4a7..3f95d4281d 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -87,7 +87,7 @@ if __name__ == '__main__': outfile.write(cmd+'\n') bts = re.search(r"\s(([\S]+)(.bts|.192|.pkt|.fer))$", cmd) if bts: - outfile.write('$DIFF_BIN '+bts.group(1).replace(CUT_PATH + r'/dut',REF_PATH + r'/ref')+' '+bts.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('$DIFF_BIN '+bts.group(1).replace(CUT_PATH + r'/ref',REF_PATH + r'/ref')+' '+bts.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) @@ -112,11 +112,11 @@ if __name__ == '__main__': arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # For .fer cases the bitstream is in ref arg = re.sub('tests/split_rendering/renderer_configs', REF_PATH + r'/split_rendering/renderer_configs', arg) if re.search("^tests.*192$",arg): - arg = re.sub('tests/split_rendering/cut', REF_PATH + r'/split_rendering/ref', arg) + arg = re.sub('tests/split_rendering/ref', REF_PATH + r'/split_rendering/ref', arg) if re.search("\.wav$",arg): arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav else: - arg = re.sub('tests/dut', REF_PATH + r'/ref', arg) # Input argument + arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # Input argument if re.search("^tests.*bit$",arg): arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav args.append(arg) @@ -138,12 +138,12 @@ if __name__ == '__main__': for output in glob.glob(absolute_out.group(1) + '*'): output = path.relpath(output).replace('\\','/') output = re.sub('tests', CUT_PATH, output) - diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/dut',REF_PATH + r'/ref')+' '+output+' >> $LOG_FILE 2>&1') + diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/ref',REF_PATH + r'/ref')+' '+output+' >> $LOG_FILE 2>&1') outfile.write(('; ').join(diff_cmds)) - if isar_out and "cut" in isar_out.group(1): - outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1') - if isar_md_out and "cut" in isar_md_out.group(1): - outfile.write('; $DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n') + if isar_out and "ref" in isar_out.group(1): + outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH + r'/split_rendering/ref',REF_PATH + r'/split_rendering/ref')+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1') + if isar_md_out and "ref" in isar_md_out.group(1): + outfile.write('; $DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH + r'/split_rendering/ref',REF_PATH + r'/split_rendering/ref')+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n') else: outfile.write('\n') outfile.write('\n\n') @@ -174,7 +174,7 @@ if __name__ == '__main__': outfile.write(cmd+'\n') out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) if out and "ref" in out.group(1): - outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/renderer/cut',REF_PATH + r'/renderer/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/renderer/ref',REF_PATH + r'/renderer/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) @@ -191,7 +191,7 @@ if __name__ == '__main__': arg = re.sub('ISAR_post_rend_ref(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) if re.search("^tests.*bit$",arg): - arg = re.sub('tests/split_rendering/cut', REF_PATH + r'/split_rendering/ref', arg) + arg = re.sub('tests/split_rendering/ref', REF_PATH + r'/split_rendering/ref', arg) arg = re.sub('tests', CUT_PATH, arg) args.append(arg) cmd = ' '.join(args) @@ -200,7 +200,7 @@ if __name__ == '__main__': outfile.write(cmd+'\n') out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) if out and "ref" in out.group(1): - outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_rendering/ref',REF_PATH + r'/split_rendering/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) \ No newline at end of file -- GitLab From b711a74c3d7d6f16320d137ead67a56a311b2fc0 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 20 Oct 2025 13:59:52 +0200 Subject: [PATCH 06/23] Updates for decoder tests in parse_commands.py --- scripts/parse_commands.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 3f95d4281d..9592b587a4 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -103,22 +103,32 @@ if __name__ == '__main__': absolute_out = re.search(r"\s(([\S]+)(.wav))$", cmd) args = [] + input_set = False for arg in cmd.split(): # Adjust file arguments, pass other arguments as they are if path.exists(arg): arg = path.relpath(arg).replace('\\','/') arg = re.sub('IVAS_dec_ref(.exe)?', '$CUT_DEC_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) - arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # For .fer cases the bitstream is in ref - arg = re.sub('tests/split_rendering/renderer_configs', REF_PATH + r'/split_rendering/renderer_configs', arg) - if re.search("^tests.*192$",arg): - arg = re.sub('tests/split_rendering/ref', REF_PATH + r'/split_rendering/ref', arg) - if re.search("\.wav$",arg): - arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav - else: - arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # Input argument - if re.search("^tests.*bit$",arg): - arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav + if 'dectrace' in arg: + arg = re.sub('tests', CUT_PATH, arg) # dectrace output files occur before the input file + if 'tests' in arg: + if not input_set: + arg = re.sub('tests', REF_PATH, arg) # First occurence of tests/* is the input file + input_set = True + else: + arg = re.sub('tests', CUT_PATH, arg) # Remaining occurences of tests/* are output files + + #arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # For .fer cases the bitstream is in ref + # arg = re.sub('tests/split_rendering/renderer_configs', REF_PATH + r'/split_rendering/renderer_configs', arg) + # if re.search("^tests.*192$",arg): + # arg = re.sub('tests/split_rendering/ref', REF_PATH + r'/split_rendering/ref', arg) + # if re.search("\.wav$",arg): + # arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav + # else: + # arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # Input argument + # if re.search("^tests.*bit$",arg): + # arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav args.append(arg) cmd = ' '.join(args) -- GitLab From c9503d7532c51bd11c1958f8ad5ebc474d6cfc9d Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 20 Oct 2025 14:49:21 +0200 Subject: [PATCH 07/23] Fix for ISAR dec cases --- scripts/parse_commands.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 9592b587a4..5a6b27916c 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -29,7 +29,7 @@ if __name__ == '__main__': cmds_rend = re.findall(r'Running command\s*\n\s*(\S.*)', report, re.MULTILINE) cmds_isar_post_rend = re.findall(r'Running ISAR post renderer command\s*\n\s*(\S.*)', report, re.MULTILINE) - + # Depending on pytest-html version, the data may be in a json-blob. If so, the code below should handle this format. if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]): lines = re.split(r'\\n', report) enc_cmd = False @@ -110,6 +110,9 @@ if __name__ == '__main__': arg = path.relpath(arg).replace('\\','/') arg = re.sub('IVAS_dec_ref(.exe)?', '$CUT_DEC_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) + # Identify input files + if '.txt' in arg: + arg = re.sub('tests', REF_PATH, arg) # Decoder configuration file -- input file if 'dectrace' in arg: arg = re.sub('tests', CUT_PATH, arg) # dectrace output files occur before the input file if 'tests' in arg: -- GitLab From 59df04d914ea4a5e85ae99924bd09880797b9565 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 20 Oct 2025 15:37:54 +0200 Subject: [PATCH 08/23] Fixes for identifying input and output files in parse_commands.py --- scripts/parse_commands.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 5a6b27916c..d891f75caf 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -112,26 +112,16 @@ if __name__ == '__main__': arg = re.sub('scripts', TESTV_PATH, arg) # Identify input files if '.txt' in arg: - arg = re.sub('tests', REF_PATH, arg) # Decoder configuration file -- input file - if 'dectrace' in arg: - arg = re.sub('tests', CUT_PATH, arg) # dectrace output files occur before the input file + arg = re.sub('tests', REF_PATH, arg) + # Identify special cases of output files: dectrace, spltmd.bit + if 'spltmd.bit' in arg or 'dectrace' in arg: + arg = re.sub('tests', CUT_PATH, arg) if 'tests' in arg: if not input_set: arg = re.sub('tests', REF_PATH, arg) # First occurence of tests/* is the input file input_set = True else: arg = re.sub('tests', CUT_PATH, arg) # Remaining occurences of tests/* are output files - - #arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # For .fer cases the bitstream is in ref - # arg = re.sub('tests/split_rendering/renderer_configs', REF_PATH + r'/split_rendering/renderer_configs', arg) - # if re.search("^tests.*192$",arg): - # arg = re.sub('tests/split_rendering/ref', REF_PATH + r'/split_rendering/ref', arg) - # if re.search("\.wav$",arg): - # arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav - # else: - # arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # Input argument - # if re.search("^tests.*bit$",arg): - # arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav args.append(arg) cmd = ' '.join(args) -- GitLab From c44e3a6d6d0661bd3d0af9cf337ebfca76be9d43 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 08:24:20 +0200 Subject: [PATCH 09/23] Remove duplicate encoder tests. Revert to old renderer test --- .gitlab-ci.yml | 2 +- scripts/parse_commands.py | 8 + tests/renderer_short/README.md | 44 + tests/renderer_short/__init__.py | 31 + tests/renderer_short/compare_audio.py | 105 ++ tests/renderer_short/constants.py | 421 ++++++++ tests/renderer_short/test_renderer.py | 1285 +++++++++++++++++++++++++ tests/renderer_short/utils.py | 786 +++++++++++++++ 8 files changed, 2681 insertions(+), 1 deletion(-) create mode 100644 tests/renderer_short/README.md create mode 100644 tests/renderer_short/__init__.py create mode 100644 tests/renderer_short/compare_audio.py create mode 100644 tests/renderer_short/constants.py create mode 100644 tests/renderer_short/test_renderer.py create mode 100644 tests/renderer_short/utils.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ce484f4950..a266ba9af4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1384,7 +1384,7 @@ ivas-conformance-linux: # Reference creation - python3 scripts/prepare_combined_format_inputs.py - - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" + - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py" - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index d891f75caf..91c352d306 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -70,6 +70,14 @@ if __name__ == '__main__': cmds_rend.sort() cmds_isar_post_rend.sort() + # Remove duplicates from cmds_enc -- some decoder tests use same encoder options + i = 0 + while i + 1 < len(cmds_enc): + if " ".join(cmds_enc[i].split()[:-1]) == " ".join(cmds_enc[i+1].split()[:-1]): + del cmds_enc[i+1] + else: + i = i + 1 + with open(txt_file.replace('.','_enc.'),'w', newline='\n') as outfile: with open('scripts/enc_header.txt','r') as header: outfile.write(header.read()) diff --git a/tests/renderer_short/README.md b/tests/renderer_short/README.md new file mode 100644 index 0000000000..713821c5d3 --- /dev/null +++ b/tests/renderer_short/README.md @@ -0,0 +1,44 @@ +======================================== +THIS FOLDER WILL NOT BE PART OF DELIVERY +======================================== + +# External Renderer Tests + +See also the [contribution page](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/Contributions/2-external-renderer) for related presentations. + +### Run tests with: + +## Smoke test: + +```bash +python3 -m pytest -q -n auto tests/renderer/test_renderer.py +``` + +## Comparison test: + +```bash +python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref # requires IVAS_rend_ref in root! +python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut +``` + +### Important flags (see [pytest docs](https://docs.pytest.org/en/7.2.x/) for more information): + +- `-k` flag can filter test cases, e.g. `-k "test_ism_binaural_static"` +- `-rA` reports ALL (pass, xpass, xfail, fail) instead of the default behaviour of reporting only failed tests\ + this option will also report captured logs, **required for obtaining the commandline of testcases that pass or xfail** +- `--last-failed` re-runs only the cases that failed in the last test run +- `--collect-only` is useful when adding new testcases to check if argument parametrization is working correctly + +### Directory tree + +``` +. +├── compare_audio.py -> Python implementation of CompAudio, used for comparisons in tests +├── conftest.py -> Pytest configuration (enable commandline argument ingestion) +├── constants.py -> Important paths, formats, metadata files and commandline templates +├── cut -> Default location for output files for test conditions +├── data -> Input test vectors +├── ref -> Default location for output files for reference conditions +├── test_renderer.py -> Runs the renderer for all modes +└── utils.py -> Wrapper functions for executables for use in testcases +``` diff --git a/tests/renderer_short/__init__.py b/tests/renderer_short/__init__.py new file mode 100644 index 0000000000..2655594043 --- /dev/null +++ b/tests/renderer_short/__init__.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +""" + (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" diff --git a/tests/renderer_short/compare_audio.py b/tests/renderer_short/compare_audio.py new file mode 100644 index 0000000000..49bfa86c06 --- /dev/null +++ b/tests/renderer_short/compare_audio.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 + +""" + (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import sys +import warnings +from typing import Tuple + +import numpy as np + +from .constants import SCRIPTS_DIR + +sys.path.append(str(SCRIPTS_DIR)) +from pyaudio3dtools.audioarray import getdelay + + +def compare_audio_arrays( + left: np.ndarray, left_fs: int, right: np.ndarray, right_fs: int +) -> Tuple[float, float]: + if left_fs != right_fs: + return ValueError(f"Differing samplerates: {left_fs} vs {right_fs}!") + + if left.shape[1] != right.shape[1]: + cmp_ch = min(left.shape[1], right.shape[1]) + warnings.warn( + f"Differing number of channels: {left.shape[1]} vs {right.shape[1]}! Comparing first {cmp_ch} channel(s)", + category=RuntimeWarning, + ) + left = left[:, :cmp_ch] + right = right[:, :cmp_ch] + + if left.shape[0] != right.shape[0]: + cmp_smp = min(left.shape[0], right.shape[0]) + warnings.warn( + f"Warning - different durations: {left.shape[0] / left_fs:.2f}s vs {right.shape[0] / right_fs:.2f}s! Comparing first {cmp_smp / left_fs : .2f} sample(s)", + category=RuntimeWarning, + ) + left = left[:cmp_smp, :] + right = right[:cmp_smp, :] + + if not np.array_equal(left, right): + delay = getdelay(left, right) + delay_abs = np.abs(delay) + # getdelay can return large values if signals are quite different + # limit any delay compensation to 20 ms + if delay != 0 and (delay_abs < left_fs / 50): + warnings.warn( + f"File B is delayed by {delay} samples ({delay*1000 / left_fs : .2f}ms)!", + category=RuntimeWarning, + ) + + # shift array + left = np.roll(left, delay, axis=0) + + # zero shifted out samples + if delay < 0: + left[-np.abs(delay) :, :] = 0 + elif delay > 0: + left[: np.abs(delay), :] = 0 + """ + http://www-mmsp.ece.mcgill.ca/Documents/Software/Packages/AFsp/AFsp/CompAudio.html + """ + num = np.sum(left * right) + den = np.sqrt(np.sum(left**2) * np.sum(right**2)) + if den > 0: + r = num / den + else: + r = np.inf + snr = 10 * np.log10(1 / (1 - (r**2))) + gain_b = num / np.sum(right**2) + max_diff = np.abs(np.max(left - right)) + else: + snr = np.inf + gain_b = 1 + max_diff = 0 + + return snr, gain_b, max_diff diff --git a/tests/renderer_short/constants.py b/tests/renderer_short/constants.py new file mode 100644 index 0000000000..766e64722b --- /dev/null +++ b/tests/renderer_short/constants.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 + +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +from pathlib import Path +import platform + +""" Set up paths """ +TESTS_DIR = Path(__file__).parent +SCRIPTS_DIR = TESTS_DIR.parents[1].joinpath("scripts").resolve() +TEST_VECTOR_DIR = SCRIPTS_DIR.joinpath("testv") + +OUTPUT_PATH_REF = TESTS_DIR.joinpath("ref") +OUTPUT_PATH_CUT = TESTS_DIR.joinpath("cut") + +CUSTOM_LAYOUT_DIR = SCRIPTS_DIR.joinpath("ls_layouts") +HR_TRAJECTORY_DIR = SCRIPTS_DIR.joinpath("trajectories") +TESTV_DIR = SCRIPTS_DIR.joinpath("testv") +LTV_DIR = TESTV_DIR + +BIN_SUFFIX_MERGETARGET = "_ref" + +if platform.system() == "Windows": + EXE_SUFFIX = ".exe" +elif platform.system() in ["Linux", "Darwin"]: + EXE_SUFFIX = "" +else: + assert False, f"Unsupported platform {platform.system()}" + +""" Renderer commandline template """ +RENDERER_CMD = [ + str(TESTS_DIR.parent.parent.joinpath("IVAS_rend")), + "-i", + "", # 2 -> input file + "-if", + "", # 4 -> input format + "-o", + "/dev/null", # 6 -> output file + "-of", + "", # 8 -> output format + "-fs", + "48", # 10 -> input fs + "-no_delay_cmp", + # "-ndl", + "-q", +] + + +""" Format to file mappings """ +NCHAN_TO_FILE = { + 1: TEST_VECTOR_DIR.joinpath("spectral_test_1ch_48kHz.wav"), + 2: TEST_VECTOR_DIR.joinpath("spectral_test_2ch_48kHz.wav"), + 3: TEST_VECTOR_DIR.joinpath("spectral_test_3ch_48kHz.wav"), + 4: TEST_VECTOR_DIR.joinpath("spectral_test_4ch_48kHz.wav"), + 5: TEST_VECTOR_DIR.joinpath("spectral_test_5ch_48kHz.wav"), + 6: TEST_VECTOR_DIR.joinpath("spectral_test_6ch_48kHz.wav"), + 7: TEST_VECTOR_DIR.joinpath("spectral_test_7ch_48kHz.wav"), + 8: TEST_VECTOR_DIR.joinpath("spectral_test_8ch_48kHz.wav"), + 9: TEST_VECTOR_DIR.joinpath("spectral_test_9ch_48kHz.wav"), + 10: TEST_VECTOR_DIR.joinpath("spectral_test_10ch_48kHz.wav"), + 11: TEST_VECTOR_DIR.joinpath("spectral_test_11ch_48kHz.wav"), + 12: TEST_VECTOR_DIR.joinpath("spectral_test_12ch_48kHz.wav"), + 13: TEST_VECTOR_DIR.joinpath("spectral_test_13ch_48kHz.wav"), + 15: TEST_VECTOR_DIR.joinpath("spectral_test_15ch_48kHz.wav"), + 16: TEST_VECTOR_DIR.joinpath("spectral_test_16ch_48kHz.wav"), + 17: TEST_VECTOR_DIR.joinpath("spectral_test_17ch_48kHz.wav"), + 18: TEST_VECTOR_DIR.joinpath("spectral_test_18ch_48kHz.wav"), + 19: TEST_VECTOR_DIR.joinpath("spectral_test_19ch_48kHz.wav"), + 20: TEST_VECTOR_DIR.joinpath("spectral_test_20ch_48kHz.wav"), +} + +FORMAT_TO_FILE_SMOKETEST = { + "MONO": NCHAN_TO_FILE[1], + "STEREO": NCHAN_TO_FILE[2], + "5_1": NCHAN_TO_FILE[6], + "7_1": NCHAN_TO_FILE[8], + "5_1_2": NCHAN_TO_FILE[8], + "5_1_4": NCHAN_TO_FILE[10], + "7_1_4": NCHAN_TO_FILE[12], + "FOA": NCHAN_TO_FILE[4], + "HOA2": NCHAN_TO_FILE[9], + "HOA3": NCHAN_TO_FILE[16], + "ISM1": NCHAN_TO_FILE[1], + "ISM2": NCHAN_TO_FILE[2], + "ISM3": NCHAN_TO_FILE[3], + "ISM4": NCHAN_TO_FILE[4], + "NDP_ISM4": NCHAN_TO_FILE[4], + "MASA1": NCHAN_TO_FILE[1], + "MASA2": NCHAN_TO_FILE[2], + "OMASA_1_1": NCHAN_TO_FILE[2], + "OMASA_1_2": NCHAN_TO_FILE[3], + "OMASA_1_3": NCHAN_TO_FILE[4], + "OMASA_1_4": NCHAN_TO_FILE[5], + "OMASA_2_1": NCHAN_TO_FILE[3], + "OMASA_2_2": NCHAN_TO_FILE[4], + "OMASA_2_3": NCHAN_TO_FILE[5], + "OMASA_2_4": NCHAN_TO_FILE[6], + "OSBA_1_1": NCHAN_TO_FILE[5], + "OSBA_2_1": NCHAN_TO_FILE[6], + "OSBA_3_1": NCHAN_TO_FILE[7], + "OSBA_4_1": NCHAN_TO_FILE[8], + "OSBA_1_2": NCHAN_TO_FILE[10], + "OSBA_2_2": NCHAN_TO_FILE[11], + "OSBA_3_2": NCHAN_TO_FILE[12], + "OSBA_4_2": NCHAN_TO_FILE[13], + "OSBA_1_3": NCHAN_TO_FILE[17], + "OSBA_2_3": NCHAN_TO_FILE[18], + "OSBA_3_3": NCHAN_TO_FILE[19], + "OSBA_4_3": NCHAN_TO_FILE[20], + "META": TEST_VECTOR_DIR.joinpath("mixed_scene.txt"), + "16ch_8+4+4": NCHAN_TO_FILE[16], + "4d4": NCHAN_TO_FILE[8], + "t_design_4": NCHAN_TO_FILE[12], +} + +FORMAT_TO_FILE_COMPARETEST = { + "MONO": TESTV_DIR.joinpath("stv48c.wav"), + "STEREO": TESTV_DIR.joinpath("stvST48c.wav"), + "5_1": TESTV_DIR.joinpath("stv51MC48c.wav"), + "7_1": TESTV_DIR.joinpath("stv71MC48c.wav"), + "5_1_2": TESTV_DIR.joinpath("stv512MC48c.wav"), + "5_1_4": TESTV_DIR.joinpath("stv514MC48c.wav"), + "7_1_4": TESTV_DIR.joinpath("stv714MC48c.wav"), + "FOA": TESTV_DIR.joinpath("stvFOA48c.wav"), + "HOA2": TESTV_DIR.joinpath("stv2OA48c.wav"), + "HOA3": TESTV_DIR.joinpath("stv3OA48c.wav"), + "ISM1": TESTV_DIR.joinpath("stv1ISM48s.wav"), + "ISM2": TESTV_DIR.joinpath("stv2ISM48s.wav"), + "ISM3": TESTV_DIR.joinpath("stv3ISM48s.wav"), + "ISM4": TESTV_DIR.joinpath("stv4ISM48s.wav"), + "MASA1": TESTV_DIR.joinpath("stv1MASA1TC48c.wav"), + "MASA2": TESTV_DIR.joinpath("stv2MASA2TC48c.wav"), + "OMASA_1_1": TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA1TC48c.wav"), + "OMASA_1_2": TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA1TC48c.wav"), + "OMASA_1_3": TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA1TC48c.wav"), + "OMASA_1_4": TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA1TC48c.wav"), + "OMASA_2_1": TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA2TC48c.wav"), + "OMASA_2_2": TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA2TC48c.wav"), + "OMASA_2_3": TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA2TC48c.wav"), + "OMASA_2_4": TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA2TC48c.wav"), + "OSBA_1_1": TESTV_DIR.joinpath("stvOSBA_1ISM_FOA48c.wav"), + "OSBA_1_2": TESTV_DIR.joinpath("stvOSBA_1ISM_2OA48c.wav"), + "OSBA_1_3": TESTV_DIR.joinpath("stvOSBA_1ISM_3OA48c.wav"), + "OSBA_2_1": TESTV_DIR.joinpath("stvOSBA_2ISM_FOA48c.wav"), + "OSBA_2_2": TESTV_DIR.joinpath("stvOSBA_2ISM_2OA48c.wav"), + "OSBA_2_3": TESTV_DIR.joinpath("stvOSBA_2ISM_3OA48c.wav"), + "OSBA_3_1": TESTV_DIR.joinpath("stvOSBA_3ISM_FOA48c.wav"), + "OSBA_3_2": TESTV_DIR.joinpath("stvOSBA_3ISM_2OA48c.wav"), + "OSBA_3_3": TESTV_DIR.joinpath("stvOSBA_3ISM_3OA48c.wav"), + "OSBA_4_1": TESTV_DIR.joinpath("stvOSBA_4ISM_FOA48c.wav"), + "OSBA_4_2": TESTV_DIR.joinpath("stvOSBA_4ISM_2OA48c.wav"), + "OSBA_4_3": TESTV_DIR.joinpath("stvOSBA_4ISM_3OA48c.wav"), + "META": TEST_VECTOR_DIR.joinpath("mixed_scene.txt"), + "16ch_8+4+4": TESTV_DIR.joinpath("stv3OA48c.wav"), + "4d4": TESTV_DIR.joinpath("stv71MC48c.wav"), + "t_design_4": TESTV_DIR.joinpath("stv714MC48c.wav"), +} + +FORMAT_TO_FILE_LTV = { + "MONO": LTV_DIR.joinpath("ltv48_MONO.wav"), + "STEREO": LTV_DIR.joinpath("ltv48_STEREO.wav"), + "5_1": LTV_DIR.joinpath("ltv48_MC51.wav"), + "7_1": LTV_DIR.joinpath("ltv48_MC71.wav"), + "5_1_2": LTV_DIR.joinpath("ltv48_MC512.wav"), + "5_1_4": LTV_DIR.joinpath("ltv48_MC514.wav"), + "7_1_4": LTV_DIR.joinpath("ltv48_MC714.wav"), + "FOA": LTV_DIR.joinpath("ltv48_FOA.wav"), + "HOA2": LTV_DIR.joinpath("ltv48_HOA2.wav"), + "HOA3": LTV_DIR.joinpath("ltv48_HOA3.wav"), + "ISM1": LTV_DIR.joinpath("ltv48_1ISM.wav"), + "ISM2": LTV_DIR.joinpath("ltv48_2ISM.wav"), + "ISM3": LTV_DIR.joinpath("ltv48_3ISM.wav"), + "ISM4": LTV_DIR.joinpath("ltv48_4ISM.wav"), + "MASA1": LTV_DIR.joinpath("ltv48_MASA1TC.wav"), + "MASA2": LTV_DIR.joinpath("ltv48_MASA2TC.wav"), + "OMASA_1_1": LTV_DIR.joinpath("ltv48_OMASA_1ISM_1TC.wav"), + "OMASA_1_2": LTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.wav"), + "OMASA_1_3": LTV_DIR.joinpath("ltv48_OMASA_3ISM_1TC.wav"), + "OMASA_1_4": LTV_DIR.joinpath("ltv48_OMASA_4ISM_1TC.wav"), + "OMASA_2_1": LTV_DIR.joinpath("ltv48_OMASA_1ISM_2TC.wav"), + "OMASA_2_2": LTV_DIR.joinpath("ltv48_OMASA_2ISM_2TC.wav"), + "OMASA_2_3": LTV_DIR.joinpath("ltv48_OMASA_3ISM_2TC.wav"), + "OMASA_2_4": LTV_DIR.joinpath("ltv48_OMASA_4ISM_2TC.wav"), + "OSBA_1_1": LTV_DIR.joinpath("ltv48_OSBA_1ISM_FOA.wav"), + "OSBA_1_2": LTV_DIR.joinpath("ltv48_OSBA_1ISM_HOA2.wav"), + "OSBA_1_3": LTV_DIR.joinpath("ltv48_OSBA_1ISM_HOA3.wav"), + "OSBA_2_1": LTV_DIR.joinpath("ltv48_OSBA_2ISM_FOA.wav"), + "OSBA_2_2": LTV_DIR.joinpath("ltv48_OSBA_2ISM_HOA2.wav"), + "OSBA_2_3": LTV_DIR.joinpath("ltv48_OSBA_2ISM_HOA3.wav"), + "OSBA_3_1": LTV_DIR.joinpath("ltv48_OSBA_3ISM_FOA.wav"), + "OSBA_3_2": LTV_DIR.joinpath("ltv48_OSBA_3ISM_HOA2.wav"), + "OSBA_3_3": LTV_DIR.joinpath("ltv48_OSBA_3ISM_HOA3.wav"), + "OSBA_4_1": LTV_DIR.joinpath("ltv48_OSBA_4ISM_FOA.wav"), + "OSBA_4_2": LTV_DIR.joinpath("ltv48_OSBA_4ISM_HOA2.wav"), + "OSBA_4_3": LTV_DIR.joinpath("ltv48_OSBA_4ISM_HOA3.wav"), + "META": TEST_VECTOR_DIR.joinpath("mixed_scene.txt"), + "16ch_8+4+4": LTV_DIR.joinpath("ltv48_HOA3.wav"), + "4d4": LTV_DIR.joinpath("ltv48_MC71.wav"), + "t_design_4": LTV_DIR.joinpath("ltv48_MC714.wav"), +} + +FORMAT_TO_METADATA_FILES = { + "ISM1": [str(TESTV_DIR.joinpath("stvISM1.csv"))], + "ISM2": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + ], + "ISM3": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + ], + "ISM4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + ], + "NDP_ISM4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2_non-diegetic-pan.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + ], + "MASA1": [str(TESTV_DIR.joinpath("stv1MASA1TC48c.met"))], + "MASA2": [str(TESTV_DIR.joinpath("stv2MASA2TC48c.met"))], + "OMASA_1_1": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA1TC48c.met")), + ], + "OMASA_1_2": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA1TC48c.met")), + ], + "OMASA_1_3": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA1TC48c.met")), + ], + "OMASA_1_4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + str(TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA1TC48c.met")), + ], + "OMASA_2_1": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA2TC48c.met")), + ], + "OMASA_2_2": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA2TC48c.met")), + ], + "OMASA_2_3": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA2TC48c.met")), + ], + "OMASA_2_4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + str(TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA2TC48c.met")), + ], +} + +FORMAT_TO_METADATA_FILES_LTV = { + "ISM1": [str(LTV_DIR.joinpath("ltvISM1.csv"))], + "ISM2": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + ], + "ISM3": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + ], + "ISM4": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltvISM4.csv")), + ], + "NDP_ISM4": [ # Should not be needed, because it is included in all ISM metadata files. + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltvISM4.csv")), + ], + "MASA1": [str(LTV_DIR.joinpath("ltv48_MASA1TC.met"))], + "MASA2": [str(LTV_DIR.joinpath("ltv48_MASA2TC.met"))], + "OMASA_1_1": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_1ISM_1TC.met")), + ], + "OMASA_1_2": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met")), + ], + "OMASA_1_3": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_3ISM_1TC.met")), + ], + "OMASA_1_4": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltvISM4.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_4ISM_1TC.met")), + ], + "OMASA_2_1": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_1ISM_2TC.met")), + ], + "OMASA_2_2": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_2ISM_2TC.met")), + ], + "OMASA_2_3": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_3ISM_2TC.met")), + ], + "OMASA_2_4": [ + str(LTV_DIR.joinpath("ltvISM1.csv")), + str(LTV_DIR.joinpath("ltvISM2.csv")), + str(LTV_DIR.joinpath("ltvISM3.csv")), + str(LTV_DIR.joinpath("ltvISM4.csv")), + str(LTV_DIR.joinpath("ltv48_OMASA_4ISM_2TC.met")), + ], +} + +""" Input formats """ +INPUT_FORMATS_AMBI = ["FOA", "HOA2", "HOA3"] +INPUT_FORMATS_MC = ["MONO", "STEREO", "5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"] +INPUT_FORMATS_ISM = ["ISM1", "ISM2", "ISM3", "ISM4"] +INPUT_FORMATS_MASA = ["MASA1", "MASA2"] + +""" Non binaural / parametric output formats """ +OUTPUT_FORMATS = [ + "MONO", + "STEREO", + "5_1", + "5_1_2", + "5_1_4", + "7_1", + "7_1_4", + "FOA", + "HOA2", + "HOA3", +] + +""" Custom loudspeaker input/output """ +CUSTOM_LS_TO_TEST = [ + "t_design_4", + "4d4", + "16ch_8+4+4", +] + +""" Mixed scene ( metadata ) rendering """ +METADATA_SCENES_TO_TEST = ["mixed_scene", "mixed_scene_simple"] +METADATA_SCENES_TO_TEST_NO_BE = ["masa_scene"] +METADATA_SCENES_TO_TEST_MASA_PREREND = ["mixed_mc714_foa_masa2_ism4"] + +""" Binaural rendering """ +OUTPUT_FORMATS_BINAURAL = ["BINAURAL", "BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"] +HR_TRAJECTORIES_TO_TEST = [ + "full_circle_in_15s", + # "rotate_yaw_pitch_roll1", +] + +""" Frame Size """ +FRAMING_TO_TEST = ["5ms", "20ms"] + +PEAQ_SUPPORTED_FMT = [ + "MONO", + "STEREO", + "BINAURAL", + "BINAURAL_ROOM_IR", + "BINAURAL_ROOM_REVERB", +] + diff --git a/tests/renderer_short/test_renderer.py b/tests/renderer_short/test_renderer.py new file mode 100644 index 0000000000..8cfda2f6db --- /dev/null +++ b/tests/renderer_short/test_renderer.py @@ -0,0 +1,1285 @@ +#!/usr/bin/env python3 + +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pytest + +from .constants import ( + FORMAT_TO_METADATA_FILES_LTV, + OUTPUT_FORMATS, + INPUT_FORMATS_AMBI, + FRAMING_TO_TEST, + EXE_SUFFIX, + OUTPUT_FORMATS_BINAURAL, + HR_TRAJECTORIES_TO_TEST, + HR_TRAJECTORY_DIR, + INPUT_FORMATS_MC, + INPUT_FORMATS_ISM, + INPUT_FORMATS_MASA, + METADATA_SCENES_TO_TEST_MASA_PREREND, + TEST_VECTOR_DIR, + CUSTOM_LS_TO_TEST, + CUSTOM_LAYOUT_DIR, + METADATA_SCENES_TO_TEST, +) +from .utils import run_renderer, compare_renderer_args +from ..conftest import props_to_record + +############################################################################## +# Bit-exactness tests +# +# These tests are run in REF and CUT mode and the outputs are compared for +# bit-exactness +############################################################################## +""" Ambisonics """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ambisonics( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ambisonics_binaural_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ambisonics_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.skip(reason="Not supported for BASOP code currently") +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +@pytest.mark.parametrize("aeid", ["1", "0"]) +def test_dynamic_acoustic_environment( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + aeid, + split_comparison, +): + rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") + rend_config_path.with_stem(f"rend_config") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + config_file=rend_config_path, + aeid=aeid, + split_comparison=split_comparison, + ) + + +@pytest.mark.skip("MSAN errors in BASOP need to be fixed") +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_dynamic_acoustic_environment_file( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") + rend_config_path.with_stem(f"rend_config") + + aeid = TEST_VECTOR_DIR.joinpath(f"aeid1.txt") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + config_file=rend_config_path, + aeid=aeid, + split_comparison=split_comparison, + ) + + +""" Multichannel """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_multichannel( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_multichannel_binaural_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_multichannel_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" ISM """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ism( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ism_binaural_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_ism_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" MASA """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_masa( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_masa_binaural_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: + pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_masa_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: + pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) +def test_masa_prerend( + record_property, + props_to_record, + test_info, + in_fmt, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + "META", + "MASA2", + metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" Custom loudspeaker layouts """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_custom_ls_input( + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize( + "in_fmt", + [*INPUT_FORMATS_AMBI, *INPUT_FORMATS_MC, *INPUT_FORMATS_ISM, *INPUT_FORMATS_MASA], +) +def test_custom_ls_output( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + # TODO: revert once BASOP is brought up-to-date + if in_fmt in INPUT_FORMATS_MASA: + pytest.skip("MASA to custom LS not supported on ivas-float-update yet") + if in_fmt in INPUT_FORMATS_ISM and out_fmt == "t_design_4": + pytest.skip("ISMx + t_design_4 skipped until bug in BASOP is fixed") + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) +def test_custom_ls_input_output( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), + CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_custom_ls_input_binaural( + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_custom_ls_input_binaural_headrotation( + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + trj_file, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" Metadata / scene description input """ + + +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) +@pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) +def test_metadata( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + "META", + out_fmt, + metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + binary_suffix=EXE_SUFFIX, + frame_size=frame_size, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +""" non diegetic pan """ + + +@pytest.mark.parametrize("out_fmt", ["STEREO"]) +@pytest.mark.parametrize("in_fmt", ["MONO"]) +@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) +def test_non_diegetic_pan_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan=non_diegetic_pan, + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +@pytest.mark.parametrize("out_fmt", ["STEREO"]) +@pytest.mark.parametrize("in_fmt", ["ISM1"]) +@pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) +def test_non_diegetic_pan_ism_static( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan, + get_mld, + get_mld_lim, + get_ssnr, + get_odg, + get_odg_bin, + split_comparison, +): + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan=non_diegetic_pan, + binary_suffix=EXE_SUFFIX, + get_mld=get_mld, + mld_lim=get_mld_lim, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + split_comparison=split_comparison, + ) + + +############################################################################## +# Smoke tests +# +# These tests are run only for the smoke test and do not perform a +# bit-exactness comparison between REF and CUT, but between CUT1 and CUT2 +############################################################################## + + +# Test compares rendering with just a trajectory file against rendering with a trajectory file + a zero ref rotation. +# These should be binary equivalent. +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refrotzero( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refrotzero", + "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "refrot_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# Second test compares rendering with no head rotation against rendering with equal ref and head rotation. +# These should also be binary equivalent. +# Note that reference rotation is supplied per 4 subframes; head rotation per subframe. +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refrotequal( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refrotequal", + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "azi_plus_2-ele_plus_2-every-100-frames.csv" + ), + "refrot_file": HR_TRAJECTORY_DIR.joinpath( + "azi_plus_2-ele_plus_2-every-25-rows.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: head rotation trajectory file (OTR=NONE) +# cut: identical head rotation trajectory file as ref but in addition a constant +# reference vector in the looking direction of the coordinate system (OTR=REF_VEC) +@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refveczero( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refveczero", + "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath("const000-Vector3.csv"), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: no head rotation (OTR=NONE) +# cut: rendering with head rotation and a ref vector which moves in the +# looking-direction of the head rotation and therefore compensates it (OTR=REF_VEC) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refvecequal( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + # TODO revert + if in_fmt == "HOA3" and out_fmt == "BINAURAL_ROOM_REVERB": + pytest.xfail("WIP : minor differences to be resolved") + else: + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvecequal", + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s.csv" + ), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a head rotation trajectory with elevation (OTR=NONE) +# cut: a static head rotation and a reference position trajectory which moves +# in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refvec_rotating( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + # TODO revert + if in_fmt == "HOA2" and out_fmt == "BINAURAL_ROOM_REVERB": + pytest.xfail("WIP : minor differences to be resolved") + else: + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvec_rotating", + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-ccw-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a head rotation trajectory with elevation (OTR=NONE) +# cut: a static head rotation and a reference position trajectory which moves +# in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) +# which also contains a fixed position offset between listener and reference position (which +# gets compensated in the REF_VEV OTR modes) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvec_rotating", + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-ccw.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-fixed-pos-offset-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a reference position trajectory with elevation and REF_VEC_LEV OTR mode (OTR=REF_VEC_LEV) +# cut: a reference position trajectory without the elevation and REF_VEC OTR mode (OTR=REF_VEC) +# Since the only difference between REF_VEC_LEV and REF_VEC is that *LEV ignores +# the height difference in positions, the output must be binary equivalent. +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) +def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refveclevel", + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refveclev_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-Vector3.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath("full-circle-4s-Vector3.csv"), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a head rotation trajectory with elevation (OTR=NONE) +# cut: a static head rotation and a reference position trajectory which moves +# in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) +def test_multichannel_binaural_headrotation_refvec_rotating( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + if in_fmt in ["MONO", "STEREO"]: + pytest.skip("MONO or STEREO to Binaural rendering unsupported") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvec_rotating", + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-ccw-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) + + +# This test compares rendering with: +# ref: a head rotation trajectory with elevation (OTR=NONE) +# cut: a static head rotation and a reference position trajectory which moves +# in a way that produces the same acoustic output as the ref head rot trajectory (OTR=REF_VEC) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) +def test_ism_binaural_headrotation_refvec_rotating( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + split_comparison, +): + if test_info.config.option.create_ref or test_info.config.option.create_cut: + pytest.skip("OTR tests only run for smoke test") + + compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs={ + "name_extension": "refvec_rotating", + "trj_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s.csv" + ), + "frame_size": "5", + }, + cut_kwargs={ + "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "refvec_file": HR_TRAJECTORY_DIR.joinpath( + "full-circle-with-up-and-down-4s-ccw-Vector3.csv" + ), + "frame_size": "5", + }, + split_comparison=split_comparison, + ) diff --git a/tests/renderer_short/utils.py b/tests/renderer_short/utils.py new file mode 100644 index 0000000000..9afda68b94 --- /dev/null +++ b/tests/renderer_short/utils.py @@ -0,0 +1,786 @@ +#!/usr/bin/env python3 + +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import filecmp +import logging +import os +from pathlib import Path +import subprocess as sp +import sys +from typing import Dict, Optional, Union + +import numpy as np +import pytest +import re +import errno +import tempfile + +from .compare_audio import compare_audio_arrays +from .constants import ( + LTV_DIR, + SCRIPTS_DIR, + OUTPUT_PATH_REF, + OUTPUT_PATH_CUT, + FORMAT_TO_FILE_COMPARETEST, + FORMAT_TO_FILE_LTV, + FORMAT_TO_METADATA_FILES, + FORMAT_TO_METADATA_FILES_LTV, + FORMAT_TO_FILE_SMOKETEST, + RENDERER_CMD, + BIN_SUFFIX_MERGETARGET, + PEAQ_SUPPORTED_FMT, +) +from ..constants import CAT_NORMAL + +sys.path.append(SCRIPTS_DIR) +from pyaudio3dtools.audiofile import readfile +from ..cmp_pcm import cmp_pcm +from ..conftest import parse_properties, get_split_idx + + +def _run_cmd(cmd, env, test_info=None): + """ + Helper function for running some command. + Raises a SystemError if either the return code is non-zero or a USAN printout is detected + """ + proc = sp.run(cmd, capture_output=True, text=True, env=env) + stdout = proc.stdout + proc.stderr + + # check for USAN error first + if "UndefinedBehaviorSanitizer" in stdout: + error = f"USAN error detected in stdout of command: {' '.join(cmd)}\n{stdout}" + if test_info is not None: + test_info.error = error + raise SystemError(error) + + # then handle possible crash + try: + proc.check_returncode() + except sp.CalledProcessError as e: + error = f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" + if test_info is not None: + test_info.error = error + raise SystemError(error) + + +def run_cmd(cmd, test_info, env=None): + logging.info(f"\nRunning command\n{' '.join(cmd)}\n") + _run_cmd(cmd, env, test_info) + + +def run_isar_ext_rend_cmd(cmd, env=None): + logging.info(f"\nRunning ISAR EXT REND command\n{' '.join(cmd)}\n") + _run_cmd(cmd, env) + + +def run_ivas_isar_enc_cmd(cmd, env=None): + logging.info(f"\nRunning IVAS ISAR encoder command\n{' '.join(cmd)}\n") + _run_cmd(cmd, env) + + +def run_ivas_isar_dec_cmd(cmd, env=None): + logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") + _run_cmd(cmd, env) + + +def run_isar_post_rend_cmd(cmd, env=None): + logging.info(f"\nRunning ISAR post renderer command\n{' '.join(cmd)}\n") + _run_cmd(cmd, env) + + +def check_BE( + test_info, + ref: np.ndarray, + ref_fs: int, + cut: np.ndarray, + cut_fs: int, + atol: int = 2, +) -> tuple: + if ref is None or np.array_equal(ref, np.zeros_like(ref)): + pytest.fail("REF signal does not exist or is zero!") + + if cut is None or np.array_equal(cut, np.zeros_like(cut)): + pytest.fail("CuT signal does not exist or is zero!") + + snr, gain_b, max_diff = compare_audio_arrays(ref, ref_fs, cut, cut_fs) + + if np.isnan(snr) or gain_b == 0: + pytest.fail("Invalid comparison result, check your signals!") + + if ref.shape[0] < cut.shape[0]: + ref = np.pad(ref, [(0, cut.shape[0] - ref.shape[0]), (0, 0)]) + elif ref.shape[0] > cut.shape[0]: + cut = np.pad(cut, [(0, ref.shape[0] - cut.shape[0]), (0, 0)]) + + # check max_diff as well, since compare_audio_arrays will try to adjust for small delay differences + diff_found = not np.allclose(ref, cut, rtol=0, atol=atol) and max_diff > atol + + return diff_found, snr, gain_b, max_diff + + +def run_renderer( + record_property, + props_to_record, + test_info, + in_fmt: str, + out_fmt: str, + metadata_input: Optional[str] = None, + in_meta_files: Optional[list] = None, + trj_file: Optional[str] = None, + non_diegetic_pan: Optional[str] = None, + name_extension: Optional[str] = None, + refrot_file: Optional[str] = None, + refvec_file: Optional[str] = None, + refveclev_file: Optional[str] = None, + config_file: Optional[str] = None, + binary_suffix: str = "", + frame_size: Optional[str] = "20ms", + hrtf_file: Optional[str] = None, + get_mld=False, + mld_lim=0, + get_mld_lim=0, + abs_tol=0, + get_ssnr=False, + get_odg=False, + get_odg_bin=False, + aeid: Optional[Union[Path, int]] = None, + in_file=None, + out_file=None, + sr=48, + render_for_peaq=False, + split_comparison=False, +) -> str: + # prepare arguments and filepaths + if trj_file is not None: + trj_name = f"_{trj_file.stem}" + else: + trj_name = "" + + if refrot_file is not None: + refrot_name = f"_{refrot_file.stem}" + else: + refrot_name = "" + + if refvec_file is not None: + refvec_name = f"_{refvec_file.stem}" + else: + refvec_name = "" + + if refveclev_file is not None: + refveclev_name = f"_{refveclev_file.stem}" + else: + refveclev_name = "" + + if config_file is not None: + config_name = f"_{config_file.stem}" + else: + config_name = "" + + if frame_size: + framing_name = f"_{frame_size}" + else: + framing_name = "" + + if aeid is not None: + if isinstance(aeid, Path): + aeid_name = f"_{aeid.stem}" + else: + aeid_name = aeid + else: + aeid_name = "" + + if not isinstance(out_fmt, str): + out_name = f"{out_fmt.stem}" + else: + out_name = out_fmt + + if hrtf_file is not None: + hrtf_file_name = f"_{hrtf_file.stem}" + else: + hrtf_file_name = "" + + if test_info.config.option.create_ref: + output_path_base = OUTPUT_PATH_REF + else: + output_path_base = OUTPUT_PATH_CUT + + # if in REF or CUT creation mode use the comparetestv + if test_info.config.option.create_ref or test_info.config.option.create_cut: + format_to_file = FORMAT_TO_FILE_COMPARETEST + else: + format_to_file = FORMAT_TO_FILE_SMOKETEST + + format_to_metadata_files = FORMAT_TO_METADATA_FILES + + if test_info.config.option.use_ltv: + if test_info.config.option.ltv_dir: + format_to_file = dict() + format_to_metadata_files = dict() + for k, v in FORMAT_TO_FILE_LTV.items(): + format_to_file[k] = str(v).replace( + str(LTV_DIR), str(test_info.config.option.ltv_dir) + ) + for k, v in FORMAT_TO_METADATA_FILES_LTV.items(): + format_to_file[k] = str(v).replace( + str(LTV_DIR), str(test_info.config.option.ltv_dir) + ) + else: + format_to_file = FORMAT_TO_FILE_LTV + format_to_metadata_files = FORMAT_TO_METADATA_FILES_LTV + + if in_file is None: + if metadata_input is not None: + in_file = metadata_input + in_name = metadata_input.stem + elif not isinstance(in_fmt, str): + in_file = format_to_file[in_fmt.stem] + in_name = in_fmt.stem + else: + in_file = format_to_file[in_fmt] + in_name = in_fmt + + if in_meta_files is None and in_fmt in format_to_metadata_files: + in_meta_files = format_to_metadata_files[in_fmt] + + if out_file is None: + out_file_stem = f"{in_name}_to_{out_name}{trj_name}{non_diegetic_pan}{refrot_name}{refvec_name}{refveclev_name}{config_name}{framing_name}{hrtf_file_name}{name_extension}{aeid_name}.wav" + out_file = str(output_path_base.joinpath(out_file_stem)) + + cmd = RENDERER_CMD[:] + cmd[2] = str(in_file) + cmd[4] = str(in_fmt) + cmd[6] = str(out_file) + cmd[8] = str(out_fmt) + cmd[10] = str(sr) + + if test_info.config.option.create_ref: + cmd[0] += BIN_SUFFIX_MERGETARGET + cmd[0] += binary_suffix + + if in_meta_files is not None: + cmd.extend(["-im", *in_meta_files]) + + if trj_file is not None: + cmd.extend(["-T", str(trj_file)]) + + if hrtf_file is not None: + cmd.extend(["-hrtf", str(hrtf_file)]) + + if non_diegetic_pan is not None: + cmd.extend(["-non_diegetic_pan", str(non_diegetic_pan)]) + if refrot_file is not None: + cmd.extend(["-rf", str(refrot_file)]) + cmd.extend(["-otr", "ref"]) + + if refvec_file is not None: + cmd.extend(["-rvf", str(refvec_file)]) + cmd.extend(["-otr", "ref_vec"]) + + if refveclev_file is not None: + cmd.extend(["-rvf", str(refveclev_file)]) + cmd.extend(["-otr", "ref_vec_lev"]) + + if config_file is not None: + cmd.extend(["-render_config", str(config_file)]) + + if frame_size: + cmd.extend(["-fr", str(frame_size.replace("ms", ""))]) + + if aeid is not None: + cmd.extend(["-aeid", str(aeid)]) + + # Set env variables for UBSAN + env = os.environ.copy() + if test_info.node.name and "UBSAN_OPTIONS" in env.keys(): + env["UBSAN_OPTIONS"] = ( + env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_info.node.name}" + ) + + testcase_props = { + "format": "Renderer", + "category": CAT_NORMAL, + } + + if record_property is not None: + for k, v in testcase_props.items(): + record_property(k, v) + + # run the renderer + run_cmd(cmd, test_info, env) + + if test_info.config.option.create_cut and not render_for_peaq: + # CUT creation mode will run a comparison with REF + out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) + + # Check if we need to render to mono, stereo or binaural for PEAQ comparison + odg_input = None + odg_test = None + odg_ref = None + if get_odg_bin: + odg_input = out_file_ref[0:-4] + ".INPUT.BINAURAL.wav" + odg_test = str(out_file)[0:-4] + ".BINAURAL.wav" + odg_ref = out_file_ref[0:-4] + ".BINAURAL.wav" + + if out_fmt not in PEAQ_SUPPORTED_FMT: + if in_fmt in PEAQ_SUPPORTED_FMT: + new_fmt = in_fmt # MONO or STEREO + else: + # If input is META which contains stereo, new_fmt needs to be STEREO. + if in_fmt == "META": + with open(in_file, "r") as scene: + if "STEREO" in scene.read(): + new_fmt = "STEREO" + else: + new_fmt = "BINAURAL" + else: + new_fmt = "BINAURAL" + + # Render test to PEAQ supported format (MONO, STEREO or BINAURAL) + cmd2 = RENDERER_CMD[:] + cmd2[2] = str(out_file) # in_file + cmd2[4] = str(out_fmt) # in_fmt + cmd2[6] = odg_test # out_file + cmd2[8] = new_fmt # out_fmt + cmd2[10] = str(sr) + cmd2[0] += BIN_SUFFIX_MERGETARGET # Use IVAS_rend_ref for re-rendering + cmd2[0] += binary_suffix + if "MASA" in str(out_fmt): + cmd2.extend(["-im", out_file + ".met"]) + run_cmd(cmd2, test_info, env) + + # Render ref to BINAURAL with same settings as test + cmd2[2] = str(out_file_ref) # in_file + cmd2[6] = odg_ref # out_file + run_cmd(cmd2, test_info, env) + out_fmt_bin = new_fmt + else: + out_fmt_bin = out_fmt + odg_test = out_file + odg_ref = out_file_ref + + if out_fmt_bin != in_fmt: + # Render input to match out_fmt_bin using same config as input, but with IVAS_rend_ref + cmd[0] += BIN_SUFFIX_MERGETARGET + cmd[0] += binary_suffix + cmd[6] = odg_input # out_file + cmd[8] = out_fmt_bin # out_fmt + run_cmd(cmd, test_info, env) + else: + odg_input = in_file + + ### run the comparison tools + split_idx = np.empty(0) + prop_suffix = [""] + + # 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison + ref_fs = int(cmd[10]) * 1000 + output_differs_parts, reason_parts = cmp_pcm( + out_file_ref, + out_file, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + + # 2. run comparison on split files if --split_comparison is given + # for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles + # -> skip split comparison for these cases + if split_comparison: + split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000) + + # this extra if takes care of cases where no splits are found, e.g. the "NOOP" case in the self_test_ltv prm file + # if this would not be there, then the comparison of the whole file would run twice + if len(split_idx) > 0: + output_differs_splits, reason_splits = cmp_pcm( + out_file_ref, + out_file, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + get_odg=get_odg, + get_odg_bin=get_odg_bin, + odg_input=odg_input, + odg_test=odg_test, + odg_ref=odg_ref, + scalefac=test_info.config.option.scalefac, + split_idx=split_idx, + ) + output_differs_parts += output_differs_splits + reason_parts += reason_splits + + if split_comparison: + prop_suffix = ["_whole"] + [ + f"_split{i:03d}" for i in range(1, len(split_idx) + 1) + ] + + for output_differs, reason, suffix in zip( + output_differs_parts, reason_parts, prop_suffix + ): + result_props = parse_properties( + reason, output_differs, props_to_record, suffix + ) + for k, v in result_props.items(): + record_property(k, v) + + if output_differs_parts[0]: + logging.error(f"Command line was: {' '.join(cmd)}") + pytest.fail(f"Output differs: ({reason_parts[0]})") + + # compare metadata files in case of MASA prerendering + if "MASA" in str(out_fmt): + meta_file_ref = out_file_ref + ".met" + meta_file_cut = out_file + ".met" + if not filecmp.cmp(meta_file_cut, meta_file_ref): + pytest.fail("Metadata file differs from reference") + + return out_file + + +def compare_renderer_args( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs: Dict, + cut_kwargs: Dict, + split_comparison=False, +): + out_file_ref = run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + **ref_kwargs, + split_comparison=split_comparison, + ) + ref, ref_fs = readfile(out_file_ref) + out_file_cut = run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + **cut_kwargs, + split_comparison=split_comparison, + ) + cut, cut_fs = readfile(out_file_cut) + [diff_found, snr, gain_b, max_diff] = check_BE(test_info, ref, ref_fs, cut, cut_fs) + if diff_found: + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) + + +def binauralize_input_and_output( + record_property, + props_to_record, + test_info, + input_file, + dut_output_file, + ref_output_file, + in_fmt, + output_config, + enc_opts, + dec_opts, + in_sr, + out_sr, +): + # Use current folder as location for temporary directory, since scene description does not handle spaces in path + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: + tmp_dir = Path(tmp_dir) + scene_dut = str(tmp_dir.joinpath("scene_dut.txt")) + scene_ref = str(tmp_dir.joinpath("scene_ref.txt")) + scene_in = str(tmp_dir.joinpath("scene_in.txt")) + + # File names for binauralized signals, if needed + ref_input_file_binaural = ref_output_file[0:-4] + ".INPUT.BINAURAL.wav" + dut_output_file_binaural = dut_output_file[0:-4] + ".BINAURAL.wav" + ref_output_file_binaural = ref_output_file[0:-4] + ".BINAURAL.wav" + + # Identify metadata + in_meta_files = [ + str(SCRIPTS_DIR.joinpath(m)) if m != "NULL" else m + for m in re.findall(r"\b\S+\.csv|NULL\b", enc_opts) + ] # All .csv or NULL files in enc_opts are ISM metadata files. + n_obj = len(in_meta_files) + + # If extended metadata is not used, strip the metadata for the external renderer + extended_md_used = ( + re.search(r"-ism\s?\+[1-4]", enc_opts) + and not "OMASA" in in_fmt + and not "OSBA" in in_fmt + ) + if not extended_md_used and n_obj > 0: + truncated_meta_files = [] + for md in in_meta_files: + if md != "NULL": + md_out_file = str(tmp_dir.joinpath(os.path.basename(md))) + with open(md_out_file, "w") as fp_out, open(md, "r") as fp_in: + for line in fp_in: + fp_out.write( + ",".join(line.split(",")[:2]) + "\n" + ) # Keep only first two elements: azim, elev + else: + md_out_file = "NULL" # Cannot truncate NULL, just insert it without modification + truncated_meta_files.append(md_out_file) + in_meta_files = truncated_meta_files + + in_meta_files = in_meta_files + [ + str(SCRIPTS_DIR.joinpath(m)) for m in re.findall(r"\b\S+\.met\b", enc_opts) + ] # All .met files in enc_opts are MASA metadata files. + out_meta_files = None + if output_config == "EXT": + out_meta_files = [] + if n_obj > 0: + out_meta_files = out_meta_files + [ + f"{dut_output_file}.{i}.csv" for i in range(0, n_obj) + ] + if "MASA" in in_fmt: + out_meta_files = out_meta_files + [f"{dut_output_file}.met"] + if output_config == "EXT": + output_config = in_fmt + if output_config == "": + output_config = "MONO" # EVS mono + metadata_input = None + + if "OSBA" in in_fmt or "OMASA" in in_fmt: + scene_description_file(in_fmt, scene_in, n_obj, input_file, in_meta_files) + input_file = scene_in + in_meta_files = None + in_fmt = "META" + + if "OSBA" in output_config or "OMASA" in output_config: + if "OSBA" in output_config: + output_config = ( + output_config[:-1] + "3" + ) # Temporary fix to handle than IVAS_dec produces HOA3 for all OSBA configs. Needs to be removed when this fix is ported to BASOP. + scene_description_file( + output_config, scene_dut, n_obj, dut_output_file, out_meta_files + ) + dut_output_file = scene_dut + scene_description_file( + output_config, scene_ref, n_obj, ref_output_file, out_meta_files + ) + ref_output_file = scene_ref + out_meta_files = None + output_config = "META" + + # Identify headtracking and orientation trajectories + trj_file = findstr(r"-t\s+(\S+)", dec_opts) + non_diegetic_pan = findstr(r"-non_diegetic_pan\s+(\S+)", dec_opts) + if non_diegetic_pan is not None: + output_config = "STEREO" + name_extension = None + refrot_file = findstr(r"-rf\s+(\S+)", dec_opts) + rot_tmp_file = findstr(r"-rvf\s+(\S+)", dec_opts) + refveclev_file = None + refvec_file = None + if "-otr ref_vec_lev".upper() in dec_opts.upper(): + refveclev_file = rot_tmp_file + else: + if "-otr ref_vec".upper() in dec_opts.upper(): + refvec_file = rot_tmp_file + + # Rendering configuration + config_file = findstr(r"-render_config\s+(\S+)", dec_opts) + binary_suffix = "_ref" + frame_size = findstr(r"-fr\s+(\S+)", dec_opts) + # hrtf_file = findstr(r'-hrtf\s+(\S+)', dec_opts) + hrtf_file = None # Default HRTFs used for binaural rendering of output + + aeid = findstr(r"-aeid\s+(\S+)", dec_opts) + + if not output_config.upper() in PEAQ_SUPPORTED_FMT: + # Render output to BINAURAL + output_reformat = "BINAURAL" + + check_and_makedir(str(Path(dut_output_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + output_config, + output_reformat, + metadata_input, + out_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=dut_output_file, + out_file=dut_output_file_binaural, + sr=out_sr, + render_for_peaq=True, + ) + + check_and_makedir(str(Path(ref_output_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + output_config, + output_reformat, + metadata_input, + out_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=ref_output_file, + out_file=ref_output_file_binaural, + sr=out_sr, + render_for_peaq=True, + ) + + # Update output_config to rendered format + output_config = output_reformat + else: + # Signal already mono, stereo or binaural + dut_output_file_binaural = dut_output_file + ref_output_file_binaural = ref_output_file + + if in_fmt.upper() != output_config.upper(): + # Render input to match output_config + out_fmt = output_config + + check_and_makedir(str(Path(ref_input_file_binaural).parent)) + + run_renderer( + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + metadata_input, + in_meta_files, + trj_file, + non_diegetic_pan, + name_extension, + refrot_file, + refvec_file, + refveclev_file, + config_file, + binary_suffix, + frame_size, + hrtf_file, + aeid, + in_file=input_file, + out_file=ref_input_file_binaural, + sr=in_sr, + render_for_peaq=True, + ) + else: + ref_input_file_binaural = input_file + return ( + ref_input_file_binaural, + dut_output_file_binaural, + ref_output_file_binaural, + ) + + +def findstr(exp, s, one_element=True): + result = [SCRIPTS_DIR.joinpath(x) for x in re.findall(exp, s)] + if len(result) == 0: + return None + if one_element: + return result[0] + return result + + +def check_and_makedir(dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise # raises the error again + + +def scene_description_file(in_fmt, metadata_tmp, n_obj, input_file, in_meta_files): + with open(metadata_tmp, "w") as fp_meta: + currdir = Path( + metadata_tmp + ).parent # File names must be relative to config file location + fp_meta.write(f"{os.path.relpath(input_file, currdir)}\n") # Input file + fp_meta.write(f"{n_obj+1}\n") # Number of sources + for n in range(0, n_obj): + if in_meta_files[n] == "NULL": + md_file = "1\n1,0,0" # NULL metadata position: azim=0,elev=0 for 1 frame, looped throughout all frames. + else: + md_file = os.path.relpath(in_meta_files[n], currdir) + fp_meta.write(f"ISM\n{n+1}\n{md_file}\n") # ISM metadata + if "OSBA" in in_fmt: + fp_meta.write( + "gain_dB:-6\n" + ) # Set -6 dB on all components for OSBA to match IVAS_dec + fp_meta.write(f"{in_fmt.split('_')[0][1:]}\n") # SBA or MASA + fp_meta.write(f"{n_obj+1}\n") + fp_meta.write(f"{in_fmt.split('_')[-1]}\n") # SBA or MASA parameter + if "OMASA" in in_fmt: + fp_meta.write( + f"{os.path.relpath(in_meta_files[n_obj], currdir)}\n" + ) # MASA metadata + if "OSBA" in in_fmt: + fp_meta.write( + "gain_dB:-6\n" + ) # Set -6 dB on all components for OSBA to match IVAS_dec -- GitLab From b70b69c5463974f8ed35adc7c1fd5a3393c79197 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 08:33:22 +0200 Subject: [PATCH 10/23] Add cut/ref folders in renderer_short --- tests/renderer_short/cut/.gitignore | 1 + tests/renderer_short/ref/.gitignore | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/renderer_short/cut/.gitignore create mode 100644 tests/renderer_short/ref/.gitignore diff --git a/tests/renderer_short/cut/.gitignore b/tests/renderer_short/cut/.gitignore new file mode 100644 index 0000000000..f935021a8f --- /dev/null +++ b/tests/renderer_short/cut/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/tests/renderer_short/ref/.gitignore b/tests/renderer_short/ref/.gitignore new file mode 100644 index 0000000000..f935021a8f --- /dev/null +++ b/tests/renderer_short/ref/.gitignore @@ -0,0 +1 @@ +!.gitignore -- GitLab From a06c2bda548d3d4da02ae1967546594163a9f641 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 08:46:15 +0200 Subject: [PATCH 11/23] Add back old mixed_scene_simple.txt for renderer_short --- scripts/testv/mixed_scene_simple.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 scripts/testv/mixed_scene_simple.txt diff --git a/scripts/testv/mixed_scene_simple.txt b/scripts/testv/mixed_scene_simple.txt new file mode 100644 index 0000000000..85d38b8e93 --- /dev/null +++ b/scripts/testv/mixed_scene_simple.txt @@ -0,0 +1,12 @@ +spectral_test_4ch_48kHz.wav +3 +ISM +1 +ism_0a_0e.csv +ISM +2 +1 +1,-30,0 +MC +3 +STEREO -- GitLab From ffbcbc51c6d855c04d203da7a946cc234639ee70 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 08:52:58 +0200 Subject: [PATCH 12/23] Add back old mixed_scene.txt for renderer_short --- scripts/testv/mixed_scene.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 scripts/testv/mixed_scene.txt diff --git a/scripts/testv/mixed_scene.txt b/scripts/testv/mixed_scene.txt new file mode 100644 index 0000000000..2501c75383 --- /dev/null +++ b/scripts/testv/mixed_scene.txt @@ -0,0 +1,15 @@ +spectral_test_16ch_48kHz.wav +4 +ISM +1 +ism_0a_0e.csv +ISM +2 +1 +1,-30,0 +SBA +3 +1 +MC +7 +5_1_4 -- GitLab From c8ba774452b009241e78e25f600b4cea09bde5ef Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 08:56:42 +0200 Subject: [PATCH 13/23] Fixes for ivas-conformance (Windows version) --- .gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a266ba9af4..5044cfb6ab 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1284,7 +1284,7 @@ ivas-conformance: # Reference creation - python scripts/prepare_combined_format_inputs.py - - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" + - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer_short/test_renderer.py", "tests/split_rendering/test_split_rendering.py" - python -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files # Output creation @@ -1312,9 +1312,8 @@ ivas-conformance: - cp -r -force -ErrorAction Ignore scripts/trajectories testvec - cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref - - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r -force -ErrorAction Ignore tests/renderer/ref testvec/testv/renderer/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/ref testvec/testv/split_rendering/ref - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns -- GitLab From f52cf2eb35270bd809b973e4d224c63469997e7d Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 09:08:42 +0200 Subject: [PATCH 14/23] Cleanup of parse_commands.py --- scripts/parse_commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 91c352d306..c2cc6a4188 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -95,7 +95,7 @@ if __name__ == '__main__': outfile.write(cmd+'\n') bts = re.search(r"\s(([\S]+)(.bts|.192|.pkt|.fer))$", cmd) if bts: - outfile.write('$DIFF_BIN '+bts.group(1).replace(CUT_PATH + r'/ref',REF_PATH + r'/ref')+' '+bts.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('$DIFF_BIN '+bts.group(1).replace(CUT_PATH, REF_PATH)+' '+bts.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) @@ -149,7 +149,7 @@ if __name__ == '__main__': for output in glob.glob(absolute_out.group(1) + '*'): output = path.relpath(output).replace('\\','/') output = re.sub('tests', CUT_PATH, output) - diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/ref',REF_PATH + r'/ref')+' '+output+' >> $LOG_FILE 2>&1') + diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH, REF_PATH)+' '+output+' >> $LOG_FILE 2>&1') outfile.write(('; ').join(diff_cmds)) if isar_out and "ref" in isar_out.group(1): outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH + r'/split_rendering/ref',REF_PATH + r'/split_rendering/ref')+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1') @@ -214,4 +214,4 @@ if __name__ == '__main__': outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_rendering/ref',REF_PATH + r'/split_rendering/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: - outfile.write(footer.read()) \ No newline at end of file + outfile.write(footer.read()) -- GitLab From c4a29865a1a200839ff074454a3b00d489a524b8 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 09:18:41 +0200 Subject: [PATCH 15/23] Cleanup of parse_command.py --- scripts/parse_commands.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index c2cc6a4188..670a94b307 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -24,6 +24,8 @@ if __name__ == '__main__': with open(input,'r') as infile: report = infile.read() + + # Try finding all command lines. May fail if the pytest-html version does not produce a multiple line output. cmds_enc = re.findall(r'REF encoder command:\s*\n\s*(\S.*)', report, re.MULTILINE) cmds_dec = re.findall(r'REF decoder command:\s*\n\s*(\S.*)', report, re.MULTILINE) cmds_rend = re.findall(r'Running command\s*\n\s*(\S.*)', report, re.MULTILINE) @@ -60,7 +62,7 @@ if __name__ == '__main__': elif "Running ISAR post renderer command" in line: isar_post_rend_cmd = True - # Remove HTML tags + # Remove trailing HTML tags, if any cmds_rend = [x.split('<')[0] for x in cmds_rend] cmds_isar_post_rend = [x.split('<')[0] for x in cmds_isar_post_rend] @@ -71,6 +73,7 @@ if __name__ == '__main__': cmds_isar_post_rend.sort() # Remove duplicates from cmds_enc -- some decoder tests use same encoder options + # Relies on the list being sorted i = 0 while i + 1 < len(cmds_enc): if " ".join(cmds_enc[i].split()[:-1]) == " ".join(cmds_enc[i+1].split()[:-1]): @@ -152,9 +155,9 @@ if __name__ == '__main__': diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH, REF_PATH)+' '+output+' >> $LOG_FILE 2>&1') outfile.write(('; ').join(diff_cmds)) if isar_out and "ref" in isar_out.group(1): - outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH + r'/split_rendering/ref',REF_PATH + r'/split_rendering/ref')+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1') + outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1') if isar_md_out and "ref" in isar_md_out.group(1): - outfile.write('; $DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH + r'/split_rendering/ref',REF_PATH + r'/split_rendering/ref')+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('; $DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n') else: outfile.write('\n') outfile.write('\n\n') @@ -202,7 +205,7 @@ if __name__ == '__main__': arg = re.sub('ISAR_post_rend_ref(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) if re.search("^tests.*bit$",arg): - arg = re.sub('tests/split_rendering/ref', REF_PATH + r'/split_rendering/ref', arg) + arg = re.sub('tests', REF_PATH, arg) arg = re.sub('tests', CUT_PATH, arg) args.append(arg) cmd = ' '.join(args) @@ -211,7 +214,7 @@ if __name__ == '__main__': outfile.write(cmd+'\n') out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) if out and "ref" in out.group(1): - outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_rendering/ref',REF_PATH + r'/split_rendering/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH, REF_PATH)+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) -- GitLab From 3398f7ce007b35da871f31a113edad10ab84240b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 09:43:08 +0200 Subject: [PATCH 16/23] Add back mixed_mc714_foa_masa2_ism4.txt for old renderer test --- scripts/testv/mixed_mc714_foa_masa2_ism4.txt | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 scripts/testv/mixed_mc714_foa_masa2_ism4.txt diff --git a/scripts/testv/mixed_mc714_foa_masa2_ism4.txt b/scripts/testv/mixed_mc714_foa_masa2_ism4.txt new file mode 100644 index 0000000000..800e88ae7c --- /dev/null +++ b/scripts/testv/mixed_mc714_foa_masa2_ism4.txt @@ -0,0 +1,24 @@ +mixed_mc714_foa_masa2_ism4.wav +7 +MC +1 +7_1_4 +SBA +13 +1 +MASA +17 +2 +stv2MASA2TC48c.met +ISM +19 +stvISM1.csv +ISM +20 +stvISM2.csv +ISM +21 +stvISM3.csv +ISM +22 +stvISM4.csv -- GitLab From 374f0dc06d95d1eb73ea87b40082a0299fbc9629 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 09:54:25 +0200 Subject: [PATCH 17/23] Add back mixed_mc714_foa_masa2_ism4.wav for old renderer test --- scripts/testv/mixed_mc714_foa_masa2_ism4.wav | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 scripts/testv/mixed_mc714_foa_masa2_ism4.wav diff --git a/scripts/testv/mixed_mc714_foa_masa2_ism4.wav b/scripts/testv/mixed_mc714_foa_masa2_ism4.wav new file mode 100644 index 0000000000..c680493c22 --- /dev/null +++ b/scripts/testv/mixed_mc714_foa_masa2_ism4.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900feffb59cec3a831c23b07e6f2f305e74d382c4d64903e65cd7cc321bd7083 +size 6336044 -- GitLab From d8058c376c05e5397bfd04f05eb375763bd4fe31 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 13:32:30 +0200 Subject: [PATCH 18/23] Fixes for using renderer_short test in ivas-conformance* --- .gitlab-ci.yml | 8 ++++---- scripts/parse_commands.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5044cfb6ab..240cad7d9c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1302,7 +1302,7 @@ ivas-conformance: - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - - mkdir testvec/testv/renderer + - mkdir testvec/testv/renderer_short - mkdir testvec/testv/split_rendering - mkdir testvec/bin - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv @@ -1312,7 +1312,7 @@ ivas-conformance: - cp -r -force -ErrorAction Ignore scripts/trajectories testvec - cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - - cp -r -force -ErrorAction Ignore tests/renderer/ref testvec/testv/renderer/ref + - cp -r -force -ErrorAction Ignore tests/renderer_short/ref testvec/testv/renderer_short/ref - cp -r -force -ErrorAction Ignore tests/split_rendering/ref testvec/testv/split_rendering/ref - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns @@ -1392,7 +1392,7 @@ ivas-conformance-linux: - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - - mkdir testvec/testv/renderer + - mkdir testvec/testv/renderer_short - mkdir testvec/testv/split_rendering - mkdir testvec/bin - cp -r scripts/testv/* testvec/testv @@ -1402,7 +1402,7 @@ ivas-conformance-linux: - cp -r scripts/trajectories testvec - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - cp -r tests/ref testvec/testv/ref - - cp -r tests/renderer/ref testvec/testv/renderer/ref + - cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref - cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 670a94b307..99b13da01d 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -179,7 +179,7 @@ if __name__ == '__main__': arg = path.relpath(arg).replace('\\','/') arg = re.sub('IVAS_rend_ref(.exe)?', '$CUT_REND_BIN', arg) arg = re.sub('scripts', TESTV_PATH, arg) - arg = re.sub('tests/renderer/data', TESTV_PATH + r'renderer/data/', arg) + arg = re.sub('tests/renderer_short/data', TESTV_PATH + r'renderer_short/data/', arg) arg = re.sub('tests', CUT_PATH, arg) args.append(arg) cmd = ' '.join(args) @@ -188,7 +188,7 @@ if __name__ == '__main__': outfile.write(cmd+'\n') out = re.search(r"-o\s(([\S]+)(.wav|.raw|.pcm))", cmd) if out and "ref" in out.group(1): - outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/renderer/ref',REF_PATH + r'/renderer/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') + outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/renderer_short/ref',REF_PATH + r'/renderer_short/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n') outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) -- GitLab From 9e0a74dbdc17baf7ce6ea8e545fefca551fca64e Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 13:50:57 +0200 Subject: [PATCH 19/23] Another fix for using renderer_short test in ivas-conformance* --- scripts/rend_header.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/rend_header.txt b/scripts/rend_header.txt index a83b0a17bc..ce2e274c66 100644 --- a/scripts/rend_header.txt +++ b/scripts/rend_header.txt @@ -15,7 +15,7 @@ LOG_FILE=Readme_IVAS_rend_log.txt rm -rf tmp rm -rf $CUT_PATH -mkdir -p $CUT_PATH/renderer/ref -mkdir -p $CUT_PATH/renderer/data +mkdir -p $CUT_PATH/renderer_short/ref +mkdir -p $CUT_PATH/renderer_short/data -- GitLab From 658e766803422c65517732aa5155ebd17af15ca3 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 14:47:38 +0200 Subject: [PATCH 20/23] Revert to old html parsing in parse_commands.py. Seems to work better on Windows runners --- scripts/parse_commands.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 99b13da01d..aaafed1e1d 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -24,12 +24,10 @@ if __name__ == '__main__': with open(input,'r') as infile: report = infile.read() - - # Try finding all command lines. May fail if the pytest-html version does not produce a multiple line output. - cmds_enc = re.findall(r'REF encoder command:\s*\n\s*(\S.*)', report, re.MULTILINE) - cmds_dec = re.findall(r'REF decoder command:\s*\n\s*(\S.*)', report, re.MULTILINE) - cmds_rend = re.findall(r'Running command\s*\n\s*(\S.*)', report, re.MULTILINE) - cmds_isar_post_rend = re.findall(r'Running ISAR post renderer command\s*\n\s*(\S.*)', report, re.MULTILINE) + cmds_enc = [] + cmds_dec = [] + cmds_rend = [] + cmds_isar_post_rend = [] # Depending on pytest-html version, the data may be in a json-blob. If so, the code below should handle this format. if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]): @@ -62,10 +60,22 @@ if __name__ == '__main__': elif "Running ISAR post renderer command" in line: isar_post_rend_cmd = True + # If the above reading failed, try this variant instead. + if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]): + with open(input,'r') as infile: + for line in infile.readlines(): + cmds_enc.extend(re.findall(r"DUT encoder command:\\n\\t(.*?)\\n", line)) + cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line)) + cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line)) + cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line)) + + # Remove trailing HTML tags, if any cmds_rend = [x.split('<')[0] for x in cmds_rend] cmds_isar_post_rend = [x.split('<')[0] for x in cmds_isar_post_rend] + + # Sort lists to keep deterministic order between runs cmds_enc.sort() cmds_dec.sort() -- GitLab From 328391ba395174734d2e38ad15460dbd9a16ed4b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 15:12:33 +0200 Subject: [PATCH 21/23] Revert html parsing --- scripts/parse_commands.py | 88 +++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index aaafed1e1d..370e18cbdf 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -2,7 +2,6 @@ import argparse import re -import json from os import path from pathlib import Path import glob @@ -22,59 +21,56 @@ if __name__ == '__main__': here = Path(__file__).parent.resolve() - with open(input,'r') as infile: - report = infile.read() - cmds_enc = [] - cmds_dec = [] - cmds_rend = [] - cmds_isar_post_rend = [] + cmds_enc=[] + cmds_dec=[] + cmds_rend=[] + cmds_isar_post_rend=[] - # Depending on pytest-html version, the data may be in a json-blob. If so, the code below should handle this format. - if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]): - lines = re.split(r'\\n', report) - enc_cmd = False - dec_cmd = False - rend_cmd = False - isar_post_rend_cmd = False - for line in lines: - line = line.split("
")[0].replace('\\t','') # Remove trailing html tags - if enc_cmd: - cmds_enc.append(line) - enc_cmd = False - elif dec_cmd: - cmds_dec.append(line) - dec_cmd = False - elif rend_cmd: - cmds_rend.append(line) - rend_cmd = False - elif isar_post_rend_cmd: - cmds_isar_post_rend.append(line) - isar_post_rend_cmd = False - else: - if "REF encoder command" in line: - enc_cmd = True - elif "REF decoder command" in line: - dec_cmd = True - elif "Running command" in line: - rend_cmd = True - elif "Running ISAR post renderer command" in line: - isar_post_rend_cmd = True - # If the above reading failed, try this variant instead. - if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]): - with open(input,'r') as infile: + if path.isdir(input): + input = Path(input).rglob('*.html') + else: + input = [input] + for html_report in input: + + with open(html_report,'r') as infile: for line in infile.readlines(): cmds_enc.extend(re.findall(r"DUT encoder command:\\n\\t(.*?)\\n", line)) cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line)) cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line)) cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line)) - - # Remove trailing HTML tags, if any - cmds_rend = [x.split('<')[0] for x in cmds_rend] - cmds_isar_post_rend = [x.split('<')[0] for x in cmds_isar_post_rend] - - + # If pytest-html < v4 is used, the parsing will fail and render empty lists. This is a work-around in case that happens. + if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]): + for html_report in input: + with open(html_report,'r') as infile: + enc_cmd = False + dec_cmd = False + rend_cmd = False + isar_post_rend_cmd = False + for line in infile.readlines(): + line = line.split("
")[0] # Remove trailing html tags + if enc_cmd: + cmds_enc.append(line) + enc_cmd = False + elif dec_cmd: + cmds_dec.append(line) + dec_cmd = False + elif rend_cmd: + cmds_rend.append(line) + rend_cmd = False + elif isar_post_rend_cmd: + cmds_isar_post_rend.append(line) + isar_post_rend_cmd = False + else: + if "DUT encoder command" in line: + enc_cmd = True + elif "DUT decoder command" in line: + dec_cmd = True + elif "Running command" in line: + rend_cmd = True + elif "Running ISAR post renderer command" in line: + isar_post_rend_cmd = True # Sort lists to keep deterministic order between runs cmds_enc.sort() -- GitLab From 2df5bf8e5d6ec1709bee9e112b919967db3ca444 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 21 Oct 2025 20:32:57 +0200 Subject: [PATCH 22/23] Restore REF in command line pattern --- scripts/parse_commands.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index 370e18cbdf..c5f4cd71fd 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -13,7 +13,7 @@ if __name__ == '__main__': parser.add_argument('txt_file',type=str,help='Output txt file, e.g. output.txt') args = parser.parse_args() input = args.input - txt_file = args.txt_file + txt_file = args.txt_file TESTV_PATH='$TESTV_PATH' REF_PATH='$REF_PATH' @@ -26,6 +26,7 @@ if __name__ == '__main__': cmds_rend=[] cmds_isar_post_rend=[] + all_args = set() if path.isdir(input): input = Path(input).rglob('*.html') @@ -35,8 +36,8 @@ if __name__ == '__main__': with open(html_report,'r') as infile: for line in infile.readlines(): - cmds_enc.extend(re.findall(r"DUT encoder command:\\n\\t(.*?)\\n", line)) - cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line)) + cmds_enc.extend(re.findall(r"REF encoder command:\\n\\t(.*?)\\n", line)) + cmds_dec.extend(re.findall(r"REF decoder command:\\n\\t(.*?)\\n", line)) cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line)) cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line)) @@ -63,9 +64,9 @@ if __name__ == '__main__': cmds_isar_post_rend.append(line) isar_post_rend_cmd = False else: - if "DUT encoder command" in line: + if "REF encoder command" in line: enc_cmd = True - elif "DUT decoder command" in line: + elif "REF decoder command" in line: dec_cmd = True elif "Running command" in line: rend_cmd = True @@ -87,6 +88,9 @@ if __name__ == '__main__': else: i = i + 1 + # Filter out networkSimulator_g192 commands from cmds_rend + cmds_rend = [cmd for cmd in cmds_rend if not "networkSimulator_g192" in cmd] + with open(txt_file.replace('.','_enc.'),'w', newline='\n') as outfile: with open('scripts/enc_header.txt','r') as header: outfile.write(header.read()) @@ -224,3 +228,4 @@ if __name__ == '__main__': outfile.write('\n') with open('scripts/script_footer.txt','r') as footer: outfile.write(footer.read()) + -- GitLab From 4a48e012a1d098bf3bf6cdc60b4569587f5f55df Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 22 Oct 2025 08:27:54 +0200 Subject: [PATCH 23/23] Fix for ivas-conformance job --- .gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 240cad7d9c..c46b57b5e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1285,10 +1285,7 @@ ivas-conformance: # Reference creation - python scripts/prepare_combined_format_inputs.py - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer_short/test_renderer.py", "tests/split_rendering/test_split_rendering.py" - - python -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files - - # Output creation - - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data -- GitLab