Merge branch 'isar_conformance_tests_addition' into 'main' (e5acb70e) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+29 −7

Original line number	Diff line number	Diff line
		@@ -513,7 +513,6 @@ build-codec-windows-msbuild:
		script:
		- *print-common-info-windows
		- *activate-WX-windows
		- python .\scripts\strip_split_rendering.py
		- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug

		# ---------------------------------------------------------------
		@@ -1380,7 +1379,6 @@ ivas-conformance:
		- 123
		script:
		- *print-common-info-windows
		- python .\scripts\strip_split_rendering.py
		- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug
		- cp -force IVAS_cod.exe IVAS_cod_ref.exe
		- cp -force IVAS_dec.exe IVAS_dec_ref.exe
		@@ -1389,11 +1387,12 @@ ivas-conformance:
		# Reference creation
		- python tests/create_short_testvectors.py
		- python scripts/prepare_combined_format_inputs.py
		- python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files
		- python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files
		- $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py"
		- python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files
		- python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files

		# Output creation
		- python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html
		- python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html
		- python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt

		# Copy input data and output ref data
		@@ -1402,10 +1401,13 @@ ivas-conformance:
		- if (Test-Path TMP_ENC) {rm -r -force TMP_ENC}
		- if (Test-Path TMP_JBM) {rm -r -force TMP_JBM}
		- if (Test-Path TMP_REND) {rm -r -force TMP_REND}
		- if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND}
		- if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR}
		- mkdir testvec
		- mkdir testvec/binauralRenderer_interface
		- mkdir testvec/testv
		- mkdir testvec/testv/renderer
		- mkdir testvec/testv/split_rendering
		- mkdir testvec/bin
		- cp -force -ErrorAction Ignore scripts/testv/* testvec/testv
		- cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec
		@@ -1415,6 +1417,9 @@ ivas-conformance:
		- cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref
		- cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref
		- cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref
		- cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref
		- cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs
		- cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns

		# Remove redundant files
		- python scripts/cleanup_26252.py
		@@ -1425,9 +1430,12 @@ ivas-conformance:
		- cp Readme_IVAS_enc.txt testvec
		- cp Readme_IVAS_rend.txt testvec
		- cp Readme_IVAS_JBM_dec.txt testvec
		- cp Readme_IVAS_ISAR_dec.txt testvec
		- cp Readme_IVAS_ISAR_post_rend.txt testvec
		- cp IVAS_cod.exe testvec/bin
		- cp IVAS_dec.exe testvec/bin
		- cp IVAS_rend.exe testvec/bin
		- cp ISAR_post_rend.exe testvec/bin


		# Test run generated scripts in testvec
		@@ -1447,6 +1455,8 @@ ivas-conformance:
		- Readme_IVAS_enc.txt
		- Readme_IVAS_rend.txt
		- Readme_IVAS_JBM_dec.txt
		- Readme_IVAS_ISAR_dec.txt
		- Readme_IVAS_ISAR_post_rend.txt
		expose_as: "Draft IVAS conformance"
		reports:
		junit: report-junit.xml
		@@ -1471,7 +1481,7 @@ ivas-conformance-linux:
		# Reference creation
		- python3 tests/create_short_testvectors.py
		- python3 scripts/prepare_combined_format_inputs.py
		- TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py"
		- TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py"
		- python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files
		- python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files

		@@ -1485,6 +1495,7 @@ ivas-conformance-linux:
		- mkdir testvec/binauralRenderer_interface
		- mkdir testvec/testv
		- mkdir testvec/testv/renderer
		- mkdir testvec/testv/split_rendering
		- mkdir testvec/bin
		- cp -r scripts/testv/* testvec/testv
		- cp -r scripts/ls_layouts testvec
		@@ -1494,6 +1505,9 @@ ivas-conformance-linux:
		- cp -r tests/ref testvec/testv/ref
		- cp -r tests/dut/* testvec/testv/ref
		- cp -r tests/renderer/cut testvec/testv/renderer/ref
		- cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref
		- cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs
		- cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns

		# Remove redundant files
		- python3 scripts/cleanup_26252.py
		@@ -1504,6 +1518,8 @@ ivas-conformance-linux:
		- cp Readme_IVAS_enc.txt testvec
		- cp Readme_IVAS_rend.txt testvec
		- cp Readme_IVAS_JBM_dec.txt testvec
		- cp Readme_IVAS_ISAR_dec.txt testvec
		- cp Readme_IVAS_ISAR_post_rend.txt testvec

		# Create GCOV execs for coverage analysis
		- make clean
		@@ -1512,6 +1528,7 @@ ivas-conformance-linux:
		- cp IVAS_cod testvec/bin
		- cp IVAS_dec testvec/bin
		- cp IVAS_rend testvec/bin
		- cp ISAR_post_rend testvec/bin

		# Test run generated scripts in testvec
		- cd testvec
		@@ -1528,6 +1545,9 @@ ivas-conformance-linux:
		- commit_sha=$(git rev-parse HEAD)
		- genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha"

		# Check for failures
		- if [ $exit_code -eq 1 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi

		artifacts:
		name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA"
		expire_in: 1 week
		@@ -1540,6 +1560,8 @@ ivas-conformance-linux:
		- Readme_IVAS_enc.txt
		- Readme_IVAS_rend.txt
		- Readme_IVAS_JBM_dec.txt
		- Readme_IVAS_ISAR_dec.txt
		- Readme_IVAS_ISAR_post_rend.txt
		- coverage.info
		- coverage
		expose_as: "Draft IVAS conformance -- Linux"

scripts/cleanup_26252.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -4,7 +4,7 @@ from pathlib import Path
		import os
		import glob

		files = ['Readme_IVAS_JBM_dec.txt','Readme_IVAS_dec.txt','Readme_IVAS_enc.txt','Readme_IVAS_rend.txt'];
		files = ['Readme_IVAS_JBM_dec.txt','Readme_IVAS_dec.txt','Readme_IVAS_enc.txt','Readme_IVAS_rend.txt', 'Readme_IVAS_ISAR_dec.txt', 'Readme_IVAS_ISAR_post_rend.txt'];

		used_files = []
		kept = 0

scripts/dec_isar_header.txt

0 → 100644

+21 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env bash

		if [ "$#" -ne 1 ]; then
		echo "Usage: $0 <CUT_DEC_BIN>"
		exit -1
		fi

		CUT_DEC_BIN=$1
		DIFF_BIN="diff"

		TESTV_PATH="."
		REF_PATH="./testv"
		CUT_PATH="./TMP_DEC_ISAR"
		LOG_FILE=Readme_IVAS_ISAR_dec_log.txt

		rm -rf tmp
		rm -rf $CUT_PATH
		mkdir -p $CUT_PATH
		mkdir -p $CUT_PATH/split_rendering/cut

scripts/isar_post_rend_header.txt

0 → 100644

+21 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env bash

		if [ "$#" -ne 1 ]; then
		echo "Usage: $0 <CUT_ISAR_POST_REND_BIN>"
		exit -1
		fi

		CUT_ISAR_POST_REND_BIN=$1
		DIFF_BIN="diff"

		TESTV_PATH="."
		REF_PATH="./testv"
		CUT_PATH="./TMP_ISAR_POST_REND"
		LOG_FILE=Readme_IVAS_isar_post_rend_log.txt

		rm -rf tmp
		rm -rf $CUT_PATH
		mkdir -p $CUT_PATH/split_rendering/cut

scripts/parse_commands.py

+59 −5

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@ if __name__ == '__main__':
		cmds_enc=[]
		cmds_dec=[]
		cmds_rend=[]
		cmds_isar_post_rend=[]


		if path.isdir(input):
		@@ -37,15 +38,18 @@ if __name__ == '__main__':
		cmds_enc.extend(re.findall(r"DUT encoder command:\\n\\t(.*?)\\n", line))
		cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line))
		cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line))
		cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line))

		# If pytest-html < v4 is used, the parsing will fail and render empty lists. This is a work-around in case that happens.
		if all(not x for x in [cmds_enc, cmds_dec, cmds_rend]):
		if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]):
		for html_report in input:
		with open(html_report,'r') as infile:
		enc_cmd = False
		dec_cmd = False
		rend_cmd = False
		isar_post_rend_cmd = False
		for line in infile.readlines():
		line = line.split("<br/>")[0] # Remove trailing html tags
		if enc_cmd:
		cmds_enc.append(line)
		enc_cmd = False
		@@ -55,6 +59,9 @@ if __name__ == '__main__':
		elif rend_cmd:
		cmds_rend.append(line)
		rend_cmd = False
		elif isar_post_rend_cmd:
		cmds_isar_post_rend.append(line)
		isar_post_rend_cmd = False
		else:
		if "DUT encoder command" in line:
		enc_cmd = True
		@@ -62,11 +69,14 @@ if __name__ == '__main__':
		dec_cmd = True
		elif "Running command" in line:
		rend_cmd = True
		elif "Running ISAR post renderer command" in line:
		isar_post_rend_cmd = True

		# Sort lists to keep deterministic order between runs
		cmds_enc.sort()
		cmds_dec.sort()
		cmds_rend.sort()
		cmds_isar_post_rend.sort()

		with open(txt_file.replace('.','_enc.'),'w', newline='\n') as outfile:
		with open('scripts/enc_header.txt','r') as header:
		@@ -90,11 +100,13 @@ if __name__ == '__main__':
		with open('scripts/script_footer.txt','r') as footer:
		outfile.write(footer.read())

		with open(txt_file.replace('.','_dec.'),'w', newline='\n') as outfile_dec, open(txt_file.replace('.','_JBM_dec.'),'w', newline='\n') as outfile_jbm:
		with open(txt_file.replace('.','_dec.'),'w', newline='\n') as outfile_dec, open(txt_file.replace('.','_JBM_dec.'),'w', newline='\n') as outfile_jbm, open(txt_file.replace('.','_ISAR_dec.'),'w', newline='\n') as outfile_isar:
		with open('scripts/dec_header.txt','r') as header:
		outfile_dec.write(header.read())
		with open('scripts/jbm_header.txt','r') as header:
		outfile_jbm.write(header.read())
		with open('scripts/dec_isar_header.txt','r') as header:
		outfile_isar.write(header.read())
		for cmd in cmds_dec:
		absolute_out = re.search(r"\s(([\S]+)(.wav))$", cmd)

		@@ -106,20 +118,29 @@ if __name__ == '__main__':
		arg = re.sub('IVAS_dec(.exe)?', '$CUT_DEC_BIN', arg)
		arg = re.sub('scripts', TESTV_PATH, arg)
		arg = re.sub('tests/ref', REF_PATH + r'/ref', arg) # For .fer cases the bitstream is in ref
		arg = re.sub('tests/split_rendering/renderer_configs', REF_PATH + r'/split_rendering/renderer_configs', arg)
		if re.search("^tests.*192$",arg):
		arg = re.sub('tests/split_rendering/cut', REF_PATH + r'/split_rendering/ref', arg)
		if re.search("\.wav$",arg):
		arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav
		else:
		arg = re.sub('tests/dut', REF_PATH + r'/ref', arg) # Input argument
		if re.search("^tests.*bit$",arg):
		arg = re.sub('tests', CUT_PATH, arg) # Output argument for decoder ends with .wav
		args.append(arg)
		cmd = ' '.join(args)

		if 'VOIP' in cmd:
		outfile = outfile_jbm
		elif 'BINAURAL_SPLIT' in cmd:
		outfile = outfile_isar
		else:
		outfile = outfile_dec

		outfile.write(cmd+'\n')
		out = re.search(r"\s(([\S]+)(.wav))$", cmd)
		isar_out = re.search(r"\s(([\S]+)(.bit))$", cmd)
		isar_md_out = re.search(r"-om\s(([\S]+)(.bit))", cmd)
		if out:
		diff_cmds=[]
		for output in glob.glob(absolute_out.group(1) + '*'):
		@@ -127,11 +148,18 @@ if __name__ == '__main__':
		output = re.sub('tests', CUT_PATH, output)
		diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/dut',REF_PATH + r'/ref')+' '+output+' >> $LOG_FILE 2>&1')
		outfile.write(('; ').join(diff_cmds))
		if isar_out and "cut" in isar_out.group(1):
		outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1\n')
		if isar_md_out and "cut" in isar_md_out.group(1):
		outfile.write('$DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n')

		outfile.write('\n\n')
		with open('scripts/script_footer.txt','r') as footer:
		outfile_dec.write(footer.read())
		footer.seek(0)
		outfile_jbm.write(footer.read())
		footer.seek(0)
		outfile_isar.write(footer.read())

		with open(txt_file.replace('.','_rend.'),'w', newline='\n') as outfile:
		with open('scripts/rend_header.txt','r') as header:
		@@ -157,3 +185,29 @@ if __name__ == '__main__':
		outfile.write('\n')
		with open('scripts/script_footer.txt','r') as footer:
		outfile.write(footer.read())

		with open(txt_file.replace('.','_ISAR_post_rend.'),'w', newline='\n') as outfile:
		with open('scripts/isar_post_rend_header.txt','r') as header:
		outfile.write(header.read())
		for cmd in cmds_isar_post_rend:
		args = []
		for arg in cmd.split():
		# Adjust file arguments, pass other arguments as they are
		if path.exists(arg):
		arg = path.relpath(arg).replace('\\','/')
		arg = re.sub('ISAR_post_rend(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg)
		arg = re.sub('scripts', TESTV_PATH, arg)
		if re.search("^tests.*bit$",arg):
		arg = re.sub('tests/split_rendering/cut', REF_PATH + r'/split_rendering/ref', arg)
		arg = re.sub('tests', CUT_PATH, arg)
		args.append(arg)
		cmd = ' '.join(args)

		if "cut" in cmd:
		outfile.write(cmd+'\n')
		out = re.search(r"-o\s(([\S]+)(.wav\|.raw\|.pcm))", cmd)
		if out and "cut" in out.group(1):
		outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_rendering/cut',REF_PATH + r'/split_rendering/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n')
		outfile.write('\n')
		with open('scripts/script_footer.txt','r') as footer:
		outfile.write(footer.read())
		No newline at end of file