addition of isar confirmance tests (df740dab) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+21 −7

Original line number	Diff line number	Diff line
		@@ -513,7 +513,6 @@ build-codec-windows-msbuild:
		script:
		- *print-common-info-windows
		- *activate-WX-windows
		- python .\scripts\strip_split_rendering.py
		- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug

		# ---------------------------------------------------------------
		@@ -1380,7 +1379,6 @@ ivas-conformance:
		- 123
		script:
		- *print-common-info-windows
		- python .\scripts\strip_split_rendering.py
		- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug
		- cp -force IVAS_cod.exe IVAS_cod_ref.exe
		- cp -force IVAS_dec.exe IVAS_dec_ref.exe
		@@ -1389,11 +1387,11 @@ ivas-conformance:
		# Reference creation
		- python tests/create_short_testvectors.py
		- python scripts/prepare_combined_format_inputs.py
		- python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files
		- python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files
		- python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files
		- python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files

		# Output creation
		- python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html
		- python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html
		- python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt

		# Copy input data and output ref data
		@@ -1402,10 +1400,13 @@ ivas-conformance:
		- if (Test-Path TMP_ENC) {rm -r -force TMP_ENC}
		- if (Test-Path TMP_JBM) {rm -r -force TMP_JBM}
		- if (Test-Path TMP_REND) {rm -r -force TMP_REND}
		- if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND}
		- if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR}
		- mkdir testvec
		- mkdir testvec/binauralRenderer_interface
		- mkdir testvec/testv
		- mkdir testvec/testv/renderer
		- mkdir testvec/testv/split_renderer
		- mkdir testvec/bin
		- cp -force -ErrorAction Ignore scripts/testv/* testvec/testv
		- cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec
		@@ -1415,6 +1416,7 @@ ivas-conformance:
		- cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref
		- cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref
		- cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref
		- cp -r -force -ErrorAction Ignore tests/split_renderer/cut testvec/testv/split_renderer/ref

		# Remove redundant files
		- python scripts/cleanup_26252.py
		@@ -1425,9 +1427,12 @@ ivas-conformance:
		- cp Readme_IVAS_enc.txt testvec
		- cp Readme_IVAS_rend.txt testvec
		- cp Readme_IVAS_JBM_dec.txt testvec
		- cp Readme_IVAS_ISAR_dec.txt testvec
		- cp Readme_IVAS_ISAR_post_rend.txt testvec
		- cp IVAS_cod.exe testvec/bin
		- cp IVAS_dec.exe testvec/bin
		- cp IVAS_rend.exe testvec/bin
		- cp ISAR_post_rend.exe testvec/bin


		# Test run generated scripts in testvec
		@@ -1447,6 +1452,8 @@ ivas-conformance:
		- Readme_IVAS_enc.txt
		- Readme_IVAS_rend.txt
		- Readme_IVAS_JBM_dec.txt
		- Readme_IVAS_ISAR_dec.txt
		- Readme_IVAS_ISAR_post_rend.txt
		expose_as: "Draft IVAS conformance"
		reports:
		junit: report-junit.xml
		@@ -1471,7 +1478,7 @@ ivas-conformance-linux:
		# Reference creation
		- python3 tests/create_short_testvectors.py
		- python3 scripts/prepare_combined_format_inputs.py
		- TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py"
		- TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py"
		- python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files
		- python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files

		@@ -1485,6 +1492,7 @@ ivas-conformance-linux:
		- mkdir testvec/binauralRenderer_interface
		- mkdir testvec/testv
		- mkdir testvec/testv/renderer
		- mkdir testvec/testv/split_renderer
		- mkdir testvec/bin
		- cp -r scripts/testv/* testvec/testv
		- cp -r scripts/ls_layouts testvec
		@@ -1494,6 +1502,7 @@ ivas-conformance-linux:
		- cp -r tests/ref testvec/testv/ref
		- cp -r tests/dut/* testvec/testv/ref
		- cp -r tests/renderer/cut testvec/testv/renderer/ref
		- cp -r tests/split_renderer/cut testvec/testv/split_renderer/ref

		# Remove redundant files
		- python3 scripts/cleanup_26252.py
		@@ -1504,6 +1513,8 @@ ivas-conformance-linux:
		- cp Readme_IVAS_enc.txt testvec
		- cp Readme_IVAS_rend.txt testvec
		- cp Readme_IVAS_JBM_dec.txt testvec
		- cp Readme_IVAS_ISAR_dec.txt testvec
		- cp Readme_IVAS_ISAR_post_rend.txt testvec

		# Create GCOV execs for coverage analysis
		- make clean
		@@ -1512,6 +1523,7 @@ ivas-conformance-linux:
		- cp IVAS_cod testvec/bin
		- cp IVAS_dec testvec/bin
		- cp IVAS_rend testvec/bin
		- cp ISAR_post_rend testvec/bin

		# Test run generated scripts in testvec
		- cd testvec
		@@ -1540,6 +1552,8 @@ ivas-conformance-linux:
		- Readme_IVAS_enc.txt
		- Readme_IVAS_rend.txt
		- Readme_IVAS_JBM_dec.txt
		- Readme_IVAS_ISAR_dec.txt
		- Readme_IVAS_ISAR_post_rend.txt
		- coverage.info
		- coverage
		expose_as: "Draft IVAS conformance -- Linux"

scripts/dec_isar_header.txt

0 → 100644

+20 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env bash

		if [ "$#" -ne 1 ]; then
		echo "Usage: $0 <CUT_DEC_BIN>"
		exit -1
		fi

		CUT_DEC_BIN=$1
		DIFF_BIN="diff"

		TESTV_PATH="."
		REF_PATH="./testv"
		CUT_PATH="./TMP_DEC_ISAR"
		LOG_FILE=Readme_IVAS_ISAR_dec_log.txt

		rm -rf tmp
		rm -rf $CUT_PATH
		mkdir -p $CUT_PATH
		mkdir -p $CUT_PATH/dut/isar/pre_rend_out

scripts/isar_post_rend_header.txt

0 → 100644

+21 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env bash

		if [ "$#" -ne 1 ]; then
		echo "Usage: $0 <CUT_ISAR_POST_REND_BIN>"
		exit -1
		fi

		CUT_ISAR_POST_REND_BIN=$1
		DIFF_BIN="diff"

		TESTV_PATH="."
		REF_PATH="./testv"
		CUT_PATH="./TMP_ISAR_POST_REND"
		LOG_FILE=Readme_IVAS_isar_post_rend_log.txt

		rm -rf tmp
		rm -rf $CUT_PATH
		mkdir -p $CUT_PATH/split_renderer/cut
		mkdir -p $CUT_PATH/split_renderer/data

scripts/parse_commands.py

+43 −5

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@ if __name__ == '__main__':
		cmds_enc=[]
		cmds_dec=[]
		cmds_rend=[]
		cmds_isar_post_rend=[]


		if path.isdir(input):
		@@ -37,9 +38,10 @@ if __name__ == '__main__':
		cmds_enc.extend(re.findall(r"DUT encoder command:\\n\\t(.*?)\\n", line))
		cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line))
		cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line))
		cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line))

		# If pytest-html < v4 is used, the parsing will fail and render empty lists. This is a work-around in case that happens.
		if all(not x for x in [cmds_enc, cmds_dec, cmds_rend]):
		if all(not x for x in [cmds_enc, cmds_dec, cmds_rend, cmds_isar_post_rend]):
		for html_report in input:
		with open(html_report,'r') as infile:
		enc_cmd = False
		@@ -55,6 +57,9 @@ if __name__ == '__main__':
		elif rend_cmd:
		cmds_rend.append(line)
		rend_cmd = False
		elif cmds_isar_post_rend:
		cmds_isar_post_rend.append(line)
		cmds_isar_post_rend = False
		else:
		if "DUT encoder command" in line:
		enc_cmd = True
		@@ -62,11 +67,15 @@ if __name__ == '__main__':
		dec_cmd = True
		elif "Running command" in line:
		rend_cmd = True
		elif "Running ISAR post renderer command" in line:
		cmds_isar_post_rend = True

		# Sort lists to keep deterministic order between runs
		cmds_enc.sort()
		cmds_dec.sort()
		cmds_rend.sort()
		cmds_ivas_isar_dec.sort()
		cmds_isar_post_rend.sort()

		with open(txt_file.replace('.','_enc.'),'w', newline='\n') as outfile:
		with open('scripts/enc_header.txt','r') as header:
		@@ -90,11 +99,13 @@ if __name__ == '__main__':
		with open('scripts/script_footer.txt','r') as footer:
		outfile.write(footer.read())

		with open(txt_file.replace('.','_dec.'),'w', newline='\n') as outfile_dec, open(txt_file.replace('.','_JBM_dec.'),'w', newline='\n') as outfile_jbm:
		with open(txt_file.replace('.','_dec.'),'w', newline='\n') as outfile_dec, open(txt_file.replace('.','_JBM_dec.'),'w', newline='\n') as outfile_jbm, open(txt_file.replace('.','_ISAR_dec.'),'w', newline='\n') as outfile_isar:
		with open('scripts/dec_header.txt','r') as header:
		outfile_dec.write(header.read())
		with open('scripts/jbm_header.txt','r') as header:
		outfile_jbm.write(header.read())
		with open('scripts/dec_isar_header.txt','r') as header:
		outfile_isar.write(header.read())
		for cmd in cmds_dec:
		absolute_out = re.search(r"\s(([\S]+)(.wav))$", cmd)

		@@ -115,6 +126,8 @@ if __name__ == '__main__':

		if 'VOIP' in cmd:
		outfile = outfile_jbm
		elif 'BINAURAL_SPLIT' in cmd:
		outfile = outfile_isar
		else:
		outfile = outfile_dec

		@@ -157,3 +170,28 @@ if __name__ == '__main__':
		outfile.write('\n')
		with open('scripts/script_footer.txt','r') as footer:
		outfile.write(footer.read())

		with open(txt_file.replace('.','_ISAR_post_rend.'),'w', newline='\n') as outfile:
		with open('scripts/isar_post_rend_header.txt','r') as header:
		outfile.write(header.read())
		for cmd in cmds_isar_post_rend:
		args = []
		for arg in cmd.split():
		# Adjust file arguments, pass other arguments as they are
		if path.exists(arg):
		arg = path.relpath(arg).replace('\\','/')
		arg = re.sub('ISAR_post_rend(.exe)?', '$CUT_ISAR_POST_REND_BIN', arg)
		arg = re.sub('scripts', TESTV_PATH, arg)
		arg = re.sub('tests/split_renderer/data', TESTV_PATH + r'split_renderer/data/', arg)
		arg = re.sub('tests', CUT_PATH, arg)
		args.append(arg)
		cmd = ' '.join(args)

		if "cut" in cmd:
		outfile.write(cmd+'\n')
		out = re.search(r"-o\s(([\S]+)(.wav\|.raw\|.pcm))", cmd)
		if out and "cut" in out.group(1):
		outfile.write('$DIFF_BIN '+out.group(1).replace(CUT_PATH + r'/split_renderer/cut',REF_PATH + r'/split_renderer/ref')+' '+out.group(1)+' >> $LOG_FILE 2>&1\n')
		outfile.write('\n')
		with open('scripts/script_footer.txt','r') as footer:
		outfile.write(footer.read())
		No newline at end of file

tests/conformance-test/conftest.py

+12 −0

Original line number	Diff line number	Diff line
		@@ -54,6 +54,13 @@ def pytest_addoption(parser):
		type=pathlib.Path,
		default='./bin/IVAS_rend',
		)
		parser.addoption(
		"--isar_post_renderer_path",
		action="store",
		help="path to renderer binary ISAR_post_rend(.exe)",
		type=pathlib.Path,
		default='./bin/ISAR_post_rend',
		)


		@pytest.fixture(scope="session")
		@@ -69,3 +76,8 @@ def decoder_path(request) -> str:
		@pytest.fixture(scope="session")
		def renderer_path(request) -> str:
		return str(request.config.option.renderer_path.absolute())


		@pytest.fixture(scope="session")
		def isar_post_renderer_path(request) -> str:
		return str(request.config.option.isar_post_renderer_path.absolute())