update renderer tests and gitlab-ci configuration to accomodate a change in... (da447061) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+8 −3

Original line number	Diff line number	Diff line
		@@ -636,17 +636,22 @@ renderer-pytest-on-merge-request:
		- make -j IVAS_rend
		- mv IVAS_rend IVAS_rend_ref

		### If ref_using_main is not set, checkout the source branch to use scripts and input from there
		- if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi
		- exit_code=0
		- python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py --create_ref \|\| exit_code=$?

		# back to source branch
		- git checkout $source_branch_commit_sha
		- make clean
		- make -j IVAS_rend

		### Run test using scripts and input from main
		- if [ $ref_using_main == 1 ]; then git checkout $target_commit; fi
		### Run test using branch scripts and input
		- if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi

		# run test
		- exit_code=0
		- python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py \|\| exit_code=$?
		- python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py --create_cut \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- *merge-request-comparison-check

tests/renderer/README.md

+15 −6

Original line number	Diff line number	Diff line
		@@ -8,26 +8,35 @@ See also the [contribution page](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-c

		### Run tests with:

		## Smoke test:

		`python3 -m pytest -q -n auto tests/renderer/test_renderer.py`

		## Comparison test:

		`python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref`
		(note: requires IVAS_rend_ref in root!)

		`python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut`

		### Important flags (see [pytest docs](https://docs.pytest.org/en/7.2.x/) for more information):

		* `-k` flag can filter test cases, e.g. `-k "test_ism_binaural_static"`
		* `-rA` reports ALL (pass, xpass, xfail, fail) instead of the default behaviour of reporting only failed tests\
		- `-k` flag can filter test cases, e.g. `-k "test_ism_binaural_static"`
		- `-rA` reports ALL (pass, xpass, xfail, fail) instead of the default behaviour of reporting only failed tests\
		this option will also report captured logs, required for obtaining the commandline of testcases that pass or xfail
		* `--last-failed` re-runs only the cases that failed in the last test run
		* `--collect-only` is useful when adding new testcases to check if argument parametrization is working correctly
		- `--last-failed` re-runs only the cases that failed in the last test run
		- `--collect-only` is useful when adding new testcases to check if argument parametrization is working correctly

		### Directory tree

		```
		.
		├── compare_audio.py -> Python implementation of CompAudio, used for comparisons in tests
		├── conftest.py -> Pytest configuration (enable commandline argument ingestion)
		├── constants.py -> Important paths, formats, metadata files and commandline templates
		├── cut -> Default location for output files for test conditions
		├── data -> Input test vectors
		├── ref -> Default location for output files for reference conditions
		├── test_renderer_be_comparison.py -> Tests for CI Merge Request pipeline to compare renderer bit-exactness
		├── test_renderer.py -> Runs the renderer for all modes
		└── utils.py -> Wrapper functions for executables for use in testcases
		```

tests/renderer/conftest.py

0 → 100644

+44 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		"""
		(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository. All Rights Reserved.

		This software is protected by copyright law and by international treaties.
		The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository retain full ownership rights in their respective contributions in
		the software. This notice grants no license of any kind, including but not limited to patent
		license, nor is any license granted by implication, estoppel or otherwise.

		Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
		contributions.

		This software is provided "AS IS", without any express or implied warranties. The software is in the
		development stage. It is intended exclusively for experts who have experience with such software and
		solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
		and fitness for a particular purpose are hereby disclaimed and excluded.

		Any dispute, controversy or claim arising under or in relation to providing this software shall be
		submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""


		def pytest_addoption(parser):
		parser.addoption(
		"--create_ref",
		action="store_true",
		default=False,
		)
		parser.addoption(
		"--create_cut",
		action="store_true",
		default=False,
		)

tests/renderer/test_renderer.py

+269 −243

File changed.

Preview size limit exceeded, changes collapsed.

tests/renderer/test_renderer_be_comparison.py

deleted100644 → 0

+0 −328

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		"""
		(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository. All Rights Reserved.

		This software is protected by copyright law and by international treaties.
		The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository retain full ownership rights in their respective contributions in
		the software. This notice grants no license of any kind, including but not limited to patent
		license, nor is any license granted by implication, estoppel or otherwise.

		Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
		contributions.

		This software is provided "AS IS", without any express or implied warranties. The software is in the
		development stage. It is intended exclusively for experts who have experience with such software and
		solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
		and fitness for a particular purpose are hereby disclaimed and excluded.

		Any dispute, controversy or claim arising under or in relation to providing this software shall be
		submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		import pytest

		from .utils import *

		""" Ambisonics """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		def test_ambisonics(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, is_comparetest=True)


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt, framing_5ms):
		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		framing_5ms=(framing_5ms == "5ms"),
		is_comparetest=True,
		)


		@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_ambisonics_binaural_headrotation(
		test_info, in_fmt, out_fmt, trj_file, framing_5ms
		):
		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		framing_5ms=(framing_5ms == "5ms"),
		is_comparetest=True,
		)


		""" Multichannel """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		def test_multichannel(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, is_comparetest=True)


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, framing_5ms):
		if in_fmt in ["MONO", "STEREO"]:
		pytest.skip("MONO or STEREO to Binaural rendering unsupported")

		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		framing_5ms=(framing_5ms == "5ms"),
		is_comparetest=True,
		)


		@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_multichannel_binaural_headrotation(
		test_info, in_fmt, out_fmt, trj_file, framing_5ms
		):
		if in_fmt in ["MONO", "STEREO"]:
		pytest.skip("MONO or STEREO to Binaural rendering unsupported")

		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		framing_5ms=(framing_5ms == "5ms"),
		is_comparetest=True,
		)


		""" ISM """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		def test_ism(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]
		)


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_ism_binaural_static(test_info, in_fmt, out_fmt, framing_5ms):

		try:
		in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt]
		except:
		in_meta_files = None

		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		in_meta_files=in_meta_files,
		framing_5ms=(framing_5ms == "5ms"),
		is_comparetest=True,
		)


		@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms):

		try:
		in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt]
		except:
		in_meta_files = None

		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		in_meta_files=in_meta_files,
		framing_5ms=(framing_5ms == "5ms"),
		is_comparetest=True,
		)


		""" MASA """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA)
		def test_masa(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]
		)


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_masa_binaural_static(test_info, in_fmt, out_fmt, framing_5ms):
		if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]:
		pytest.skip("Skipping binaural room outputs for MASA as unimplemented.")

		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt],
		framing_5ms=(framing_5ms == "5ms"),
		)


		@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_masa_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms):
		if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]:
		pytest.skip("Skipping binaural room outputs for MASA as unimplemented.")

		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt],
		framing_5ms=(framing_5ms == "5ms"),
		)


		@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND)
		def test_masa_prerend(test_info, in_fmt):
		compare_renderer_vs_mergetarget_plus_meta(
		test_info,
		"META",
		"MASA2",
		metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"),
		)


		""" Custom loudspeaker layouts """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST)
		def test_custom_ls_input(test_info, in_layout, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info,
		CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"),
		out_fmt,
		is_comparetest=True,
		)


		@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS)
		def test_custom_ls_output(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"),
		is_comparetest=True,
		)


		@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST)
		def test_custom_ls_input_output(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info,
		CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"),
		CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"),
		is_comparetest=True,
		)


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_custom_ls_input_binaural(test_info, in_layout, out_fmt, framing_5ms):
		compare_renderer_vs_mergetarget(
		test_info,
		CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"),
		out_fmt,
		framing_5ms=(framing_5ms == "5ms"),
		is_comparetest=True,
		)


		@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_custom_ls_input_binaural_headrotation(
		test_info, in_layout, out_fmt, trj_file, framing_5ms
		):
		compare_renderer_vs_mergetarget(
		test_info,
		CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"),
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		framing_5ms=(framing_5ms == "5ms"),
		is_comparetest=True,
		)


		""" Metadata / scene description input """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST)
		@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST)
		def test_metadata(test_info, in_fmt, out_fmt, framing_5ms):
		compare_renderer_vs_mergetarget(
		test_info,
		"META",
		out_fmt,
		metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"),
		framing_5ms=(framing_5ms == "5ms"),
		is_comparetest=True,
		)


		""" non diegetic pan """


		# @pytest.mark.parametrize("out_fmt", ["STEREO"])
		# @pytest.mark.parametrize("in_fmt", ["MONO"])
		# @pytest.mark.parametrize("non_diegetic_pan", ["0", "-0.2", "0.5", "1", "-1"])
		# def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan):
		# compare_renderer_vs_mergetarget(
		# test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan)


		# @pytest.mark.parametrize("out_fmt", ["STEREO"])
		# @pytest.mark.parametrize("in_fmt", ["ISM1"])
		# @pytest.mark.parametrize("non_diegetic_pan", ["0", "-0.2", "0.5", "1", "-1"])
		# def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan):
		# compare_renderer_vs_mergetarget(
		# test_info, in_fmt, out_fmt, non_diegetic_pan=non_diegetic_pan)