Merge branch 'kiene/external-renderer-ci-rework' into 'FhG/external-renderer' (a4d55835) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+19 −16

Original line number	Diff line number	Diff line
		@@ -408,46 +408,49 @@ external-renderer-cmake-msan-pytest:
		- .test-job-linux
		- .rules-merge-request
		needs: ["build-codec-linux-make"]
		timeout: "12 minutes"
		# TODO: set reasonable timeout, will most likely take less
		timeout: "20 minutes"
		stage: compare
		script:
		- *print-common-info

		# some helper variables - "\|\| true" to prevent failures from grep not finding anything
		- non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[rend(erer)[ -]non[ -]*be\]") \|\| true
		- ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[ref[ -]using[ -]main\]") \|\| true
		# TODO: needs splitting the test between reference and cut generation
		#- ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[ref[ -]using[ -]main\]") \|\| true

		# store the current commit hash
		- source_branch_commit_sha=$(git rev-parse HEAD)

		# TODO: implement ref_using_main

		- *mr-fetch-target-branch
		- *mr-get-target-commit
		- git checkout $target_commit

		# run ext renderer test on target branch
		# build reference binaries
		- make -j IVAS_rend
		- make -j unittests
		- make -j --directory scripts/td_object_renderer/object_renderer_standalone
		- python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py
		- mv tests/renderer/ref tests/renderer/ref_main
		- mv IVAS_rend IVAS_rend_ref

		# run ext renderer test on branch to be merged
		# back to source branch
		- git checkout $source_branch_commit_sha
		- make clean
		- make -j IVAS_rend
		- make -j unittests
		- make -j --directory scripts/td_object_renderer/object_renderer_standalone
		- python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py
		- python scripts/batch_comp_audio.py ./tests/renderer/ref_main ./tests/renderer/ref \| tee comparison_result.txt
		# TODO: implmement actual comparison

		# run test
		- exit_code=0
		- python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- *merge-request-comparison-check

		allow_failure:
		exit_codes:
		- 123
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
		when: always
		paths:
		- report-junit.xml
		expose_as: "external renderer pytest on merge request results"
		expose_as: "pytest external renderer results"
		reports:
		junit:
		- report-junit.xml

tests/renderer/constants.py

+2 −0

Original line number	Diff line number	Diff line
		@@ -40,6 +40,8 @@ CUSTOM_LAYOUT_DIR = SCRIPTS_DIR.joinpath("ls_layouts")
		HR_TRAJECTORY_DIR = SCRIPTS_DIR.joinpath("trajectories")
		TESTV_DIR = SCRIPTS_DIR.joinpath("testv")

		BIN_SUFFIX_MERGETARGET = "_ref"

		""" Encoder commandline template """
		IVAS_COD_CMD = [
		str(TESTS_DIR.parent.parent.joinpath("IVAS_cod")),

tests/renderer/test_renderer_be_comparison.py

0 → 100644

+201 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		"""
		(C) 2022 Baseline Development Group with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange,
		Panasonic Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation. All Rights Reserved.

		This software is protected by copyright law and by international treaties.
		The Baseline Development Group consisting of Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange,
		Panasonic Corporation, Qualcomm Technologies, Inc., and VoiceAge Corporation retain full ownership
		rights in their respective contributions in the software. No license of any kind, including but not
		limited to patent license, of any foregoing parties is hereby granted by implication, estoppel or
		otherwise.

		This software is provided "AS IS", without any express or implied warranties. The software is in the
		development stage. It is intended exclusively for experts who have experience with such software and
		solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
		and/or fitness for a particular purpose are hereby disclaimed and excluded.

		Any dispute, controversy or claim arising under or in relation to providing this software shall be
		submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""


		import pytest
		from .utils import *


		""" Ambisonics """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		def test_ambisonics(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt)


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt)


		@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		def test_ambisonics_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file):
		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		)


		""" Multichannel """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		def test_multichannel(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt)


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		def test_multichannel_binaural_static(test_info, in_fmt, out_fmt):
		if in_fmt in ["MONO", "STEREO"]:
		pytest.skip("MONO or STEREO to Binaural rendering unsupported")

		compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt)


		@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		def test_multichannel_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file):
		if in_fmt in ["MONO", "STEREO"]:
		pytest.skip("MONO or STEREO to Binaural rendering unsupported")

		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		)


		""" ISM """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		def test_ism(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]
		)


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		def test_ism_binaural_static(test_info, in_fmt, out_fmt):
		try:
		in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt]
		except:
		in_meta_files = None

		compare_renderer_vs_mergetarget(
		test_info, in_fmt, out_fmt, in_meta_files=in_meta_files
		)


		@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file):
		try:
		in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt]
		except:
		in_meta_files = None

		compare_renderer_vs_mergetarget(
		test_info,
		in_fmt,
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		in_meta_files=in_meta_files,
		)


		""" MASA """
		# TODO: MASA inputs not supported yet

		""" Custom loudspeaker layouts """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST)
		def test_custom_ls_input(test_info, in_layout, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt
		)


		@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS)
		def test_custom_ls_output(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info, in_fmt, CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt")
		)


		@pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST)
		def test_custom_ls_input_output(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info,
		CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"),
		CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"),
		)


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST)
		def test_custom_ls_input_binaural(test_info, in_layout, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info,
		CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"),
		out_fmt,
		)


		@pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL)
		@pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST)
		def test_custom_ls_input_binaural_headrotation(test_info, in_layout, out_fmt, trj_file):
		compare_renderer_vs_mergetarget(
		test_info,
		CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"),
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		)


		""" Metadata / scene description input """


		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS)
		@pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST)
		def test_metadata(test_info, in_fmt, out_fmt):
		compare_renderer_vs_mergetarget(
		test_info,
		"META",
		out_fmt,
		metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"),
		)

tests/renderer/utils.py

+13 −3

Original line number	Diff line number	Diff line
		@@ -151,6 +151,7 @@ def run_renderer(
		metadata_input: Optional[str] = None,
		in_meta_files: Optional[list] = None,
		trj_file: Optional[str] = None,
		binary_suffix: str = "",
		) -> Tuple[np.ndarray, int]:
		"""CuT creation with standalone renderer"""
		if trj_file is not None:
		@@ -175,6 +176,9 @@ def run_renderer(

		out_file = str(OUTPUT_PATH_CUT.joinpath(f"{in_name}_to_{out_name}{trj_name}.wav"))

		# append suffix to binary filename
		RENDERER_CMD[0] += binary_suffix

		cmd = RENDERER_CMD[:]
		cmd[2] = str(in_file)
		cmd[4] = str(in_fmt)
		@@ -259,9 +263,7 @@ def run_crend_unittest(
		in_file = FORMAT_TO_FILE[in_fmt]
		in_name = in_fmt

		out_file = str(
		OUTPUT_PATH_REF.joinpath(f"{in_name}_to_{out_name}{trj_name}.wav")
		)
		out_file = str(OUTPUT_PATH_REF.joinpath(f"{in_name}_to_{out_name}{trj_name}.wav"))

		cmd = CREND_CMD[:]
		cmd[6] = FORMAT_TO_CREND_FORMAT[str(in_fmt)]
		@@ -384,6 +386,14 @@ def run_pyscripts(
		return pyaudio3dtools.audiofile.readfile(out_file)


		def compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, **kwargs):
		ref, ref_fs = run_renderer(
		in_fmt, out_fmt, binary_suffix=BIN_SUFFIX_MERGETARGET, **kwargs
		)
		cut, cut_fs = run_renderer(in_fmt, out_fmt, **kwargs)
		check_BE(test_info, ref, ref_fs, cut, cut_fs)


		def compare_renderer_vs_pyscripts(test_info, in_fmt, out_fmt, **kwargs):
		ref, ref_fs = run_pyscripts(in_fmt, out_fmt, **kwargs)
		cut, cut_fs = run_renderer(in_fmt, out_fmt, **kwargs)