[tests] split rendering : add both smoke test and BE comparison, add CI jobs and fix some bugs (ef9f5155) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+76 −0

Original line number	Diff line number	Diff line
		@@ -504,6 +504,82 @@ renderer-pytest-on-merge-request:
		junit:
		- report-junit.xml

		# test split rendering
		split-rendering-smoke-test:
		extends:
		- .test-job-linux
		- .rules-merge-request
		needs: ["build-codec-linux-make"]
		stage: test
		script:
		- make -j IVAS_rend
		- python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
		expire_in: 1 week
		when: always
		paths:
		- report-junit.xml
		expose_as: "split rendering make pytest results"
		reports:
		junit:
		- report-junit.xml

		# compare split-rendering bitexactness between target and source branch
		split-rendering-pytest-on-merge-request:
		extends:
		- .test-job-linux
		- .rules-merge-request
		needs: ["build-codec-linux-make"]
		# TODO: set reasonable timeout, will most likely take less
		timeout: "20 minutes"
		stage: compare
		script:
		- *print-common-info

		# some helper variables - "\|\| true" to prevent failures from grep not finding anything
		- non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[split[ -]non[ -]*be\]") \|\| true
		# TODO: needs splitting the test between reference and cut generation
		#- ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[ref[ -]using[ -]main\]") \|\| true

		# store the current commit hash
		- source_branch_commit_sha=$(git rev-parse HEAD)

		- *mr-fetch-target-branch
		- *mr-get-target-commit
		- git checkout $target_commit

		# build reference binaries
		- make -j IVAS_rend
		- mv IVAS_rend IVAS_rend_ref

		# back to source branch
		- git checkout $source_branch_commit_sha
		- make clean
		- make -j IVAS_rend

		# run test
		- exit_code=0
		- python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering_be_comparison.py \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- *merge-request-comparison-check

		allow_failure:
		exit_codes:
		- 123
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
		expire_in: 2 week
		when: always
		paths:
		- report-junit.xml
		- report.html
		expose_as: "pytest split rendering results"
		reports:
		junit:
		- report-junit.xml

		# compare bit exactness between target and source branch
		ivas-pytest-on-merge-request:
		extends:

tests/split_rendering/constants.py

+6 −7

Original line number	Diff line number	Diff line
		@@ -26,9 +26,8 @@
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		from pathlib import Path
		import sys

		from pathlib import Path

		""" Set up paths """
		TESTS_DIR = Path(__file__).parent
		@@ -46,15 +45,15 @@ BIN_SUFFIX_MERGETARGET = "_ref"

		sys.path.append(TESTS_DIR)
		from renderer.constants import (
		CUSTOM_LS_TO_TEST,
		FORMAT_TO_FILE_COMPARETEST,
		FORMAT_TO_METADATA_FILES,
		HR_TRAJECTORIES_TO_TEST,
		INPUT_FORMATS_AMBI,
		INPUT_FORMATS_ISM,
		INPUT_FORMATS_MASA,
		INPUT_FORMATS_MC,
		CUSTOM_LS_TO_TEST,
		METADATA_SCENES_TO_TEST,
		HR_TRAJECTORIES_TO_TEST,
		)

		""" Renderer configurations """
		@@ -112,11 +111,11 @@ SPLIT_PRE_REND_CMD = [
		"-if",
		"", # 8 -> input format
		"-o",
		"BINAURAL_SPLIT_CLDFB",
		"-of",
		"", # 10 -> split rendering bitstream
		"-of",
		"BINAURAL_SPLIT_CLDFB",
		"-tf",
		"", # 12 -> post-trajectory file
		"", # 14 -> post-trajectory file
		]

		""" Split-post Renderer commandline template """

tests/split_rendering/test_split_rendering.py

+44 −37

Original line number	Diff line number	Diff line
		@@ -27,6 +27,7 @@
		"""

		import pytest

		from .utils import *

		""" Ambisonics """
		@@ -36,15 +37,15 @@ from .utils import *
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		def test_ambisonics_full_chain_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(trajectory)
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed.csv")
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_full_chain_split_args(
		test_info,
		run_full_chain_split_rendering(
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		output_path_base=OUTPUT_PATH_CUT,
		)


		@@ -52,15 +53,15 @@ def test_ambisonics_full_chain_split(test_info, in_fmt, render_config, trajector
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(trajectory)
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed.csv")
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_full_chain_split_args(
		test_info,
		run_external_split_rendering(
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		output_path_base=OUTPUT_PATH_CUT,
		)


		@@ -69,33 +70,33 @@ def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory)

		@pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC[2:])
		def test_multichannel_full_chain_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(trajectory)
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed.csv")
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_full_chain_split_args(
		test_info,
		run_full_chain_split_rendering(
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		output_path_base=OUTPUT_PATH_CUT,
		)


		@pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC[2:])
		def test_multichannel_external_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(trajectory)
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed.csv")
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_full_chain_split_args(
		test_info,
		run_external_split_rendering(
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		output_path_base=OUTPUT_PATH_CUT,
		)


		@@ -106,15 +107,15 @@ def test_multichannel_external_split(test_info, in_fmt, render_config, trajector
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		def test_ism_full_chain_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(trajectory)
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed.csv")
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_full_chain_split_args(
		test_info,
		run_full_chain_split_rendering(
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		output_path_base=OUTPUT_PATH_CUT,
		)


		@@ -122,15 +123,15 @@ def test_ism_full_chain_split(test_info, in_fmt, render_config, trajectory):
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		def test_ism_external_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(trajectory)
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed.csv")
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_full_chain_split_args(
		test_info,
		run_external_split_rendering(
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		output_path_base=OUTPUT_PATH_CUT,
		)


		@@ -142,12 +143,15 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory):
		# @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		# @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA)
		# def test_masa_full_chain_split(test_info, in_fmt, render_config, trajectory):
		# trajectory = HR_TRAJECTORY_DIR.joinpath(trajectory)
		# pre_trajectory = trajectory.with_stem(f"{trajectory.stem}_delayed.csv")
		# post_trajectory = trajectory
		# post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		# pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")
		#
		# compare_full_chain_split_args(
		# test_info, in_fmt, render_config, pre_trajectory, post_trajectory
		# run_full_chain_split_rendering(
		# in_fmt=in_fmt,
		# render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		# pre_trajectory=pre_trajectory,
		# post_trajectory=post_trajectory,
		# output_path_base=OUTPUT_PATH_CUT,
		# )
		#
		#
		@@ -155,10 +159,13 @@ def test_ism_external_split(test_info, in_fmt, render_config, trajectory):
		# @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		# @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA)
		# def test_masa_external_split(test_info, in_fmt, render_config, trajectory):
		# trajectory = HR_TRAJECTORY_DIR.joinpath(trajectory)
		# pre_trajectory = trajectory.with_stem(f"{trajectory.stem}_delayed.csv")
		# post_trajectory = trajectory
		# post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		# pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")
		#
		# compare_external_split_args(
		# test_info, in_fmt, render_config, pre_trajectory, post_trajectory
		# run_external_split_rendering(
		# in_fmt=in_fmt,
		# render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		# pre_trajectory=pre_trajectory,
		# post_trajectory=post_trajectory,
		# output_path_base=OUTPUT_PATH_CUT,
		# )
		No newline at end of file

tests/split_rendering/test_split_rendering_be_comparison.py

0 → 100644

+171 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		"""
		(C) 2022-2023 Baseline Development Group with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange,
		Panasonic Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation. All Rights Reserved.

		This software is protected by copyright law and by international treaties.
		The Baseline Development Group consisting of Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange,
		Panasonic Corporation, Qualcomm Technologies, Inc., and VoiceAge Corporation retain full ownership
		rights in their respective contributions in the software. No license of any kind, including but not
		limited to patent license, of any foregoing parties is hereby granted by implication, estoppel or
		otherwise.

		This software is provided "AS IS", without any express or implied warranties. The software is in the
		development stage. It is intended exclusively for experts who have experience with such software and
		solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
		and/or fitness for a particular purpose are hereby disclaimed and excluded.

		Any dispute, controversy or claim arising under or in relation to providing this software shall be
		submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		import pytest

		from .utils import *

		""" Ambisonics """


		@pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		def test_ambisonics_full_chain_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_full_chain_split_args(
		test_info,
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		)


		@pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI)
		def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_external_split_args(
		test_info,
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		)


		""" Multichannel """


		@pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC[2:])
		def test_multichannel_full_chain_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_full_chain_split_args(
		test_info,
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		)


		@pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC[2:])
		def test_multichannel_external_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_external_split_args(
		test_info,
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		)


		""" ISM """


		@pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		def test_ism_full_chain_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_full_chain_split_args(
		test_info,
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		)


		@pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		def test_ism_external_split(test_info, in_fmt, render_config, trajectory):
		post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")

		compare_external_split_args(
		test_info,
		in_fmt=in_fmt,
		render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		pre_trajectory=pre_trajectory,
		post_trajectory=post_trajectory,
		)


		""" MASA """
		# TODO


		# @pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		# @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		# @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA)
		# def test_masa_full_chain_split(test_info, in_fmt, render_config, trajectory):
		# post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		# pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")
		#
		# compare_full_chain_split_args(
		# test_info,
		# in_fmt=in_fmt,
		# render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		# pre_trajectory=pre_trajectory,
		# post_trajectory=post_trajectory,
		# )
		#
		#
		# @pytest.mark.parametrize("trajectory", HR_TRAJECTORIES_TO_TEST)
		# @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST)
		# @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA)
		# def test_masa_external_split(test_info, in_fmt, render_config, trajectory):
		# post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv")
		# pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")
		#
		# compare_external_split_args(
		# test_info,
		# in_fmt=in_fmt,
		# render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"),
		# pre_trajectory=pre_trajectory,
		# post_trajectory=post_trajectory,
		# )
		No newline at end of file

tests/split_rendering/utils.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -61,7 +61,7 @@ def run_full_chain_split_rendering(
		ivas_bitstream = tmp_dir.joinpath("ivas.192")
		split_bitstream = tmp_dir.joinpath("split.bit")
		out_file = output_path_base.joinpath(
		f"{in_fmt}_{pre_trajectory.stem}_to_split_binaural_{post_trajectory.stem}__config_{render_config.stem}.wav"
		f"{in_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}__config_{render_config.stem}.wav"
		)

		# check for metadata files
		@@ -124,7 +124,7 @@ def run_external_split_rendering(
		tmp_dir = Path(tmp_dir)
		split_bitstream = tmp_dir.joinpath("split.bit")
		out_file = output_path_base.joinpath(
		f"{in_fmt}_{pre_trajectory}_to_split_binaural_{post_trajectory}__config_{render_config.name}.wav"
		f"{in_fmt}_{pre_trajectory.stem}_split_ext_{post_trajectory.stem}__config_{render_config.stem}.wav"
		)

		# check for metadata files
		@@ -140,7 +140,7 @@ def run_external_split_rendering(
		cmd[6] = str(FORMAT_TO_FILE_COMPARETEST[in_fmt])
		cmd[8] = in_fmt
		cmd[10] = str(split_bitstream)
		cmd[12] = str(pre_trajectory)
		cmd[14] = str(pre_trajectory)

		if in_meta_files:
		cmd[9:9] = ["-im", *in_meta_files]