Remove xfails and issue normal fails if no diff (e101741f) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/hrtf_binary_loading/test_codec_ROM_vs_file.py

+51 −139

Original line number	Diff line number	Diff line
		@@ -30,7 +30,6 @@
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""
		import pytest
		import itertools

		from tests.hrtf_binary_loading.utils import *

		@@ -48,17 +47,10 @@ def test_binary_file(test_info, hrtf_tag, out_fs):
		""" Multichannel """


		@pytest.mark.parametrize(
		("out_fmt", "out_fs", "in_fmt", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		OUTPUT_FORMATS_BINAURAL[:-1], SAMPLE_RATE, INPUT_FORMATS_MC, HRTF_TAGS
		)
		],
		)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[:-1])
		@pytest.mark.parametrize("out_fs", SAMPLE_RATE)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, out_fs, hrtf_tag):
		bitrate = MC_BITRATE_FOR_FORMAT[in_fmt]
		in_fs = 48
		@@ -77,21 +69,11 @@ def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, out_fs, hrtf_t
		)


		@pytest.mark.parametrize(
		("trj_file", "out_fmt", "out_fs", "in_fmt", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		[HR_TRAJECTORIES_TO_TEST[0]],
		OUTPUT_FORMATS_BINAURAL[:-1],
		SAMPLE_RATE,
		INPUT_FORMATS_MC,
		HRTF_TAGS,
		)
		],
		)
		@pytest.mark.parametrize("trj_file", [HR_TRAJECTORIES_TO_TEST[0]])
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[:-1])
		@pytest.mark.parametrize("out_fs", SAMPLE_RATE)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC)
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_multichannel_binaural_headrotation(
		test_info, in_fmt, out_fmt, out_fs, trj_file, hrtf_tag
		):
		@@ -116,18 +98,12 @@ def test_multichannel_binaural_headrotation(
		""" Ambisonics """


		@pytest.mark.parametrize(
		("bitrate", "in_fmt", "fs", "out_fmt", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		[256000, 64000], INPUT_FORMATS_SBA, SAMPLE_RATE[1:], OUTPUT_FORMATS_BINAURAL[:-1], HRTF_TAGS
		)
		],
		)
		def test_sba_binaural_static(bitrate, test_info, in_fmt, fs, out_fmt, hrtf_tag):
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[:-1])
		@pytest.mark.parametrize("bitrate", [256000, 64000])
		@pytest.mark.parametrize("fs", SAMPLE_RATE[1:])
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_SBA)
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_sba_binaural_static(test_info, bitrate, in_fmt, fs, out_fmt, hrtf_tag):
		option_list = ["-sba", in_fmt]
		in_file = FORMAT_TO_FILE_SBA_WOEXT[in_fmt].format(fs)

		@@ -143,23 +119,13 @@ def test_sba_binaural_static(bitrate, test_info, in_fmt, fs, out_fmt, hrtf_tag):
		)


		@pytest.mark.parametrize(
		("bitrate", "in_fmt", "fs", "out_fmt", "trj_file", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		[256000, 64000],
		INPUT_FORMATS_SBA,
		SAMPLE_RATE[1:],
		OUTPUT_FORMATS_BINAURAL[:-1],
		[HR_TRAJECTORIES_TO_TEST[0]],
		HRTF_TAGS,
		)
		],
		)
		def test_sba_binaural_headrotation(bitrate, test_info, in_fmt, fs, out_fmt, trj_file, hrtf_tag):
		@pytest.mark.parametrize("bitrate", [256000, 64000])
		@pytest.mark.parametrize("trj_file", [HR_TRAJECTORIES_TO_TEST[0]])
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[:-1])
		@pytest.mark.parametrize("fs", SAMPLE_RATE[1:])
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_SBA)
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_sba_binaural_headrotation(test_info, bitrate, in_fmt, fs, out_fmt, trj_file, hrtf_tag):
		option_list = ["-sba", in_fmt]
		in_file = FORMAT_TO_FILE_SBA_WOEXT[in_fmt].format(fs)

		@@ -179,21 +145,11 @@ def test_sba_binaural_headrotation(bitrate, test_info, in_fmt, fs, out_fmt, trj_
		""" MASA """


		@pytest.mark.parametrize(
		("out_fmt", "fs", "in_dir", "in_tc", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		OUTPUT_FORMATS_BINAURAL[:-1],
		SAMPLE_RATE[-1:],
		INPUT_FORMATS_MASA["dir"],
		INPUT_FORMATS_MASA["tc"],
		HRTF_TAGS,
		)
		],
		)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[:-1])
		@pytest.mark.parametrize("fs", SAMPLE_RATE[-1:])
		@pytest.mark.parametrize("in_dir", INPUT_FORMATS_MASA["dir"])
		@pytest.mark.parametrize("in_tc", INPUT_FORMATS_MASA["tc"])
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_masa_binaural_static(test_info, in_tc, in_dir, fs, out_fmt, hrtf_tag):
		# TODO: remove once fixed
		if hrtf_tag == HRTF_BINARY_FILE_DIFF_FROM_ROM:
		@@ -217,22 +173,12 @@ def test_masa_binaural_static(test_info, in_tc, in_dir, fs, out_fmt, hrtf_tag):
		)


		@pytest.mark.parametrize(
		("in_tc", "in_dir", "out_fmt", "fs", "trj_file", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		INPUT_FORMATS_MASA["tc"],
		INPUT_FORMATS_MASA["dir"],
		OUTPUT_FORMATS_BINAURAL[:-1],
		SAMPLE_RATE[-1:],
		[HR_TRAJECTORIES_TO_TEST[1]],
		HRTF_TAGS,
		)
		],
		)
		@pytest.mark.parametrize("trj_file", [HR_TRAJECTORIES_TO_TEST[1]])
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[:-1])
		@pytest.mark.parametrize("fs", SAMPLE_RATE[-1:])
		@pytest.mark.parametrize("in_dir", INPUT_FORMATS_MASA["dir"])
		@pytest.mark.parametrize("in_tc", INPUT_FORMATS_MASA["tc"])
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_masa_binaural_headrotation(
		test_info, in_tc, in_dir, fs, out_fmt, trj_file, hrtf_tag
		):
		@@ -262,17 +208,10 @@ def test_masa_binaural_headrotation(
		""" ISM """


		@pytest.mark.parametrize(
		("in_fmt", "out_fs", "out_fmt", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		INPUT_FORMATS_ISM, SAMPLE_RATE, [OUTPUT_FORMATS_BINAURAL[0]], HRTF_TAGS
		)
		],
		)
		@pytest.mark.parametrize("out_fmt", [OUTPUT_FORMATS_BINAURAL[0]])
		@pytest.mark.parametrize("out_fs", SAMPLE_RATE)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_ism_binaural_static(test_info, in_fmt, out_fs, out_fmt, hrtf_tag):
		in_fs = 48
		bitrate = BITRATE_ISM[in_fmt]
		@@ -298,21 +237,11 @@ def test_ism_binaural_static(test_info, in_fmt, out_fs, out_fmt, hrtf_tag):
		)


		@pytest.mark.parametrize(
		("in_fmt", "out_fs", "out_fmt", "trj_file", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		INPUT_FORMATS_ISM,
		SAMPLE_RATE,
		[OUTPUT_FORMATS_BINAURAL[0]],
		[HR_TRAJECTORIES_TO_TEST[0]],
		HRTF_TAGS,
		)
		],
		)
		@pytest.mark.parametrize("trj_file", [HR_TRAJECTORIES_TO_TEST[0]])
		@pytest.mark.parametrize("out_fmt", [OUTPUT_FORMATS_BINAURAL[0]])
		@pytest.mark.parametrize("out_fs", SAMPLE_RATE)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_ism_binaural_headrotation(
		test_info, in_fmt, out_fs, out_fmt, trj_file, hrtf_tag
		):
		@@ -344,17 +273,10 @@ def test_ism_binaural_headrotation(
		""" ISM - Room Reverb """


		@pytest.mark.parametrize(
		("in_fmt", "out_fs", "out_fmt", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		INPUT_FORMATS_ISM, SAMPLE_RATE, OUTPUT_FORMATS_BINAURAL[-1:], HRTF_TAGS
		)
		],
		)
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[-1:])
		@pytest.mark.parametrize("out_fs", SAMPLE_RATE)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_ism_binaural_roomreverb_static(test_info, in_fmt, out_fs, out_fmt, hrtf_tag):
		in_fs = 48
		bitrate = BITRATE_ISM[in_fmt]
		@@ -380,21 +302,11 @@ def test_ism_binaural_roomreverb_static(test_info, in_fmt, out_fs, out_fmt, hrtf
		)


		@pytest.mark.parametrize(
		("in_fmt", "out_fs", "out_fmt", "trj_file", "hrtf_tag"),
		[
		pytest.param(*x, marks=pytest.mark.xfail(strict=True))
		if x[-1] == HRTF_BINARY_FILE_DIFF_FROM_ROM
		else x
		for x in itertools.product(
		INPUT_FORMATS_ISM,
		SAMPLE_RATE,
		OUTPUT_FORMATS_BINAURAL[-1:],
		[HR_TRAJECTORIES_TO_TEST[0]],
		HRTF_TAGS,
		)
		],
		)
		@pytest.mark.parametrize("trj_file", [HR_TRAJECTORIES_TO_TEST[0]])
		@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[-1:])
		@pytest.mark.parametrize("out_fs", SAMPLE_RATE)
		@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM)
		@pytest.mark.parametrize("hrtf_tag", HRTF_TAGS)
		def test_ism_binaural_roomreverb_headrotation(
		test_info, in_fmt, out_fs, out_fmt, trj_file, hrtf_tag
		):

tests/renderer/utils.py

+6 −7

Original line number	Diff line number	Diff line
		@@ -90,13 +90,12 @@ def check_BE(

		# check max_diff as well, since compare_audio_arrays will try to adjust for small delay differences
		diff_found = not np.allclose(ref, cut, rtol=0, atol=2) and max_diff > 2
		if diff_found:
		if xfail:
		pytest.xfail(f"Expected diff between CuT and REF!")
		else:
		if diff_found and not xfail:
		pytest.fail(
		f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}"
		)
		elif not diff_found and xfail:
		pytest.fail("Difference expected, but none found.")


		def run_renderer(