enable split comparison for renderer tests (93d77e5f) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/renderer/constants.py

+96 −24

Original line number	Diff line number	Diff line
		@@ -298,6 +298,77 @@ FORMAT_TO_METADATA_FILES = {
		],
		}

		FORMAT_TO_METADATA_FILES_LTV = {
		"ISM1": [str(TESTV_DIR.joinpath("ltvISM1.csv"))],
		"ISM2": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltvISM2.csv")),
		],
		"ISM3": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltvISM2.csv")),
		str(TESTV_DIR.joinpath("ltvISM3.csv")),
		],
		"ISM4": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltvISM2.csv")),
		str(TESTV_DIR.joinpath("ltvISM3.csv")),
		str(TESTV_DIR.joinpath("ltvISM4.csv")),
		],
		"NDP_ISM4": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("stvISM2_non-diegetic-pan.csv")),
		str(TESTV_DIR.joinpath("ltvISM3.csv")),
		str(TESTV_DIR.joinpath("ltvISM4.csv")),
		],
		"MASA1": [str(TESTV_DIR.joinpath("ltv48_MASA1TC.met"))],
		"MASA2": [str(TESTV_DIR.joinpath("ltv48_MASA2TC.met"))],
		"OMASA_1_1": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_1TC.met")),
		],
		"OMASA_1_2": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltvISM2.csv")),
		str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_1TC.met")),
		],
		"OMASA_1_3": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltvISM2.csv")),
		str(TESTV_DIR.joinpath("ltvISM3.csv")),
		str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_1TC.met")),
		],
		"OMASA_1_4": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltvISM2.csv")),
		str(TESTV_DIR.joinpath("ltvISM3.csv")),
		str(TESTV_DIR.joinpath("ltvISM4.csv")),
		str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_1TC.met")),
		],
		"OMASA_2_1": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltv48_OMASA_1ISM_2TC.met")),
		],
		"OMASA_2_2": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltvISM2.csv")),
		str(TESTV_DIR.joinpath("ltv48_OMASA_2ISM_2TC.met")),
		],
		"OMASA_2_3": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltvISM2.csv")),
		str(TESTV_DIR.joinpath("ltvISM3.csv")),
		str(TESTV_DIR.joinpath("ltv48_OMASA_3ISM_2TC.met")),
		],
		"OMASA_2_4": [
		str(TESTV_DIR.joinpath("ltvISM1.csv")),
		str(TESTV_DIR.joinpath("ltvISM2.csv")),
		str(TESTV_DIR.joinpath("ltvISM3.csv")),
		str(TESTV_DIR.joinpath("ltvISM4.csv")),
		str(TESTV_DIR.joinpath("ltv48_OMASA_4ISM_2TC.met")),
		],
		}


		""" Input formats """
		INPUT_FORMATS_AMBI = ["FOA", "HOA2", "HOA3"]
		@@ -348,3 +419,4 @@ PEAQ_SUPPORTED_FMT = [
		"BINAURAL_ROOM_IR",
		"BINAURAL_ROOM_REVERB",
		]

tests/renderer/test_renderer.py

+42 −7

Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@ the United Nations Convention on Contracts on the International Sales of Goods.
		import pytest

		from .constants import (
		FORMAT_TO_METADATA_FILES_LTV,
		OUTPUT_FORMATS,
		INPUT_FORMATS_AMBI,
		FRAMING_TO_TEST,
		@@ -385,13 +386,18 @@ def test_ism(
		get_odg_bin,
		split_comparison,
		):
		md_files = (
		FORMAT_TO_METADATA_FILES_LTV[in_fmt]
		if test_info.config.option.use_ltv
		else FORMAT_TO_METADATA_FILES[in_fmt]
		)
		run_renderer(
		record_property,
		props_to_record,
		test_info,
		in_fmt,
		out_fmt,
		in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt],
		in_meta_files=md_files,
		binary_suffix=EXE_SUFFIX,
		frame_size=frame_size,
		get_mld=get_mld,
		@@ -421,7 +427,11 @@ def test_ism_binaural_static(
		split_comparison,
		):
		try:
		in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt]
		in_meta_files = (
		FORMAT_TO_METADATA_FILES_LTV[in_fmt]
		if test_info.config.option.use_ltv
		else FORMAT_TO_METADATA_FILES[in_fmt]
		)
		except KeyError:
		in_meta_files = None

		@@ -463,7 +473,11 @@ def test_ism_binaural_headrotation(
		split_comparison,
		):
		try:
		in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt]
		in_meta_files = (
		FORMAT_TO_METADATA_FILES_LTV[in_fmt]
		if test_info.config.option.use_ltv
		else FORMAT_TO_METADATA_FILES[in_fmt]
		)
		except KeyError:
		in_meta_files = None

		@@ -506,13 +520,19 @@ def test_masa(
		get_odg_bin,
		split_comparison,
		):
		md_files = (
		FORMAT_TO_METADATA_FILES_LTV[in_fmt]
		if test_info.config.option.use_ltv
		else FORMAT_TO_METADATA_FILES[in_fmt]
		)

		run_renderer(
		record_property,
		props_to_record,
		test_info,
		in_fmt,
		out_fmt,
		in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt],
		in_meta_files=md_files,
		binary_suffix=EXE_SUFFIX,
		frame_size=frame_size,
		get_mld=get_mld,
		@@ -544,13 +564,18 @@ def test_masa_binaural_static(
		if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]:
		pytest.skip("Skipping binaural room outputs for MASA as unimplemented.")

		md_files = (
		FORMAT_TO_METADATA_FILES_LTV[in_fmt]
		if test_info.config.option.use_ltv
		else FORMAT_TO_METADATA_FILES[in_fmt]
		)
		run_renderer(
		record_property,
		props_to_record,
		test_info,
		in_fmt,
		out_fmt,
		in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt],
		in_meta_files=md_files,
		binary_suffix=EXE_SUFFIX,
		frame_size=frame_size,
		get_mld=get_mld,
		@@ -584,6 +609,12 @@ def test_masa_binaural_headrotation(
		if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]:
		pytest.skip("Skipping binaural room outputs for MASA as unimplemented.")

		md_files = (
		FORMAT_TO_METADATA_FILES_LTV[in_fmt]
		if test_info.config.option.use_ltv
		else FORMAT_TO_METADATA_FILES[in_fmt]
		)

		run_renderer(
		record_property,
		props_to_record,
		@@ -591,7 +622,7 @@ def test_masa_binaural_headrotation(
		in_fmt,
		out_fmt,
		trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"),
		in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt],
		in_meta_files=md_files,
		binary_suffix=EXE_SUFFIX,
		frame_size=frame_size,
		get_mld=get_mld,
		@@ -1270,7 +1301,11 @@ def test_ism_binaural_headrotation_refvec_rotating(
		pytest.skip("OTR tests only run for smoke test")

		try:
		in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt]
		in_meta_files = (
		FORMAT_TO_METADATA_FILES_LTV[in_fmt]
		if test_info.config.option.use_ltv
		else FORMAT_TO_METADATA_FILES[in_fmt]
		)
		except KeyError:
		in_meta_files = None

tests/renderer/utils.py

+56 −17

Original line number	Diff line number	Diff line
		@@ -62,7 +62,7 @@ from ..constants import CAT_NORMAL
		sys.path.append(SCRIPTS_DIR)
		from pyaudio3dtools.audiofile import readfile
		from ..cmp_pcm import cmp_pcm
		from ..conftest import parse_properties
		from ..conftest import parse_properties, get_split_idx


		def run_cmd(cmd, test_info, env=None):
		@@ -233,7 +233,8 @@ def run_renderer(
		# if in REF or CUT creation mode use the comparetestv
		if test_info.config.option.create_ref or test_info.config.option.create_cut:
		FORMAT_TO_FILE = FORMAT_TO_FILE_COMPARETEST
		elif test_info.config.option.use_ltv:

		if test_info.config.option.use_ltv:
		if test_info.config.option.ltv_dir:
		FORMAT_TO_FILE = dict()
		for k, v in FORMAT_TO_FILE_LTV.items():
		@@ -310,6 +311,14 @@ def run_renderer(
		env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_info.node.name}"
		)

		testcase_props = {
		"format": "Renderer",
		"category": CAT_NORMAL,
		}

		for k, v in testcase_props.items():
		record_property(k, v)

		# run the renderer
		run_cmd(cmd, test_info, env)

		@@ -373,9 +382,13 @@ def run_renderer(
		else:
		odg_input = in_file

		# see constants.py
		### run the comparison tools
		split_idx = np.empty(0)
		prop_suffix = [""]

		# 1. run comparison on whole files - this is done always, regardless of the presence of --split_comparison
		ref_fs = int(cmd[10]) * 1000
		output_differs, reason = cmp_pcm(
		output_differs_parts, reason_parts = cmp_pcm(
		out_file,
		out_file_ref,
		out_fmt,
		@@ -390,24 +403,50 @@ def run_renderer(
		odg_test=odg_test,
		odg_ref=odg_ref,
		scalefac=test_info.config.option.scalefac,
		split_idx=split_idx,
		)
		output_differs = output_differs[0]
		reason = reason[0]

		# splitting is not implemented for renderer tests yet
		# if arg is given, need to record as "whole" so that XML parsing works
		suffix = ""
		# 2. run comparison on split files if --split_comparison is given
		# for JBM cases, comparison will fail because of length mismatch beetween split wav files and tracefiles
		# -> skip split comparison for these cases
		if split_comparison:
		suffix = "_whole"
		split_idx = get_split_idx(str(Path(in_file).stem), ref_fs // 1000)

		props = parse_properties(reason, output_differs, props_to_record, suffix)
		props["format"] = "Renderer"
		props["category"] = CAT_NORMAL
		for k, v in props.items():
		output_differs_splits, reason_splits = cmp_pcm(
		out_file,
		out_file_ref,
		out_fmt,
		ref_fs,
		get_mld=get_mld,
		mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		get_odg_bin=get_odg_bin,
		odg_input=odg_input,
		odg_test=odg_test,
		odg_ref=odg_ref,
		scalefac=test_info.config.option.scalefac,
		split_idx=split_idx,
		)
		output_differs_parts += output_differs_splits
		reason_parts += reason_splits

		prop_suffix = ["_whole"] + [
		f"_split{i:03d}" for i in range(1, len(split_idx) + 1)
		]

		for output_differs, reason, suffix in zip(
		output_differs_parts, reason_parts, prop_suffix
		):
		result_props = parse_properties(
		reason, output_differs, props_to_record, suffix
		)
		for k, v in result_props.items():
		record_property(k, v)

		if output_differs:
		pytest.fail(f"Output differs: ({reason})")
		if output_differs_parts[0]:
		pytest.fail(f"Output differs: ({reason_parts[0]})")

		# compare metadata files in case of MASA prerendering
		if "MASA" in str(out_fmt):