Merge branch 'ci/peaq-for-binauralized-output-2' into 'main' (73f76817) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+1 −1

Original line number	Diff line number	Diff line
		@@ -274,6 +274,7 @@ stages:
		- if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'check-clipping'
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'
		when: never
		- when: on_success
		@@ -1750,7 +1751,6 @@ test-branch-vs-input-passthrough:
		junit:
		- report-junit.xml


		# ---------------------------------------------------------------
		# Scheduled jobs on main
		# ---------------------------------------------------------------

scripts/diff_report.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -48,7 +48,7 @@ def main(args):
		df_test = pd.read_csv(args.csv_test, sep=";")

		for col in COLUMNS_TO_COMPARE:
		df_ref[col] = df_ref[col] - df_test[col]
		df_ref[col] = df_test[col] - df_ref[col]
		df_ref.to_csv(args.csv_diff, index=False, sep=";")
		return 0

scripts/parse_xml_report.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -15,7 +15,7 @@ PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"]
		IVAS_FORMATS = {
		"Stereo": r"stereo",
		"ISM": r"ISM",
		"Multichannel": r"Multi-channel",
		"Multichannel": r"Multi-channel\|MC",
		"MASA": r"(?<!O)MASA",
		"SBA": r"(?<!O)SBA",
		"OSBA": r"OSBA",

tests/cmp_pcm.py

+35 −10

Original line number	Diff line number	Diff line
		@@ -33,6 +33,9 @@ def cmp_pcm(
		abs_tol=0,
		get_ssnr=False,
		get_odg=False,
		odg_input=None,
		odg_test=None,
		odg_ref=None,
		ref_jbm_tf: Optional[Path] = None,
		cut_jbm_tf: Optional[Path] = None,
		) -> (int, str):
		@@ -53,15 +56,20 @@ def cmp_pcm(
		else:
		nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()]

		s1, _ = pyaudio3dtools.audiofile.readfile(
		s1, fs1 = pyaudio3dtools.audiofile.readfile(
		ref_file, nchannels, fs, outdtype=np.int16
		)
		s2, _ = pyaudio3dtools.audiofile.readfile(
		s2, fs2 = pyaudio3dtools.audiofile.readfile(
		cmp_file, nchannels, fs, outdtype=np.int16
		)

		# In case of wav input, override the nchannels with the one from the wav header
		nchannels = s1.shape[1]
		fs = fs1

		if fs1 != fs2:
		reason = "FAIL: Sampling rate differs."
		return 1, reason

		# In case number of channels do not match, fail already now. Could happen in case of
		# comparison to input with for a non-passthrough mode.
		@@ -129,12 +137,29 @@ def cmp_pcm(
		reason += msg + " - "

		if get_odg:
		for n in range(nchannels):
		pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs)
		odg_files = {}
		for f in [odg_input, odg_test, odg_ref]:
		# Load PEAQ test files and ensure 48 kHz sampling rate
		s, fs = pyaudio3dtools.audiofile.readfile(
		f, nchannels, fs, outdtype=np.int16
		)
		odg_files[f] = np.clip(
		pyaudio3dtools.audioarray.resample(s.astype(float), fs, 48000),
		-32768,
		32767,
		).astype(np.int16)

		pqeval_output = pqevalaudio_wrapper(odg_files[odg_input], odg_files[odg_ref], 48000)
		match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output)
		odg_ref = float(match_odg.groups()[0])

		pqeval_output = pqevalaudio_wrapper(odg_files[odg_input], odg_files[odg_test], 48000)
		match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output)
		odg = float(match_odg.groups()[0])
		msg = f"Channel {n} ODG: {odg}"
		odg_test = float(match_odg.groups()[0])

		odg = odg_test - odg_ref # Todo: store both rather than difference?

		msg = f"ODG: {odg}"
		reason += " - " + msg
		print(msg)

tests/codec_be_on_mr_nonselection/test_param_file.py

+154 −10

Original line number	Diff line number	Diff line
		@@ -32,7 +32,6 @@ __doc__ = """
		Execute tests specified via a parameter file.
		"""

		import errno
		import filecmp
		import os
		import platform
		@@ -52,6 +51,7 @@ from tests.constants import (
		SCRIPTS_DIR,
		MAX_ENC_DIFF,
		)
		from tests.renderer.utils import check_and_makedir, binauralize_input_and_output

		PASSTHROUGH_CONF = [
		(r"[\s\S]*", "EXT"), # Any input with EXT output is pass-through
		@@ -82,6 +82,45 @@ VALID_DEC_OUTPUT_CONF = [
		"EXT",
		]

		INPUT_FMT = [
		(r"-stereo", "STEREO"),
		(r"-sba\s?[-+]?1", "FOA"),
		(r"-sba\s?[-+]?2", "HOA2"),
		(r"-sba\s?[-+]?3", "HOA3"),
		(r"-mc\s5_1\s", "5_1"),
		(r"-mc\s7_1\s", "7_1"),
		(r"-mc\s5_1_2", "5_1_2"),
		(r"-mc\s5_1_4", "5_1_4"),
		(r"-mc\s7_1_4", "7_1_4"),
		(r"-ism\s?[-+]?1", "ISM1"),
		(r"-ism\s?[-+]?2", "ISM2"),
		(r"-ism\s?[-+]?3", "ISM3"),
		(r"-ism\s?[-+]?4", "ISM4"),
		(r"-masa\s?1", "MASA1"),
		(r"-masa\s?2", "MASA2"),
		(r"-ism_masa\s?1\s?1", "OMASA_1_1"),
		(r"-ism_masa\s?2\s?1", "OMASA_2_1"),
		(r"-ism_masa\s?3\s?1", "OMASA_3_1"),
		(r"-ism_masa\s?4\s?1", "OMASA_4_1"),
		(r"-ism_masa\s?1\s?2", "OMASA_1_2"),
		(r"-ism_masa\s?2\s?2", "OMASA_2_2"),
		(r"-ism_masa\s?3\s?2", "OMASA_3_2"),
		(r"-ism_masa\s?4\s?2", "OMASA_4_2"),
		(r"-ism_sba\s?1\s?[-+]?1", "OSBA_1_1"),
		(r"-ism_sba\s?2\s?[-+]?1", "OSBA_2_1"),
		(r"-ism_sba\s?3\s?[-+]?1", "OSBA_3_1"),
		(r"-ism_sba\s?4\s?[-+]?1", "OSBA_4_1"),
		(r"-ism_sba\s?1\s?[-+]?2", "OSBA_1_2"),
		(r"-ism_sba\s?2\s?[-+]?2", "OSBA_2_2"),
		(r"-ism_sba\s?3\s?[-+]?2", "OSBA_3_2"),
		(r"-ism_sba\s?4\s?[-+]?2", "OSBA_4_2"),
		(r"-ism_sba\s?1\s?[-+]?3", "OSBA_1_3"),
		(r"-ism_sba\s?2\s?[-+]?3", "OSBA_2_3"),
		(r"-ism_sba\s?3\s?[-+]?3", "OSBA_3_3"),
		(r"-ism_sba\s?4\s?[-+]?3", "OSBA_4_3"),
		(r".*", "MONO"),
		]

		PARAM_FILE_ID = (
		"stv"
		if PARAM_FILE.stem == "self_test"
		@@ -118,15 +157,6 @@ with open(PARAM_FILE, "r", encoding="UTF-8") as fp:
		param_file_test_dict[tag] = (enc_opts, dec_opts, sim_opts, eid_opts)


		def check_and_makedir(dir_path):
		if not os.path.exists(dir_path):
		try:
		os.makedirs(dir_path)
		except OSError as e:
		if e.errno != errno.EEXIST:
		raise # raises the error again


		def convert_test_string_to_tag(test_string):
		"""
		Convert a test string (i.e. the test tag from the parameter file) to a tag string.
		@@ -152,6 +182,7 @@ def convert_test_string_to_tag(test_string):
		# hack to have stv/ltv/evs in the test name
		@pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID])
		def test_param_file_tests(
		test_info,
		props_to_record,
		encoder_only,
		decoder_only,
		@@ -175,8 +206,68 @@ def test_param_file_tests(
		get_odg,
		compare_to_input,
		):

		enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]

		run_test(
		test_info,
		props_to_record,
		encoder_only,
		decoder_only,
		dut_encoder_frontend,
		dut_decoder_frontend,
		ref_encoder_frontend,
		ref_decoder_frontend,
		enc_opts,
		dec_opts,
		sim_opts,
		eid_opts,
		reference_path,
		dut_base_path,
		test_vector_path,
		update_ref,
		rootdir,
		keep_files,
		test_tag,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_enc_stats,
		get_odg,
		compare_to_input,
		)


		def run_test(
		test_info,
		props_to_record,
		encoder_only,
		decoder_only,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		ref_encoder_frontend: EncoderFrontend,
		ref_decoder_frontend: DecoderFrontend,
		enc_opts,
		dec_opts,
		sim_opts,
		eid_opts,
		reference_path,
		dut_base_path,
		test_vector_path,
		update_ref,
		rootdir,
		keep_files,
		test_tag,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_enc_stats,
		get_odg,
		compare_to_input,
		):

		# If compare_to_input is set, only run pass-through test cases
		if compare_to_input:
		passthrough = [
		@@ -206,6 +297,7 @@ def test_param_file_tests(
		fs = enc_split.pop()
		sampling_rate = int(fs)
		bitrate = enc_split.pop()
		in_sr = sampling_rate

		# bitrate can be a filename: remove leading "../"
		if bitrate.startswith("../"):
		@@ -349,6 +441,7 @@ def test_param_file_tests(
		output_file = dec_split.pop()
		bitstream_file_dec = dec_split.pop()
		sampling_rate = int(dec_split.pop())
		out_sr = sampling_rate
		if len(dec_split) > 0:
		output_config = dec_split.pop()
		if output_config.upper() not in VALID_DEC_OUTPUT_CONF:
		@@ -396,9 +489,57 @@ def test_param_file_tests(
		)

		if update_ref in [0, 2]:

		# Output file names for comparison
		dut_output_file = f"{dut_base_path}/param_file/dec/{output_file}"
		ref_output_file = f"{reference_path}/param_file/dec/{output_file}"

		# Check if we need to render to mono, stereo or binaural for PEAQ comparison
		odg_input = None
		odg_test = None
		odg_ref = None
		if get_odg:

		# Find input format
		in_fmt = [(a, b) for (a, b) in INPUT_FMT if re.search(a, enc_opts)][0][1]

		record_property = (
		dut_decoder_frontend.record_property
		if dut_decoder_frontend
		else ref_decoder_frontend.record_property
		)

		(odg_input, odg_test) = binauralize_input_and_output(
		record_property,
		props_to_record,
		test_info,
		testv_file,
		dut_output_file,
		ref_output_file,
		in_fmt,
		output_config,
		enc_opts,
		dec_opts,
		in_sr,
		out_sr,
		)

		(_ , odg_ref) = binauralize_input_and_output(
		record_property,
		props_to_record,
		test_info,
		testv_file,
		ref_output_file,
		ref_output_file,
		in_fmt,
		output_config,
		enc_opts,
		dec_opts,
		in_sr,
		out_sr,
		skip_input_rendering=True,
		)

		# set to false per default even if this is no JBM case - makes later check for failure easier
		tracefile_last_rtp_numbers_differ = False

		@@ -443,6 +584,9 @@ def test_param_file_tests(
		allow_differing_lengths=allow_differing_lengths,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		odg_input=odg_input,
		odg_test=odg_test,
		odg_ref=odg_ref,
		ref_jbm_tf=ref_tracefile_dec,
		cut_jbm_tf=dut_tracefile_dec,
		)