added param_file test + renamed data_system_tests_path test_vector_path (00337adc) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/conftest.py

+34 −30

Original line number	Diff line number	Diff line
		@@ -58,51 +58,57 @@ def rootdir(request):


		def pytest_addoption(parser):
		parser.addoption("--update_ref", action="store", default="0")
		parser.addoption("--p4_CL", action="store")
		parser.addoption("--p4cmd_active", action="store", default="0")
		parser.addoption(
		"--update_ref",
		action="store",
		help="""Indicate whether references shall be updated.
		0: Only DUT processing, no reference generation, references need to be present.
		1: Only reference generation (unconditionally), no DUT processing.
		2: DUT processing, references are generated when not present (not supported by all tests).
		""",
		default="0",
		)

		parser.addoption(
		"--dut_encoder_path",
		action="store",
		help="If specified, use given binary as DUT encoder."
		help="If specified, use given binary as DUT encoder.",
		)

		parser.addoption(
		"--dut_decoder_path",
		action="store",
		help="If specified, use given binary as DUT decoder."
		help="If specified, use given binary as DUT decoder.",
		)

		parser.addoption(
		"--ref_encoder_path",
		action="store",
		help="If specified, use given binary as REF encoder."
		help="If specified, use given binary as REF encoder.",
		)

		parser.addoption(
		"--ref_decoder_path",
		action="store",
		help="If specified, use given binary as REF decoder."
		help="If specified, use given binary as REF decoder.",
		)

		# TODO: rename to test_vector_path
		parser.addoption(
		"--data_system_tests_path",
		"--test_vector_path",
		action="store",
		help="If specified, use given directory as base data directory for system tests."
		help="If specified, use given directory as base directory for test vector files.",
		)

		parser.addoption(
		"--reference_path",
		action="store",
		help="If specified, use given directory as base directory for reference files."
		help="If specified, use given directory as base directory for reference files.",
		)

		parser.addoption(
		"--dut_base_path",
		action="store",
		help="If specified, use given directory as base data directory for dut files."
		help="If specified, use given directory as base data directory for dut files.",
		)


		@@ -111,16 +117,6 @@ def update_ref(request):
		return int(request.config.getoption("--update_ref"))


		@pytest.fixture(scope="session", autouse=True)
		def p4_CL(request):
		return request.config.option.p4_CL


		@pytest.fixture(scope="session", autouse=True)
		def p4cmd_active(request):
		return int(request.config.getoption("--p4cmd_active"))


		@pytest.fixture(scope="session")
		def dut_encoder_path(request) -> Path:
		if request.config.option.dut_encoder_path:
		@@ -163,6 +159,7 @@ class EncoderFrontend:
		agc_op: Optional[int] = None,
		bypass_mode: Optional[int] = None,
		quiet_mode: Optional[bool] = True,
		add_option_list: Optional[list] = None,
		) -> None:
		command = [self._path]

		@@ -185,6 +182,9 @@ class EncoderFrontend:
		if quiet_mode:
		command.extend(["-q"])

		if add_option_list is not None:
		command.extend(add_option_list)

		# add mandatory parameters
		command += [
		str(bitrate),
		@@ -233,8 +233,7 @@ def ref_encoder_path(request) -> Path:
		if request.config.option.ref_encoder_path:
		return request.config.option.ref_encoder_path

		update_ref = int(request.config.getoption("--update_ref"))
		if not update_ref:
		if request.config.option.update_ref == "0":
		return None

		# assume default encoder when update_ref is selected, but no ref_encoder_path is specified
		@@ -293,6 +292,7 @@ class DecoderFrontend:
		output_path: Path,
		quiet_mode: Optional[bool] = True,
		plc_file: Optional[Path] = None,
		add_option_list: Optional[list] = None,
		) -> None:
		command = [self._path]

		@@ -303,9 +303,14 @@ class DecoderFrontend:
		if plc_file is not None:
		command.extend(["-fec", str(plc_file)])

		if add_option_list is not None:
		command.extend(add_option_list)

		# add mandatory parameters
		# output_config is mandatory for IVAS; EVS does not have this parameter, indicated by ""
		if output_config is not "":
		command += [output_config]
		command += [
		output_config,
		str(output_sampling_rate),
		str(input_bitstream_path),
		str(output_path),
		@@ -351,8 +356,7 @@ def ref_decoder_path(request) -> Path:
		if request.config.option.ref_decoder_path:
		return request.config.option.ref_decoder_path

		update_ref = int(request.config.getoption("--update_ref"))
		if not update_ref:
		if request.config.option.update_ref == "0":
		return None

		# assume default decoder when update_ref is selected, but no ref_decoder_path is specified
		@@ -374,9 +378,9 @@ def ref_decoder_path(request) -> Path:


		@pytest.fixture(scope="session")
		def data_system_tests_path(request) -> Path:
		if request.config.option.data_system_tests_path:
		return request.config.option.data_system_tests_path
		def test_vector_path(request) -> Path:
		if request.config.option.test_vector_path:
		return request.config.option.test_vector_path

		here = Path(__file__).parent.resolve()

tests/self_test_b.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -208,7 +208,7 @@ def main(argv):
		"--update_ref",
		"1",
		"-v",
		"--data_system_tests_path",
		"--test_vector_path",
		TEST_VECTOR_DIR,
		"--reference_path",
		REFERENCE_DIR,
		@@ -250,7 +250,7 @@ def main(argv):
		"-n",
		args.numprocesses,
		"-v",
		"--data_system_tests_path",
		"--test_vector_path",
		TEST_VECTOR_DIR,
		"--reference_path",
		REFERENCE_DIR,

tests/test_param_file.py

0 → 100644

+321 −0

Original line number	Diff line number	Diff line
		"""
		Execute tests specified via a parameter file.
		"""

		__copyright__ = """
		(C) 2022 Baseline Development Group with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange,
		Panasonic Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation. All Rights Reserved.

		This software is protected by copyright law and by international treaties.
		The Baseline Development Group consisting of Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies OY, Orange,
		Panasonic Corporation, Qualcomm Technologies, Inc., and VoiceAge Corporation retain full ownership
		rights in their respective contributions in the software. No license of any kind, including but not
		limited to patent license, of any foregoing parties is hereby granted by implication, estoppel or
		otherwise.

		This software is provided "AS IS", without any express or implied warranties. The software is in the
		development stage. It is intended exclusively for experts who have experience with such software and
		solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
		and/or fitness for a particular purpose are hereby disclaimed and excluded.

		Any dispute, controversy or claim arising under or in relation to providing this software shall be
		submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		import os
		import errno
		import pytest
		from cmp_custom import cmp_custom
		from conftest import EncoderFrontend, DecoderFrontend

		PARAM_FILE = "scripts/config/self_test.prm"

		param_file_test_dict = {}
		with open(PARAM_FILE, "r", encoding="UTF-8") as fp:
		data = fp.read()
		blocks = data.split("\n\n")
		for block in blocks:
		tag = ""
		enc_opts = ""
		dec_opts = ""
		for line in block.split("\n"):
		if line.startswith("// "):
		tag = line[3:]
		if line.startswith("../IVAS_cod "):
		enc_opts = line[12:]
		if line.startswith("../IVAS_dec "):
		dec_opts = line[12:]
		if tag == "" or enc_opts == "" or dec_opts == "":
		# no complete parameter set
		continue
		if tag in param_file_test_dict:
		print("non-unique tag found - ignoring new entry")
		continue
		param_file_test_dict[tag] = (enc_opts, dec_opts)


		def check_and_makedir(dir_path):
		if not os.path.exists(dir_path):
		try:
		os.makedirs(dir_path)
		except OSError as e:
		if e.errno != errno.EEXIST:
		raise # raises the error again


		def convert_test_string_to_tag(test_string):
		"""
		Convert a test string (i.e. the test tag from the parameter file) to a tag string.
		Example:
		in: "DFT stereo at 13.2 kbps, 16kHz in, 16kHz out, DTX on, random FEC at 5%"
		out: "DFT_stereo_at_13_2_kbps_16kHz_in_16kHz_out_DTX_on_random_FEC_at_5_"
		"""
		# replace certain characters by "_" or remove them
		tag_str = ""
		replace_chars = " %.-()"
		remove_chars = ","
		for char in test_string:
		if char in replace_chars:
		tag_str += "_"
		elif char not in remove_chars:
		tag_str += char
		# replace double underscore by single one
		tag_str = "_".join(tag_str.split("__"))
		return tag_str


		@pytest.mark.create_ref
		@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys()))
		def test_param_file_tests(
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		ref_encoder_path,
		ref_decoder_path,
		reference_path,
		dut_base_path,
		test_vector_path,
		update_ref,
		test_tag,
		):
		enc_opts, dec_opts = param_file_test_dict[test_tag]

		tag_str = convert_test_string_to_tag(test_tag)

		# evaluate encoder options
		enc_split = enc_opts.split()
		assert len(enc_split) >= 4

		# replace "testv/" by extended test vector path
		enc_split = [
		x.replace("testv", f"{test_vector_path}/param_file", 1) if x.startswith("testv/") else x
		for x in enc_split
		]

		bitstream_file = enc_split.pop()
		testv_file = enc_split.pop()
		sampling_rate = int(enc_split.pop())
		bitrate = enc_split.pop()

		# bitrate can be a filename: remove leading "../"
		if bitrate.startswith("../"):
		bitrate = bitrate[3:]

		testv_base = testv_file.split("/")[-1]
		if testv_base.endswith(".pcm"):
		testv_base = testv_base[:-4]

		assert bitstream_file == "bit"
		# in the parameter file, only "bit" is used as bitstream file name
		# -> construct bitstream filename
		bitstream_file = f"{testv_base}_{tag_str}.192"

		encode(
		dut_encoder_frontend,
		ref_encoder_path,
		reference_path,
		dut_base_path,
		bitrate,
		sampling_rate,
		testv_file,
		bitstream_file,
		enc_split,
		update_ref,
		)

		# evaluate decoder options
		dec_split = dec_opts.split()
		assert len(dec_split) >= 3

		# replace "testv/" by extended test vector path
		dec_split = [
		x.replace("testv", f"{test_vector_path}/param_file", 1) if x.startswith("testv/") else x
		for x in dec_split
		]
		# remove leading "../"
		dec_split = [x[3:] if x.startswith("../") else x for x in dec_split]

		output_file = dec_split.pop()
		bitstream_file_dec = dec_split.pop()
		sampling_rate = int(dec_split.pop())
		if len(dec_split) > 0:
		output_config = dec_split.pop()
		else:
		output_config = ""

		output_config_name = output_config
		if "/" in output_config:
		# the output config is a file
		output_config_name = os.path.splitext(os.path.basename(output_config))[0]

		assert bitstream_file_dec == "bit"
		# in the parameter file, only "bit" is used as bitstream file name
		# -> re-use bitstream filename from encoder call

		# the output file is not the real output filename
		# -> construct output filename
		if output_config is not "":
		output_file = f"{testv_base}_{tag_str}.dec.{output_config_name}.pcm"
		else:
		# EVS decoder command lines do not have an output_config: use "MONO" in the output filename
		output_file = f"{testv_base}_{tag_str}.dec.MONO.pcm"

		decode(
		dut_decoder_frontend,
		ref_decoder_path,
		reference_path,
		dut_base_path,
		output_config,
		sampling_rate,
		bitstream_file,
		output_file,
		dec_split,
		update_ref,
		)

		# compare
		if update_ref in [0, 2]:
		compare(
		f"{dut_base_path}/param_file/dec/{output_file}",
		f"{reference_path}/param_file/dec/{output_file}",
		)

		# clean-up
		# TODO: consider removing DUT output files when test result is OK (to save disk space)


		def encode(
		encoder_frontend,
		ref_encoder_path,
		reference_path,
		dut_base_path,
		bitrate,
		sampling_rate,
		testv_file,
		bitstream_file,
		enc_opts_list,
		update_ref,
		):
		"""
		Call REF and/or DUT decoder.
		"""
		# directories
		dut_out_dir = f"{dut_base_path}/param_file/enc"
		ref_out_dir = f"{reference_path}/param_file/enc"

		ref_out_file = f"{ref_out_dir}/{bitstream_file}"
		dut_out_file = f"{dut_out_dir}/{bitstream_file}"

		if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file):
		check_and_makedir(ref_out_dir)
		# call REF encoder
		assert ref_encoder_path
		ref_encoder = EncoderFrontend(ref_encoder_path, "REF")
		ref_encoder.run(
		bitrate,
		sampling_rate,
		testv_file,
		ref_out_file,
		add_option_list=enc_opts_list,
		)

		if update_ref in [0, 2]:
		check_and_makedir(dut_out_dir)
		# call DUT encoder
		encoder_frontend.run(
		bitrate,
		sampling_rate,
		testv_file,
		dut_out_file,
		add_option_list=enc_opts_list,
		)


		def decode(
		decoder_frontend,
		ref_decoder_path,
		reference_path,
		dut_base_path,
		output_config,
		sampling_rate,
		bitstream_file,
		output_file,
		dec_opts_list,
		update_ref,
		):
		"""
		Call REF and/or DUT decoder.
		"""
		# directories
		dut_out_dir = f"{dut_base_path}/param_file/dec"
		ref_out_dir = f"{reference_path}/param_file/dec"

		dut_in_file = f"{dut_base_path}/param_file/enc/{bitstream_file}"
		ref_in_file = f"{reference_path}/param_file/enc/{bitstream_file}"
		dut_out_file = f"{dut_out_dir}/{output_file}"
		ref_out_file = f"{ref_out_dir}/{output_file}"

		if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file):
		check_and_makedir(ref_out_dir)
		# call REF decoder
		assert ref_decoder_path
		ref_decoder = DecoderFrontend(ref_decoder_path, "REF")
		ref_decoder.run(
		output_config,
		sampling_rate,
		ref_in_file,
		ref_out_file,
		add_option_list=dec_opts_list,
		)

		if update_ref in [0, 2]:
		check_and_makedir(dut_out_dir)
		# call DUT decoder
		decoder_frontend.run(
		output_config,
		sampling_rate,
		dut_in_file,
		dut_out_file,
		add_option_list=dec_opts_list,
		)


		def compare(
		pcm_file_1,
		pcm_file_2,
		):
		"""
		Compare two PCM files.
		Currently, both PCM files are treated like mono files.
		This is just fine when checking for bit-exactness.
		More advanced comparisons are possible and might come with a future update.
		"""
		sample_size = "2" # 16-bit samples
		tolerance = "0" # zero tolerance for BE testing
		cmp_result = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance)
		assert cmp_result == 0

tests/test_sba_bs_dec_plc.py

+3 −4

Original line number	Diff line number	Diff line
		@@ -32,11 +32,10 @@ import os
		import pytest
		import shutil
		import errno
		import sys

		from il2mm import il2mm
		from cmp_custom import cmp_custom
		from conftest import EncoderFrontend, DecoderFrontend
		from conftest import DecoderFrontend

		#params
		tag_list = ['stvFOA']
		@@ -74,7 +73,7 @@ def check_and_makedir(dir_path):
		@pytest.mark.parametrize("agc", agc_list)
		def test_sba_plc_system(
		dut_decoder_frontend: DecoderFrontend,
		data_system_tests_path,
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_decoder_path,
		@@ -89,7 +88,7 @@ def test_sba_plc_system(
		tag = tag + fs + 'c'

		#dec
		sba_dec_plc(dut_decoder_frontend, data_system_tests_path, reference_path, dut_base_path, ref_decoder_path, tag, ch_count_foa, fs, ivas_br, dtx, plc_pattern, update_ref, agc)
		sba_dec_plc(dut_decoder_frontend, test_vector_path, reference_path, dut_base_path, ref_decoder_path, tag, ch_count_foa, fs, ivas_br, dtx, plc_pattern, update_ref, agc)


		#########################################################

tests/test_sba_bs_enc.py

+10 −11

Original line number	Diff line number	Diff line
		@@ -37,7 +37,6 @@ import os
		import pytest
		import errno
		import shutil
		import sys

		from il2mm import il2mm
		from cmp_custom import cmp_custom
		@@ -84,7 +83,7 @@ def check_and_makedir(dir_path):
		def test_bypass_enc(
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		data_system_tests_path,
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		@@ -108,7 +107,7 @@ def test_bypass_enc(
		# enc
		sba_enc(
		dut_encoder_frontend,
		data_system_tests_path,
		test_vector_path,
		ref_encoder_path,
		reference_path,
		dut_base_path,
		@@ -151,7 +150,7 @@ def test_bypass_enc(
		def test_sba_enc_system(
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		data_system_tests_path,
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		@@ -176,7 +175,7 @@ def test_sba_enc_system(
		# enc
		sba_enc(
		dut_encoder_frontend,
		data_system_tests_path,
		test_vector_path,
		ref_encoder_path,
		reference_path,
		dut_base_path,
		@@ -216,7 +215,7 @@ def test_sba_enc_system(
		def test_spar_hoa2_enc_system(
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		data_system_tests_path,
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		@@ -238,7 +237,7 @@ def test_spar_hoa2_enc_system(
		# enc
		sba_enc(
		dut_encoder_frontend,
		data_system_tests_path,
		test_vector_path,
		ref_encoder_path,
		reference_path,
		dut_base_path,
		@@ -278,7 +277,7 @@ def test_spar_hoa2_enc_system(
		def test_spar_hoa3_enc_system(
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		data_system_tests_path,
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		@@ -300,7 +299,7 @@ def test_spar_hoa3_enc_system(
		# enc
		sba_enc(
		dut_encoder_frontend,
		data_system_tests_path,
		test_vector_path,
		ref_encoder_path,
		reference_path,
		dut_base_path,
		@@ -342,7 +341,7 @@ def test_spar_hoa3_enc_system(
		def test_sba_enc_BWforce_system(
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		data_system_tests_path,
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		@@ -364,7 +363,7 @@ def test_sba_enc_BWforce_system(
		# enc
		sba_enc(
		dut_encoder_frontend,
		data_system_tests_path,
		test_vector_path,
		ref_encoder_path,
		reference_path,
		dut_base_path,