add command line option testcase_timeout and functionality (8f6a2fcf) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py

+2 −4

Original line number	Diff line number	Diff line
		@@ -78,8 +78,8 @@ def check_and_makedir(dir_path):
		def test_masa_enc_dec(
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		ref_encoder_path,
		ref_decoder_path,
		ref_encoder_frontend: EncoderFrontend,
		ref_decoder_frontend: DecoderFrontend,
		reference_path,
		dut_base_path,
		update_ref,
		@@ -107,8 +107,6 @@ def test_masa_enc_dec(
		pytest.skip("Skipping some DTX bitrates for other than EXT output to save time")

		# Set reference encoder and decoder
		ref_encoder_frontend = EncoderFrontend(ref_encoder_path, "REF")
		ref_decoder_frontend = DecoderFrontend(ref_decoder_path, "REF")

		# Set output paths
		out_dir_bs_ref = f"{reference_path}/masa_test/bitstreams"

tests/codec_be_on_mr_nonselection/test_param_file.py

+11 −15

Original line number	Diff line number	Diff line
		@@ -124,8 +124,8 @@ def convert_test_string_to_tag(test_string):
		def test_param_file_tests(
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		ref_encoder_path,
		ref_decoder_path,
		ref_encoder_frontend: EncoderFrontend,
		ref_decoder_frontend: DecoderFrontend,
		reference_path,
		dut_base_path,
		test_vector_path,
		@@ -180,7 +180,7 @@ def test_param_file_tests(

		encode(
		dut_encoder_frontend,
		ref_encoder_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		bitrate,
		@@ -272,7 +272,7 @@ def test_param_file_tests(

		stdout = decode(
		dut_decoder_frontend,
		ref_decoder_path,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		output_config,
		@@ -331,8 +331,8 @@ def test_param_file_tests(


		def encode(
		encoder_frontend,
		ref_encoder_path,
		dut_encoder_frontend,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		bitrate,
		@@ -355,9 +355,7 @@ def encode(
		if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file):
		check_and_makedir(ref_out_dir)
		# call REF encoder
		assert ref_encoder_path
		ref_encoder = EncoderFrontend(ref_encoder_path, "REF")
		ref_encoder.run(
		ref_encoder_frontend.run(
		bitrate,
		sampling_rate,
		testv_file,
		@@ -368,7 +366,7 @@ def encode(
		if update_ref in [0, 2]:
		check_and_makedir(dut_out_dir)
		# call DUT encoder
		encoder_frontend.run(
		dut_encoder_frontend.run(
		bitrate,
		sampling_rate,
		testv_file,
		@@ -447,7 +445,7 @@ def simulate(

		def decode(
		decoder_frontend,
		ref_decoder_path,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		output_config,
		@@ -479,9 +477,7 @@ def decode(
		for x in dec_opts_list
		]
		# call REF decoder
		assert ref_decoder_path
		ref_decoder = DecoderFrontend(ref_decoder_path, "REF")
		ref_decoder.run(
		ref_decoder_frontend.run(
		output_config,
		sampling_rate,
		ref_in_file,
		@@ -489,7 +485,7 @@ def decode(
		add_option_list=add_option_list,
		)

		stdout = ref_decoder.stdout
		stdout = ref_decoder_frontend.stdout

		if update_ref in [0, 2]:
		check_and_makedir(dut_out_dir)

tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py

+30 −31

Original line number	Diff line number	Diff line
		@@ -86,8 +86,8 @@ def test_bypass_enc(
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		ref_decoder_path,
		ref_encoder_frontend,
		ref_decoder_frontend,
		update_ref,
		keep_files,
		tag,
		@@ -109,7 +109,7 @@ def test_bypass_enc(
		sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		None,
		@@ -129,7 +129,7 @@ def test_bypass_enc(
		# dec
		sba_dec(
		dut_decoder_frontend,
		ref_decoder_path,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		tag,
		@@ -158,8 +158,8 @@ def test_sba_enc_system(
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		ref_decoder_path,
		ref_encoder_frontend,
		ref_decoder_frontend,
		br_switch_file_path,
		update_ref,
		keep_files,
		@@ -197,7 +197,7 @@ def test_sba_enc_system(
		sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		br_switch_file_path,
		@@ -219,7 +219,7 @@ def test_sba_enc_system(
		# dec
		sba_dec(
		dut_decoder_frontend,
		ref_decoder_path,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		tag,
		@@ -245,8 +245,8 @@ def test_spar_hoa2_enc_system(
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		ref_decoder_path,
		ref_encoder_frontend,
		ref_decoder_frontend,
		update_ref,
		keep_files,
		ivas_br,
		@@ -266,7 +266,7 @@ def test_spar_hoa2_enc_system(
		sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		None,
		@@ -285,7 +285,7 @@ def test_spar_hoa2_enc_system(
		# dec
		sba_dec(
		dut_decoder_frontend,
		ref_decoder_path,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		tag,
		@@ -311,8 +311,8 @@ def test_spar_hoa3_enc_system(
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		ref_decoder_path,
		ref_encoder_frontend,
		ref_decoder_frontend,
		update_ref,
		keep_files,
		ivas_br,
		@@ -332,7 +332,7 @@ def test_spar_hoa3_enc_system(
		sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		None,
		@@ -351,7 +351,7 @@ def test_spar_hoa3_enc_system(
		# dec
		sba_dec(
		dut_decoder_frontend,
		ref_decoder_path,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		tag,
		@@ -379,8 +379,8 @@ def test_sba_enc_BWforce_system(
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_encoder_path,
		ref_decoder_path,
		ref_encoder_frontend,
		ref_decoder_frontend,
		update_ref,
		keep_files,
		ivas_br,
		@@ -407,7 +407,7 @@ def test_sba_enc_BWforce_system(
		sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		None,
		@@ -427,7 +427,7 @@ def test_sba_enc_BWforce_system(
		# dec
		sba_dec(
		dut_decoder_frontend,
		ref_decoder_path,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		tag,
		@@ -449,7 +449,7 @@ def test_sba_enc_BWforce_system(
		def sba_enc(
		encoder_frontend,
		test_vector_path,
		ref_encoder_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		br_switch_file_path,
		@@ -526,10 +526,9 @@ def sba_enc(
		cut_samples(input_path, cut_file, num_channels, sampling_rate + "000", cut_from, cut_len, cut_gain)
		input_path = cut_file

		if ref_encoder_path:
		ref_encoder = EncoderFrontend(ref_encoder_path, "REF")
		if ref_encoder_frontend:
		# call REF encoder
		ref_encoder.run(
		ref_encoder_frontend.run(
		ivas_br,
		sampling_rate,
		input_path,
		@@ -566,7 +565,7 @@ def sba_enc(
		)

		if SID == 1:
		if ref_encoder_path:
		if ref_encoder_frontend:
		with open(ref_pkt_file, "rb") as fp_in:
		with open(ref_pkt_file_cut, "wb") as fp_out:
		fr_cnt, cut_cnt = cut_from_start(fp_in, fp_out, 0, True)
		@@ -580,9 +579,11 @@ def sba_enc(
		with open(dut_pkt_file_cut, "wb") as fp_out:
		fr_cnt, cut_cnt = cut_from_start(fp_in, fp_out, 0, True)
		os.remove(dut_pkt_file)


		def sba_dec(
		decoder_frontend,
		ref_decoder_path,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		tag,
		@@ -633,11 +634,9 @@ def sba_dec(
		check_and_makedir(dut_out_dir)
		check_and_makedir(ref_out_dir)

		if ref_decoder_path:
		ref_decoder = DecoderFrontend(ref_decoder_path, "REF")

		if ref_decoder_frontend:
		# call REF decoder
		ref_decoder.run(
		ref_decoder_frontend.run(
		output_config,
		sampling_rate,
		ref_in_pkt,

tests/conftest.py

+55 −11

Original line number	Diff line number	Diff line
		@@ -37,9 +37,9 @@ Pytest customization (configuration and fixtures) for the IVAS codec test suite.
		import logging
		from pathlib import Path
		import platform
		from subprocess import run
		from subprocess import run, TimeoutExpired
		import textwrap
		from typing import Optional
		from typing import Optional, Union
		import os
		from tests import testconfig
		import pytest
		@@ -140,13 +140,14 @@ def pytest_addoption(parser):
		" Use --compare_bitstream to compare IVAS encoded bitstream.",
		)


		parser.addoption(
		"--selection_be_md5_file",
		type=Path,
		help="Path to file with md5 sums for the reference signals of the selection-BE test"
		)

		parser.addoption("--testcase_timeout", type=int, default=None, help="Timeout in seconds for each individual testcase. Default is no timeout.")


		@pytest.fixture(scope="session", autouse=True)
		def update_ref(request):
		@@ -200,12 +201,14 @@ def dut_encoder_path(request) -> str:


		class EncoderFrontend:
		def __init__(self, path, enc_type) -> None:
		def __init__(self, path, enc_type, timeout=None) -> None:
		self._path = path
		self._type = enc_type
		self.returncode = None
		self.stdout = None
		self.stderr = None
		self.timeout = timeout
		print(self.timeout, type(self.timeout))

		def run(
		self,
		@@ -248,11 +251,14 @@ class EncoderFrontend:
		str(input_path),
		str(output_bitstream_path),
		]
		print(command)

		cmd_str = textwrap.indent(" ".join(command), prefix="\t")
		log_dbg_msg(f"{self._type} encoder command:\n{cmd_str}")

		result = run(command, capture_output=True, check=False)
		print(self.timeout, type(self.timeout))
		result = run(command, capture_output=True, check=True, timeout=1)

		self.returncode = result.returncode
		self.stderr = result.stderr.decode("ascii")
		self.stdout = result.stdout.decode("ascii")
		@@ -276,11 +282,12 @@ class EncoderFrontend:


		@pytest.fixture(scope="function")
		def dut_encoder_frontend(dut_encoder_path) -> EncoderFrontend:
		def dut_encoder_frontend(dut_encoder_path, request) -> EncoderFrontend:
		"""
		Return a :class:`conftest.EncoderFrontend` instance as DUT for the test session.
		"""
		encoder = EncoderFrontend(dut_encoder_path, "DUT")
		timeout = request.config.getoption("--testcase_timeout")
		encoder = EncoderFrontend(dut_encoder_path, "DUT", timeout=timeout)
		yield encoder

		# Fixture teardown
		@@ -317,6 +324,24 @@ def ref_encoder_path(request) -> str:
		return path


		@pytest.fixture(scope="function")
		def ref_encoder_frontend(ref_encoder_path, request) -> Union[None, EncoderFrontend]:
		"""
		Return a :class:`conftest.EncoderFrontend` instance as REF for the test session.
		"""
		encoder = None

		if ref_encoder_path:
		timeout = request.config.getoption("--testcase_timeout")
		encoder = EncoderFrontend(ref_encoder_path, "REF", timeout=timeout)

		yield encoder

		if encoder is not None:
		# Fixture teardown
		encoder._check_run()


		@pytest.fixture(scope="session")
		def dut_decoder_path(request) -> str:
		"""
		@@ -344,12 +369,13 @@ def dut_decoder_path(request) -> str:


		class DecoderFrontend:
		def __init__(self, path, dec_type) -> None:
		def __init__(self, path, dec_type, timeout=None) -> None:
		self._path = path
		self._type = dec_type
		self.returncode = None
		self.stdout = None
		self.stderr = None
		self.timeout = timeout

		def run(
		self,
		@@ -386,7 +412,7 @@ class DecoderFrontend:
		cmd_str = textwrap.indent(" ".join(command), prefix="\t")
		log_dbg_msg(f"{self._type} decoder command:\n{cmd_str}")

		result = run(command, capture_output=True, check=False)
		result = run(command, capture_output=True, check=False, timeout=self.timeout)
		self.returncode = result.returncode
		self.stderr = result.stderr.decode("ascii")
		self.stdout = result.stdout.decode("ascii")
		@@ -410,11 +436,11 @@ class DecoderFrontend:


		@pytest.fixture(scope="function")
		def dut_decoder_frontend(dut_decoder_path) -> DecoderFrontend:
		def dut_decoder_frontend(dut_decoder_path, request) -> DecoderFrontend:
		"""
		Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session.
		"""
		decoder = DecoderFrontend(dut_decoder_path, "DUT")
		decoder = DecoderFrontend(dut_decoder_path, "DUT", timeout=request.config.getoption("--testcase_timeout"))
		yield decoder

		# Fixture teardown
		@@ -451,6 +477,24 @@ def ref_decoder_path(request) -> str:
		return path


		@pytest.fixture(scope="function")
		def ref_decoder_frontend(ref_decoder_path, request) -> Union[None, DecoderFrontend]:
		"""
		Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session.
		"""
		decoder = None

		if ref_decoder_path:
		timeout = request.config.getoption("--testcase_timeout")
		decoder = DecoderFrontend(ref_decoder_path, "REF", timeout=timeout)

		yield decoder

		if decoder is not None:
		# Fixture teardown
		decoder._check_run()


		@pytest.fixture(scope="session")
		def test_vector_path(request) -> str:
		"""