Merge branch 'ci/refactor-sba-tests-to-make-encoder-basop-tests-integration-easier' into 'main' (a5f949bf) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/README.md

+3 −3

Original line number	Diff line number	Diff line
		@@ -88,16 +88,16 @@ To run a specific test case, you can e.g. pick a test case from the `short test

		```bash
		# run a specific test case
		pytest tests/test_sba_bs_dec_plc.py::test_sba_plc_system[0-48-PLperc12mblen5-stvFOA-0-32000]
		pytest tests/test_sba.py::test_sba_plc_system[0-48-PLperc12mblen5-stvFOA-0-32000]
		```

		More ways to select which tests to run:

		```bash
		# run all tests within a module
		pytest tests/test_sba_bs_dec_plc.py
		pytest tests/test_sba.py
		# run a specific test from a module
		pytest tests/test_sba_bs_dec_plc.py::test_sba_plc_system
		pytest tests/test_sba.py::test_sba_plc_system
		```

		## Some pytest hints

tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py→tests/codec_be_on_mr_nonselection/test_sba.py

+111 −6

Original line number	Diff line number	Diff line
		@@ -42,6 +42,7 @@ from cut_bs import cut_from_start
		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend
		from ..conftest import parse_properties
		from ..constants import TESTV_DIR

		# params

		@@ -76,6 +77,11 @@ sample_rate_bw_idx_list = [("48", "SWB"), ("48", "WB"), ("32", "WB")]

		AbsTol = "0"

		# params for PLC test
		tag_list_plc = tag_list
		plc_patterns = ["PLperc12mblen5", "PLperc40mblen50", "PLperc42mblen2"]
		ivas_br_plc = ["13200", "16400", "32000", "64000", "96000", "256000"]


		def check_and_makedir(dir_path):
		if not os.path.exists(dir_path):
		@@ -536,6 +542,87 @@ def test_sba_enc_BWforce_system(
		)


		# assumption:
		# - the needed reference bitstreams are created by test_sba_enc_system
		# -> reference bitstreams are not any longer created as part of this test
		# -> the parameters of this test (except additional parameter plc_pattern) need to be a subset of the parameters in test_sba_enc_system
		# -> the reference generation for this test (reference decoder output) needs to be done after completion of test_sba_enc_system
		# -> therefore the marker create_ref_part2
		@pytest.mark.create_ref_part2
		@pytest.mark.parametrize("ivas_br", ivas_br_plc)
		@pytest.mark.parametrize("dtx", dtx_set)
		@pytest.mark.parametrize("tag", tag_list)
		@pytest.mark.parametrize("plc_pattern", plc_patterns)
		@pytest.mark.parametrize("fs", sample_rate_list)
		@pytest.mark.parametrize("gain_flag", gain_list)
		def test_sba_plc_system(
		record_property,
		props_to_record,
		dut_decoder_frontend: DecoderFrontend,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		ref_decoder_path,
		update_ref,
		keep_files,
		ivas_br,
		dtx,
		tag,
		plc_pattern,
		fs,
		gain_flag,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg,
		):
		SID = 0
		if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]:
		# skip high bitrates for DTX until DTX issue is resolved
		pytest.skip()
		if ivas_br == "13200" or ivas_br == "16400":
		if (
		dtx == "1"
		and gain_flag == 0
		and fs != "16"
		and plc_pattern == "PLperc12mblen5"
		):
		SID = 1
		else:
		pytest.skip()
		if gain_flag == 1 and ivas_br not in ["13200", "16400", "24400", "32000"]:
		pytest.skip()
		tag = tag + fs + "c"

		# dec
		sba_dec(
		record_property,
		props_to_record,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		dut_base_path,
		tag,
		fs,
		ivas_br,
		dtx,
		SID,
		"FB",
		"FOA",
		update_ref,
		gain_flag,
		keep_files,
		False,
		plc_pattern=plc_pattern,
		get_mld=get_mld,
		get_mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		)


		#########################################################
		# -------------------- test function --------------------
		def sba_enc(
		@@ -606,9 +693,6 @@ def sba_enc(
		if cut_testv:
		# use shortened and potentially gain adjusted input PCM file - create if not present
		# cut input PCM file: currently with mostly fixed (i.e. not test dependant) values
		num_channels = "4" # currently only FOA inputs end up, here
		cut_from = "0.0"
		cut_len = "5.0"
		if cut_gain == "1.0":
		cut_file = f"{test_vector_path}/{tag_in}_cut{in_extension}"
		else:
		@@ -691,6 +775,7 @@ def sba_dec(
		gain_flag,
		keep_files,
		decoder_only,
		plc_pattern=None,
		get_mld=False,
		get_mld_lim=0,
		pca=False,
		@@ -724,6 +809,22 @@ def sba_dec(

		dut_in_pkt = f"{dut_base_path}/sba_bs/pkt/{tag_out}{long_tag_ext}.192"
		ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}.192"
		ref_in_pkt_dutenc = (
		f"{reference_path}/sba_bs/pkt/{tag_out}{short_tag_ext}_dutenc.192"
		)
		if SID == 1:
		if gain_flag != -1:
		ref_in_pkt_dutenc = f"{reference_path}/sba_bs/pkt/{tag_out}_Gain{gain_flag}_SID_dutenc_cut.192"
		else:
		ref_in_pkt_dutenc = (
		f"{reference_path}/sba_bs/pkt/{tag_out}_SID_dutenc_cut.192"
		)

		plc_file = None
		if plc_pattern is not None:
		plc_file = f"{TESTV_DIR}/{plc_pattern}.g192"
		long_tag_ext = f"{long_tag_ext}_{plc_pattern}"
		short_tag_ext = f"{short_tag_ext}_{plc_pattern}"

		dut_out_raw = f"{dut_out_dir}/{tag_out}{long_tag_ext}.wav"
		ref_out_raw = f"{ref_out_dir}/{tag_out}{short_tag_ext}.wav"
		@@ -738,10 +839,13 @@ def sba_dec(
		sampling_rate,
		ref_in_pkt,
		ref_out_raw,
		plc_file=plc_file,
		)

		if update_ref == 0:
		if decoder_only:
		if plc_file is not None:
		dut_in_pkt = ref_in_pkt_dutenc
		elif decoder_only:
		dut_in_pkt = ref_in_pkt

		# call DUT decoder
		@@ -750,6 +854,7 @@ def sba_dec(
		sampling_rate,
		dut_in_pkt,
		dut_out_raw,
		plc_file=plc_file,
		)

		fs = int(sampling_rate) * 1000
		@@ -776,5 +881,5 @@ def sba_dec(
		# remove DUT output files when test result is OK (to save disk space)
		if not keep_files:
		os.remove(dut_out_raw)
		if not decoder_only:
		if not decoder_only and plc_file is None:
		os.remove(dut_in_pkt)

tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py

deleted100644 → 0

+0 −238

Original line number	Diff line number	Diff line
		__copyright__ = """
		(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository. All Rights Reserved.

		This software is protected by copyright law and by international treaties.
		The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository retain full ownership rights in their respective contributions in
		the software. This notice grants no license of any kind, including but not limited to patent
		license, nor is any license granted by implication, estoppel or otherwise.

		Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
		contributions.

		This software is provided "AS IS", without any express or implied warranties. The software is in the
		development stage. It is intended exclusively for experts who have experience with such software and
		solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
		and fitness for a particular purpose are hereby disclaimed and excluded.

		Any dispute, controversy or claim arising under or in relation to providing this software shall be
		submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		__doc__ = """
		Execute SBA decoder tests using different PLC patterns.
		"""

		import errno
		import os

		import pytest

		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend
		from ..conftest import parse_properties

		# params
		tag_list = ["stvFOA"]
		plc_patterns = ["PLperc12mblen5", "PLperc40mblen50", "PLperc42mblen2"]
		dtx_set = ["0", "1"]
		ivas_br_list = ["13200", "16400", "32000", "64000", "96000", "256000"]
		sampling_rate_list = ["48", "32", "16"]
		gain_list = [0, 1]

		AbsTol = "0"


		def check_and_makedir(dir_path):
		if not os.path.exists(dir_path):
		try:
		os.makedirs(dir_path)
		except OSError as e:
		if e.errno != errno.EEXIST:
		raise # raises the error again


		# assumption:
		# - the needed reference bitstreams are created by test_sba_enc_system
		# -> reference bitstreams are not any longer created as part of this test
		# -> the parameters of this test (except additional parameter plc_pattern) need to be a subset of the parameters in test_sba_enc_system
		# -> the reference generation for this test (reference decoder output) needs to be done after completion of test_sba_enc_system
		# -> therefore the marker create_ref_part2
		@pytest.mark.create_ref_part2
		@pytest.mark.parametrize("ivas_br", ivas_br_list)
		@pytest.mark.parametrize("dtx", dtx_set)
		@pytest.mark.parametrize("tag", tag_list)
		@pytest.mark.parametrize("plc_pattern", plc_patterns)
		@pytest.mark.parametrize("fs", sampling_rate_list)
		@pytest.mark.parametrize("gain_flag", gain_list)
		def test_sba_plc_system(
		record_property,
		props_to_record,
		dut_decoder_frontend: DecoderFrontend,
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_decoder_path,
		update_ref,
		keep_files,
		ivas_br,
		dtx,
		tag,
		plc_pattern,
		fs,
		gain_flag,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg
		):
		SID = 0
		if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]:
		# skip high bitrates for DTX until DTX issue is resolved
		pytest.skip()
		if ivas_br == "13200" or ivas_br == "16400":
		if (
		dtx == "1"
		and gain_flag == 0
		and fs != "16"
		and plc_pattern == "PLperc12mblen5"
		):
		SID = 1
		else:
		pytest.skip()
		if gain_flag == 1 and ivas_br not in ["13200", "16400", "24400", "32000"]:
		pytest.skip()
		tag = tag + fs + "c"

		# dec
		sba_dec_plc(
		record_property,
		props_to_record,
		dut_decoder_frontend,
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_decoder_path,
		tag,
		fs,
		ivas_br,
		dtx,
		SID,
		plc_pattern,
		update_ref,
		gain_flag,
		keep_files,
		get_mld=get_mld,
		get_mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		)


		#########################################################
		# -------------------- test function --------------------
		def sba_dec_plc(
		record_property,
		props_to_record,
		decoder_frontend,
		test_vector_path,
		reference_path,
		dut_base_path,
		ref_decoder_path,
		tag,
		sampling_rate,
		ivas_br,
		dtx,
		SID,
		plc_pattern,
		update_ref,
		gain_flag,
		keep_files,
		get_mld=False,
		get_mld_lim=0,
		abs_tol=0,
		get_ssnr=False,
		get_odg=False,
		):
		# ------------ run cmd ------------

		tag_out = f"{tag}_ivasbr{ivas_br[:-3]}k_DTX{dtx}"
		if gain_flag != -1:
		tag_out += f"_Gain{gain_flag}"
		plc_tag_out = f"{tag_out}_{plc_pattern}"

		dut_out_dir = f"{dut_base_path}/sba_bs/raw"
		ref_out_dir = f"{reference_path}/sba_bs/raw"

		check_and_makedir(dut_out_dir)
		check_and_makedir(ref_out_dir)

		plc_file = f"{test_vector_path}/{plc_pattern}.g192"
		ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}.192"
		ref_in_pkt_dutenc = f"{reference_path}/sba_bs/pkt/{tag_out}_dutenc.192"
		if SID == 1:
		ref_in_pkt = f"{reference_path}/sba_bs/pkt/{tag_out}_SID_cut.192"
		ref_in_pkt_dutenc = f"{reference_path}/sba_bs/pkt/{tag_out}_SID_dutenc_cut.192"

		dut_out_raw = f"{dut_out_dir}/{plc_tag_out}.wav"
		ref_out_raw = f"{ref_out_dir}/{plc_tag_out}.wav"

		output_config = "FOA"
		if ref_decoder_path:
		ref_decoder = DecoderFrontend(ref_decoder_path, "REF")

		# call REF decoder
		ref_decoder.run(
		output_config,
		sampling_rate,
		ref_in_pkt,
		ref_out_raw,
		plc_file=plc_file,
		)

		if update_ref == 0:
		# call DUT decoder
		decoder_frontend.run(
		output_config,
		sampling_rate,
		ref_in_pkt_dutenc,
		dut_out_raw,
		plc_file=plc_file,
		)

		# -------------- compare cmd --------------
		fs = int(sampling_rate) * 1000
		cmp_result, reason = cmp_pcm(
		dut_out_raw,
		ref_out_raw,
		output_config,
		fs,
		get_mld=get_mld,
		mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		)

		props = parse_properties(reason, cmp_result!=0, props_to_record)
		for k, v in props.items():
		record_property(k, v)

		# report compare result
		if cmp_result != 0:
		pytest.fail(reason)

		# remove DUT output files when test result is OK (to save disk space)
		if not keep_files:
		os.remove(dut_out_raw)