first decoder testcases (71e6fb1c) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/codec_be_to_accepted_release/init.py

+6 −0

Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@ from pathlib import Path
		import filecmp
		import subprocess
		from ..testconfig import MD5_REF_DICT
		from .constants import TESTVECTOR_FOR_INPUT_FORMAT

		HERE = Path(__file__).parent
		TESTV_PATH = Path(os.environ.get("BE_TESTV_DIR", HERE.joinpath("testv")))
		@@ -61,3 +62,8 @@ def get_bitstream_path(
		):
		bs_name = f"{testv_name}-{encoder_format}-{bitrate}kbps-{sampling_rate}kHz-{dtx}{suffix}.192"
		return base_path.joinpath(bs_name)


		def get_testv_path(input_format, sampling_rate):
		testv = TESTVECTOR_FOR_INPUT_FORMAT[input_format].format(fs=sampling_rate)
		return TESTV_PATH.joinpath(testv)

tests/codec_be_to_accepted_release/constants.py

+177 −3

Original line number	Diff line number	Diff line
		@@ -28,6 +28,13 @@ accordance with the laws of the Federal Republic of Germany excluding its confli
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		from itertools import product


		def collapse_into_list_of_pairs(params):
		return [p[0] + p[1:] for p in params]


		DTX_ON = "DTXon"
		DTX_OFF = "DTXoff"

		@@ -107,13 +114,11 @@ INPUT_FORMATS_MASA = [
		"MASA_1TC_2DIR",
		"MASA_2TC_2DIR",
		]
		# TODO: add MASA back
		INPUT_FORMATS_SINGLE = (
		INPUT_FORMATS_CHANNEL_BASED + INPUT_FORMATS_SCENE_BASED + INPUT_FORMATS_OBJECT_BASED
		) # + INPUT_FORMATS_MASA
		# TODO: combined formats

		# bitrates
		BITRATES_ALL = [
		13200,
		16400,
		@@ -140,5 +145,174 @@ BITRATES_ISM3_EXTENDED = [b for b in BITRATES_ISM3 if b > 64000]
		BITRATES_ISM4_EXTENDED = [b for b in BITRATES_ISM4 if b > 64000]
		BITRATES_EVS = [5900, 7200, 8000, 9600] + BITRATES_ALL[:6] + BITRATES_ALL[7:9]

		# sampling rates
		SAMPLING_RATES_ALL = [16, 32, 48]

		OUTPUT_FORMATS_ALL = [
		"MONO",
		"STEREO",
		"5_1",
		"5_1_2",
		"5_1_4",
		"7_1",
		"7_1_4",
		"FOA",
		"HOA2",
		"HOA3",
		"EXT",
		"BINAURAL",
		"BINAURAL_ROOM_IR",
		"BINAURAL_ROOM_REVERB",
		]
		OUTPUT_FORMATS_BINAURAL = OUTPUT_FORMATS_ALL[-3:]


		### -------------- encoder parameter lists --------------

		STEREO_PARAMS = list(
		product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON])
		)
		MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF]))
		MASA_PARAMS = list(
		product(INPUT_FORMATS_MASA, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON])
		)
		ENCODER_CHANNEL_BASED_AND_MASA_PARAMS = STEREO_PARAMS + MC_PARAMS + MASA_PARAMS


		# ISM modes with three "flavours" of metadata
		# different ranges of bitrates allowed depending on number of objects, ext MD only for >64kbps
		ISM1_BASIC_NULL_PARAMS = list(
		product(
		["ISM1"],
		BITRATES_ISM1,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_BASIC, ISM_MD_NULL],
		)
		)
		ISM2_BASIC_NULL_PARAMS = list(
		product(
		["ISM2"],
		BITRATES_ISM2,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_BASIC, ISM_MD_NULL],
		)
		)
		ISM3_BASIC_NULL_PARAMS = list(
		product(
		["ISM3"],
		BITRATES_ISM3,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_BASIC, ISM_MD_NULL],
		)
		)
		ISM4_BASIC_NULL_PARAMS = list(
		product(
		["ISM4"],
		BITRATES_ISM4,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_BASIC, ISM_MD_NULL],
		)
		)
		ISM1_EXTENDED_PARAMS = list(
		product(
		["ISM1"],
		BITRATES_ISM1_EXTENDED,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_EXTENDED],
		)
		)
		ISM2_EXTENDED_PARAMS = list(
		product(
		["ISM2"],
		BITRATES_ISM2_EXTENDED,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_EXTENDED],
		)
		)
		ISM3_EXTENDED_PARAMS = list(
		product(
		["ISM3"],
		BITRATES_ISM3_EXTENDED,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_EXTENDED],
		)
		)
		ISM4_EXTENDED_PARAMS = list(
		product(
		["ISM4"],
		BITRATES_ISM4_EXTENDED,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_EXTENDED],
		)
		)
		ISM_PARAMS = (
		ISM1_BASIC_NULL_PARAMS
		+ ISM2_BASIC_NULL_PARAMS
		+ ISM3_BASIC_NULL_PARAMS
		+ ISM4_BASIC_NULL_PARAMS
		+ ISM1_EXTENDED_PARAMS
		+ ISM2_EXTENDED_PARAMS
		+ ISM3_EXTENDED_PARAMS
		+ ISM4_EXTENDED_PARAMS
		)


		# dtx only allowed at low bitrates, PCA only at FOA 256kbps
		SBA_LOW_BITRATES_PARAMS = list(
		(
		product(
		INPUT_FORMATS_SCENE_BASED,
		BITRATES_ALL[:7],
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[SBA_FOA_PCA_OFF],
		)
		)
		)
		SBA_HIGH_BITRATES_PARAMS = list(
		(
		product(
		INPUT_FORMATS_SCENE_BASED,
		BITRATES_ALL[7:],
		SAMPLING_RATES_ALL,
		[DTX_OFF],
		[SBA_FOA_PCA_OFF],
		)
		)
		)
		SBA_FOA_PCA_PARAMS = list(
		product(["FOA"], [256000], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_ON])
		)
		SBA_PARAMS = SBA_LOW_BITRATES_PARAMS + SBA_HIGH_BITRATES_PARAMS + SBA_FOA_PCA_PARAMS


		# All EVS bitrates, 5900 VBR always needs -dtx
		STEREO_DMX_EVS_PARAMS = list(
		product(BITRATES_EVS[1:], SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON])
		) + list(product(BITRATES_EVS[:1], SAMPLING_RATES_ALL, [DTX_ON]))


		### -------------- decoder parameter lists --------------

		DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS = (
		collapse_into_list_of_pairs(
		product(STEREO_PARAMS, OUTPUT_FORMATS_ALL[:2], SAMPLING_RATES_ALL)
		)
		+ collapse_into_list_of_pairs(
		product(
		MC_PARAMS,
		OUTPUT_FORMATS_ALL[:10] + OUTPUT_FORMATS_BINAURAL,
		SAMPLING_RATES_ALL,
		)
		)
		+ collapse_into_list_of_pairs(
		product(MASA_PARAMS, OUTPUT_FORMATS_ALL, SAMPLING_RATES_ALL)
		)
		)

tests/codec_be_to_accepted_release/test_decoder.py

0 → 100644

+92 −0

Original line number	Diff line number	Diff line
		__copyright__ = """
		(C) 2022-2023 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository. All Rights Reserved.

		This software is protected by copyright law and by international treaties.
		The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository retain full ownership rights in their respective contributions in
		the software. This notice grants no license of any kind, including but not limited to patent
		license, nor is any license granted by implication, estoppel or otherwise.

		Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
		contributions.

		This software is provided "AS IS", without any express or implied warranties. The software is in the
		development stage. It is intended exclusively for experts who have experience with such software and
		solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
		and fitness for a particular purpose are hereby disclaimed and excluded.

		Any dispute, controversy or claim arising under or in relation to providing this software shall be
		submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		import pytest
		from . import is_be_to_ref, get_bitstream_path, get_testv_path, REF_PATH, DUT_PATH
		from .constants import *


		### --------------- Helper functions ---------------


		def get_output_path(bitstream_path, output_format, output_sampling_rate):
		output_name = (
		f"{bitstream_path.stem}.dec-{output_format}-{output_sampling_rate}kHz.wav"
		)
		return DUT_PATH.joinpath(output_name)


		def run_check(
		ref_bitstream,
		output_format,
		sampling_rate,
		dut_output,
		decoder_frontend,
		is_ref_creation,
		):
		decoder_frontend.run(output_format, sampling_rate, ref_bitstream, dut_output)

		if not is_ref_creation and not is_be_to_ref(dut_output):
		pytest.fail(f"Decoder output differs from reference")


		### --------------- Actual testcase definitions ---------------

		# clean channel decoding


		@pytest.mark.parametrize(
		"input_format,bitrate,input_sampling_rate,dtx,output_format,output_sampling_rate",
		DECODER_CLEAN_CHANNEL_CHANNELBASED_AND_MASA_PARAMS,
		)
		def test_decoder_clean_channel(
		input_format,
		bitrate,
		input_sampling_rate,
		dtx,
		output_format,
		output_sampling_rate,
		dut_decoder_frontend,
		update_ref,
		):
		testv_name = get_testv_path(input_format, input_sampling_rate).stem
		ref_bitstream = get_bitstream_path(
		REF_PATH, testv_name, input_format, bitrate, input_sampling_rate, dtx
		)
		dut_output = get_output_path(ref_bitstream, output_format, output_sampling_rate)

		run_check(
		str(ref_bitstream),
		output_format,
		output_sampling_rate,
		str(dut_output),
		dut_decoder_frontend,
		update_ref == 1,
		)

tests/codec_be_to_accepted_release/test_encoder.py

+17 −146

Original line number	Diff line number	Diff line
		@@ -29,19 +29,25 @@ the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		import pytest
		from itertools import product
		from .constants import *
		from . import TESTV_PATH, DUT_PATH, is_be_to_ref, get_bitstream_path
		from .constants import (
		METADATA_FOR_INPUT_FORMAT,
		ISM_MD_NULL,
		CMDL_OPTIONS_FOR_INPUT_FORMAT,
		ISM_MD_EXTENDED,
		INPUT_FORMATS_OBJECT_BASED,
		ENCODER_CHANNEL_BASED_AND_MASA_PARAMS,
		DTX_ON,
		ISM_PARAMS,
		SBA_PARAMS,
		SBA_FOA_PCA_ON,
		STEREO_DMX_EVS_PARAMS,
		)
		from . import TESTV_PATH, DUT_PATH, is_be_to_ref, get_bitstream_path, get_testv_path


		### --------------- Helper functions ---------------


		def get_testv_path(input_format, sampling_rate):
		testv = TESTVECTOR_FOR_INPUT_FORMAT[input_format].format(fs=sampling_rate)
		return TESTV_PATH.joinpath(testv)


		def get_md(input_format, md_type=None):
		md_files = METADATA_FOR_INPUT_FORMAT.get(input_format, list())
		if md_type == ISM_MD_NULL:
		@@ -80,25 +86,14 @@ def run_check(
		add_option_list=options,
		)

		if not is_ref_creation:
		if not is_be_to_ref(dut_bitstream):
		if not is_ref_creation and not is_be_to_ref(dut_bitstream):
		pytest.fail(f"Bitstream file differs from reference")


		### --------------- Actual testcase definitions ---------------

		# channel-based modes + MASA

		STEREO_PARAMS = list(
		product(["STEREO"], BITRATES_STEREO, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON])
		)
		MC_PARAMS = list(product(INPUT_FORMATS_MC, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF]))
		MASA_PARAMS = list(
		product(INPUT_FORMATS_MASA, BITRATES_ALL, SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON])
		)
		ENCODER_CHANNEL_BASED_AND_MASA_PARAMS = STEREO_PARAMS + MC_PARAMS + MASA_PARAMS


		# channel-based modes + MASA
		@pytest.mark.parametrize(
		"input_format,bitrate,sampling_rate,dtx", ENCODER_CHANNEL_BASED_AND_MASA_PARAMS
		)
		@@ -127,93 +122,6 @@ def test_encoder_channel_based_and_masa_modes(
		)


		# ISM modes with three "flavours" of metadata

		# different ranges of bitrates allowed depending on number of objects, ext MD only for >64kbps
		ISM1_BASIC_NULL_PARAMS = list(
		product(
		["ISM1"],
		BITRATES_ISM1,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_BASIC, ISM_MD_NULL],
		)
		)
		ISM2_BASIC_NULL_PARAMS = list(
		product(
		["ISM2"],
		BITRATES_ISM2,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_BASIC, ISM_MD_NULL],
		)
		)
		ISM3_BASIC_NULL_PARAMS = list(
		product(
		["ISM3"],
		BITRATES_ISM3,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_BASIC, ISM_MD_NULL],
		)
		)
		ISM4_BASIC_NULL_PARAMS = list(
		product(
		["ISM4"],
		BITRATES_ISM4,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_BASIC, ISM_MD_NULL],
		)
		)
		ISM1_EXTENDED_PARAMS = list(
		product(
		["ISM1"],
		BITRATES_ISM1_EXTENDED,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_EXTENDED],
		)
		)
		ISM2_EXTENDED_PARAMS = list(
		product(
		["ISM2"],
		BITRATES_ISM2_EXTENDED,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_EXTENDED],
		)
		)
		ISM3_EXTENDED_PARAMS = list(
		product(
		["ISM3"],
		BITRATES_ISM3_EXTENDED,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_EXTENDED],
		)
		)
		ISM4_EXTENDED_PARAMS = list(
		product(
		["ISM4"],
		BITRATES_ISM4_EXTENDED,
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[ISM_MD_EXTENDED],
		)
		)
		ISM_PARAMS = (
		ISM1_BASIC_NULL_PARAMS
		+ ISM2_BASIC_NULL_PARAMS
		+ ISM3_BASIC_NULL_PARAMS
		+ ISM4_BASIC_NULL_PARAMS
		+ ISM1_EXTENDED_PARAMS
		+ ISM2_EXTENDED_PARAMS
		+ ISM3_EXTENDED_PARAMS
		+ ISM4_EXTENDED_PARAMS
		)


		@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,md_type", ISM_PARAMS)
		def test_encoder_object_based_modes(
		input_format,
		@@ -245,37 +153,6 @@ def test_encoder_object_based_modes(
		)


		# SBA modes

		# dtx only allowed at low bitrates, PCA only at FOA 256kbps
		SBA_LOW_BITRATES_PARAMS = list(
		(
		product(
		INPUT_FORMATS_SCENE_BASED,
		BITRATES_ALL[:7],
		SAMPLING_RATES_ALL,
		[DTX_OFF, DTX_ON],
		[SBA_FOA_PCA_OFF],
		)
		)
		)
		SBA_HIGH_BITRATES_PARAMS = list(
		(
		product(
		INPUT_FORMATS_SCENE_BASED,
		BITRATES_ALL[7:],
		SAMPLING_RATES_ALL,
		[DTX_OFF],
		[SBA_FOA_PCA_OFF],
		)
		)
		)
		SBA_FOA_PCA_PARAMS = list(
		product(["FOA"], [256000], SAMPLING_RATES_ALL, [DTX_OFF], [SBA_FOA_PCA_ON])
		)
		SBA_PARAMS = SBA_LOW_BITRATES_PARAMS + SBA_HIGH_BITRATES_PARAMS + SBA_FOA_PCA_PARAMS


		@pytest.mark.parametrize("input_format,bitrate,sampling_rate,dtx,pca", SBA_PARAMS)
		def test_encoder_sba(
		input_format, bitrate, sampling_rate, dtx, pca, dut_encoder_frontend, update_ref
		@@ -298,12 +175,6 @@ def test_encoder_sba(
		)


		# All EVS bitrates, 5900 VBR always needs -dtx
		STEREO_DMX_EVS_PARAMS = list(
		product(BITRATES_EVS[1:], SAMPLING_RATES_ALL, [DTX_OFF, DTX_ON])
		) + list(product(BITRATES_EVS[:1], SAMPLING_RATES_ALL, [DTX_ON]))


		@pytest.mark.parametrize("bitrate,sampling_rate,dtx", STEREO_DMX_EVS_PARAMS)
		def test_encoder_stereo_dmx_evs(
		bitrate, sampling_rate, dtx, dut_encoder_frontend, update_ref