unify property recording (ef16ea02) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py

+26 −71

Original line number	Diff line number	Diff line
		@@ -35,16 +35,13 @@ __doc__ = """

		import errno
		import os
		import re
		from filecmp import cmp
		from typing import Optional

		import pytest

		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend

		from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN
		from tests.conftest import DecoderFrontend, EncoderFrontend, record_properties

		# params
		# output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT']
		@@ -98,6 +95,7 @@ def check_and_makedir(dir_path):
		)
		def test_masa_enc_dec(
		record_property,
		props_to_record,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		ref_encoder_frontend: EncoderFrontend,
		@@ -207,6 +205,7 @@ def test_masa_enc_dec(
		)

		# Compare outputs. For EXT output, also compare metadata.
		metacmp_res = True
		if output_mode == "EXT":
		# Compare metadata as binary blob
		metacmp_res = cmp(dec_met_output_ref, dec_met_output_dut)
		@@ -222,67 +221,23 @@ def test_masa_enc_dec(
		abs_tol=abs_tol,
		get_ssnr=get_ssnr
		)
		if get_mld:
		mld = re.search(MLD_PATTERN, reason).groups(1)[0]
		record_property("MLD", mld)

		if get_ssnr:
		ssnrs = re.findall(SSNR_PATTERN, reason)
		min_ssnr = min(ssnrs)
		min_ssnr_channel = ssnrs.index(min_ssnr)
		record_property("MIN_SSNR", min_ssnr)
		record_property("MIN_SSNR_CHANNEL", min_ssnr_channel)

		max_diff = 0
		if pcmcmp_res:
		search_result = re.search(MAX_DIFF_PATTERN, reason)
		max_diff = search_result.groups(1)[0]
		record_property("MAXIMUM ABS DIFF", max_diff)

		record_properties(text_to_parse=reason, output_differs=pcmcmp_res != 0, record_property=record_property, props_to_record=props_to_record)

		if get_mld and get_mld_lim > 0:
		if pcmcmp_res != 0:
		pytest.fail(reason)
		else:
		# Fail if compare fails compare result
		if metacmp_res == False and pcmcmp_res != 0:
		if not metacmp_res and pcmcmp_res != 0:
		pytest.fail("Metadata and transport output difference detected")
		elif metacmp_res == False:
		elif not metacmp_res:
		pytest.fail("Metadata output difference detected")
		elif pcmcmp_res != 0:
		pytest.fail("Transport output difference detected")
		else:
		print("Comparison bit exact")

		else:
		# Compare audio outputs
		filecmp_res = cmp(dec_output_ref, dec_output_dut)
		if filecmp_res == False:
		cmp_result, reason = cmp_pcm(
		dec_output_dut,
		dec_output_ref,
		output_mode,
		int(out_fs * 1000),
		get_mld=get_mld,
		mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr
		)
		if get_mld:
		mld = re.search(MLD_PATTERN, reason).groups(1)[0]
		record_property("MLD", mld)

		search_result = re.search(MAX_DIFF_PATTERN, reason)
		max_diff = search_result.groups(1)[0]
		record_property("MAXIMUM ABS DIFF", max_diff)

		# Report compare result
		if cmp_result != 0:
		pytest.fail(reason)
		else:
		if get_mld:
		record_property("MLD", "0")
		print("Comparison bit exact")


		#########################################################
		# -------------------- test function --------------------

tests/codec_be_on_mr_nonselection/test_param_file.py

+4 −24

Original line number	Diff line number	Diff line
		@@ -35,7 +35,6 @@ Execute tests specified via a parameter file.
		import errno
		import filecmp
		import os
		import re
		import platform
		from pathlib import Path
		from subprocess import run
		@@ -43,9 +42,9 @@ import pytest
		import numpy as np

		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend
		from tests.conftest import DecoderFrontend, EncoderFrontend, record_properties
		from tests.testconfig import PARAM_FILE
		from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN


		VALID_DEC_OUTPUT_CONF = [
		"MONO",
		@@ -136,6 +135,7 @@ def convert_test_string_to_tag(test_string):
		@pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID])
		def test_param_file_tests(
		record_property,
		props_to_record,
		decoder_only,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		@@ -354,27 +354,7 @@ def test_param_file_tests(
		)
		md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)

		if get_mld:
		mld = re.search(MLD_PATTERN, reason).groups(1)[0]
		record_property("MLD", mld)

		if get_ssnr:
		ssnrs = re.findall(SSNR_PATTERN, reason)
		min_ssnr = min(ssnrs)
		min_ssnr_channel = ssnrs.index(min_ssnr)
		record_property("MIN_SSNR", min_ssnr)
		record_property("MIN_SSNR_CHANNEL", min_ssnr_channel)

		max_diff = 0
		if output_differs:
		search_result = re.search(MAX_DIFF_PATTERN, reason)
		if search_result:
		max_diff = search_result.groups(1)[0]
		else:
		msg = "Error " + MAX_DIFF_PATTERN + " not found"
		print(msg)
		pytest.fail(msg)
		record_property("MAXIMUM ABS DIFF", max_diff)
		record_properties(text_to_parse=reason, output_differs=output_differs, record_property=record_property, props_to_record=props_to_record)

		metadata_differs = False
		for md_file in md_out_files:

tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py

+6 −18

Original line number	Diff line number	Diff line
		@@ -34,13 +34,12 @@ __doc__ = """

		import errno
		import os
		import re

		import pytest

		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend
		from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN
		from ..conftest import record_properties

		# params
		tag_list = ["stvFOA"]
		@@ -77,6 +76,7 @@ def check_and_makedir(dir_path):
		@pytest.mark.parametrize("gain_flag", gain_list)
		def test_sba_plc_system(
		record_property,
		props_to_record,
		dut_decoder_frontend: DecoderFrontend,
		test_vector_path,
		reference_path,
		@@ -116,6 +116,7 @@ def test_sba_plc_system(
		# dec
		sba_dec_plc(
		record_property,
		props_to_record,
		dut_decoder_frontend,
		test_vector_path,
		reference_path,
		@@ -141,6 +142,7 @@ def test_sba_plc_system(
		# -------------------- test function --------------------
		def sba_dec_plc(
		record_property,
		props_to_record,
		decoder_frontend,
		test_vector_path,
		reference_path,
		@@ -218,22 +220,8 @@ def sba_dec_plc(
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		)
		if get_mld:
		mld = re.search(MLD_PATTERN, reason).groups(1)[0]
		record_property("MLD", mld)

		if get_ssnr:
		ssnrs = re.findall(SSNR_PATTERN, reason)
		min_ssnr = min(ssnrs)
		min_ssnr_channel = ssnrs.index(min_ssnr)
		record_property("MIN_SSNR", min_ssnr)
		record_property("MIN_SSNR_CHANNEL", min_ssnr_channel)

		max_diff = 0
		if cmp_result:
		search_result = re.search(MAX_DIFF_PATTERN, reason)
		max_diff = search_result.groups(1)[0]
		record_property("MAXIMUM ABS DIFF", max_diff)

		record_properties(text_to_parse=reason, output_differs=cmp_result!=0, record_property=record_property, props_to_record=props_to_record)

		# report compare result
		if cmp_result != 0:

tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py

+14 −18

Original line number	Diff line number	Diff line
		@@ -35,14 +35,13 @@ __doc__ = """

		import errno
		import os
		import re

		import pytest
		from cut_bs import cut_from_start

		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend
		from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN
		from ..conftest import record_properties

		# params

		@@ -92,6 +91,7 @@ def check_and_makedir(dir_path):
		@pytest.mark.parametrize("fs", sample_rate_list)
		def test_pca_enc(
		record_property,
		props_to_record,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		test_vector_path,
		@@ -144,6 +144,7 @@ def test_pca_enc(
		# dec
		sba_dec(
		record_property,
		props_to_record,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -176,6 +177,7 @@ def test_pca_enc(
		@pytest.mark.parametrize("SID", SID_list)
		def test_sba_enc_system(
		record_property,
		props_to_record,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		test_vector_path,
		@@ -254,6 +256,7 @@ def test_sba_enc_system(
		# dec
		sba_dec(
		record_property,
		props_to_record,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -281,6 +284,7 @@ def test_sba_enc_system(
		@pytest.mark.parametrize("tag", tag_list_HOA2)
		def test_spar_hoa2_enc_system(
		record_property,
		props_to_record,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		test_vector_path,
		@@ -331,6 +335,7 @@ def test_spar_hoa2_enc_system(
		# dec
		sba_dec(
		record_property,
		props_to_record,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -358,6 +363,7 @@ def test_spar_hoa2_enc_system(
		@pytest.mark.parametrize("tag", tag_list_HOA3)
		def test_spar_hoa3_enc_system(
		record_property,
		props_to_record,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		test_vector_path,
		@@ -408,6 +414,7 @@ def test_spar_hoa3_enc_system(
		# dec
		sba_dec(
		record_property,
		props_to_record,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -437,6 +444,7 @@ def test_spar_hoa3_enc_system(
		@pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list)
		def test_sba_enc_BWforce_system(
		record_property,
		props_to_record,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		test_vector_path,
		@@ -495,6 +503,7 @@ def test_sba_enc_BWforce_system(
		# dec
		sba_dec(
		record_property,
		props_to_record,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -656,6 +665,7 @@ def sba_enc(

		def sba_dec(
		record_property,
		props_to_record,
		decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -742,22 +752,8 @@ def sba_dec(
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		)
		if get_mld:
		mld = re.search(MLD_PATTERN, reason).groups(1)[0]
		record_property("MLD", mld)

		if get_ssnr:
		ssnrs = re.findall(SSNR_PATTERN, reason)
		min_ssnr = min(ssnrs)
		min_ssnr_channel = ssnrs.index(min_ssnr)
		record_property("MIN_SSNR", min_ssnr)
		record_property("MIN_SSNR_CHANNEL", min_ssnr_channel)

		max_diff = 0
		if cmp_result:
		search_result = re.search(MAX_DIFF_PATTERN, reason)
		max_diff = search_result.groups(1)[0]
		record_property("MAXIMUM ABS DIFF", max_diff)

		record_properties(text_to_parse=reason, output_differs=cmp_result!=0, record_property=record_property, props_to_record=props_to_record)

		# report compare result
		if cmp_result != 0:

tests/conftest.py

+42 −0

Original line number	Diff line number	Diff line
		@@ -34,6 +34,7 @@ Pytest customization (configuration and fixtures) for the IVAS codec test suite.

		import logging
		import os
		import re
		from tests import testconfig
		import pytest
		import platform
		@@ -41,6 +42,7 @@ import textwrap
		from pathlib import Path
		from subprocess import TimeoutExpired, run
		from typing import Optional, Union
		from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN

		logger = logging.getLogger(__name__)
		USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature
		@@ -764,3 +766,43 @@ def pytest_configure(config):
		testconfig.MD5_REF_DICT = {
		line.split()[0]: line.split()[1] for line in f.readlines()
		}


		@pytest.fixture(scope="session")
		def props_to_record(request, get_mld, get_ssnr) -> str:
		props = ["MAXIMUM ABS DIFF"]
		if get_mld:
		props.append("MLD")
		if get_ssnr:
		props.append("SSNR")

		return props


		def record_properties(text_to_parse: str, output_differs: bool, record_property, props_to_record: list):
		"""
		Record the given properties in the report by parsing their values from the text.
		"""

		for prop in props_to_record:
		if prop == "MLD":
		mld = re.search(MLD_PATTERN, text_to_parse).groups(1)[0]
		record_property("MLD", mld)
		elif prop == "MAXIMUM ABS DIFF":
		max_diff = 0
		if output_differs:
		if (match := re.search(MAX_DIFF_PATTERN, text_to_parse)) is not None:
		max_diff = match.groups(1)[0]
		else:
		raise MaxDiffPatternNotFound()
		record_property("MAXIMUM ABS DIFF", max_diff)
		elif prop == "SSNR":
		ssnrs = re.findall(SSNR_PATTERN, text_to_parse)
		min_ssnr = min(ssnrs)
		min_ssnr_channel = ssnrs.index(min_ssnr)
		record_property("MIN_SSNR", min_ssnr)
		record_property("MIN_SSNR_CHANNEL", min_ssnr_channel)


		class MaxDiffPatternNotFound(Exception):
		pass