record encoder diff in unified function, too (4514cc06) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/codec_be_on_mr_nonselection/test_param_file.py

+16 −11

Original line number	Diff line number	Diff line
		@@ -53,7 +53,6 @@ from tests.constants import (
		MIN_ENC_STATS_DIFF,
		)

		import pdb

		VALID_DEC_OUTPUT_CONF = [
		"MONO",
		@@ -199,6 +198,9 @@ def test_param_file_tests(
		# -> construct bitstream filename
		bitstream_file = f"{testv_base}_{tag_str}.192"

		cmp_result_msg = ""
		enc_test_result = 0

		if not decoder_only:
		encode(
		dut_encoder_frontend,
		@@ -231,17 +233,15 @@ def test_param_file_tests(
		min_enc_stats_diff=MIN_ENC_STATS_DIFF,
		)

		if enc_test_result:
		# find the maximum number of differences in the test result message
		search_result = re.search(MAX_ENC_DIFF_PATTERN, enc_test_result_msg)
		if search_result:
		max_enc_diff, max_enc_diff_ratio = search_result.groups(0)
		if max_enc_diff:
		max_enc_diff = float(max_enc_diff)
		record_property("MAXIMUM ENC DIFF", max_enc_diff)
		pytest.fail(enc_test_result_msg)
		cmp_result_msg += enc_test_result_msg

		if encoder_only:
		props = parse_properties(cmp_result_msg, False, props_to_record)
		for k, v in props.items():
		record_property(k, v)

		if enc_test_result:
		pytest.fail("Too high difference in encoder statistics found.")
		return

		# check for networkSimulator_g192 command line
		@@ -402,7 +402,9 @@ def test_param_file_tests(
		)
		md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)

		props = parse_properties(reason, output_differs, props_to_record)
		cmp_result_msg += reason

		props = parse_properties(cmp_result_msg, output_differs, props_to_record)
		for k, v in props.items():
		record_property(k, v)

		@@ -439,6 +441,9 @@ def test_param_file_tests(
		msg += "metadata only"
		pytest.fail(msg)

		if enc_test_result:
		pytest.fail("Too high difference in encoder statistics found.")

		# remove DUT output files when test result is OK (to save disk space)
		if not keep_files:
		os.remove(f"{dut_base_path}/param_file/dec/{output_file}")

tests/conftest.py

+42 −15

Original line number	Diff line number	Diff line
		@@ -45,7 +45,19 @@ from subprocess import TimeoutExpired, run
		import tempfile
		from typing import Optional, Union
		import numpy as np
		from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN, ENC_AUX_FILES, ODG_PATTERN
		from .constants import (
		MLD_PATTERN,
		MAX_DIFF_PATTERN,
		SSNR_PATTERN,
		ENC_AUX_FILES,
		ODG_PATTERN,
		MLD,
		MAX_ABS_DIFF,
		SSNR,
		ODG,
		MAX_ENC_DIFF,
		MAX_ENC_DIFF_PATTERN,
		)

		logger = logging.getLogger(__name__)
		USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature
		@@ -963,14 +975,22 @@ def pytest_configure(config):


		@pytest.fixture(scope="session")
		def props_to_record(request, get_mld, get_ssnr, get_odg) -> str:
		props = ["MAXIMUM ABS DIFF"]
		def props_to_record(
		request, get_mld, get_ssnr, get_odg, get_enc_stats, encoder_only
		) -> str:
		props = []

		if get_enc_stats:
		props.append(MAX_ENC_DIFF)

		if not encoder_only:
		props.append(MAX_ABS_DIFF)
		if get_mld:
		props.append("MLD")
		props.append(MLD)
		if get_ssnr:
		props.append("SSNR")
		props.append(SSNR)
		if get_odg:
		props.append("ODG")
		props.append(ODG)

		return props

		@@ -983,10 +1003,10 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record:
		props = dict()

		for prop in props_to_record:
		if prop == "MLD":
		if prop == MLD:
		mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0])
		props[prop] = mld
		elif prop == "MAXIMUM ABS DIFF":
		elif prop == MAX_ABS_DIFF:
		max_diff = 0
		if output_differs:
		if (match := re.search(MAX_DIFF_PATTERN, text_to_parse)) is not None:
		@@ -994,19 +1014,26 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record:
		else:
		raise MaxDiffPatternNotFound()
		props[prop] = max_diff
		elif prop == "SSNR":
		elif prop == SSNR:
		ssnrs = re.findall(SSNR_PATTERN, text_to_parse)
		min_ssnr = min(ssnrs)
		min_ssnr_channel = ssnrs.index(min_ssnr)
		props["MIN_SSNR"] = min_ssnr
		props["MIN_SSNR_CHANNEL"] = min_ssnr_channel
		elif prop == "ODG":
		elif prop == ODG:
		odgs = re.findall(ODG_PATTERN, text_to_parse)
		print(odgs)
		min_odg = min(odgs)
		min_odg_channel = odgs.index(min_odg)
		props["MIN_ODG"] = min_odg
		props["MIN_ODG_CHANNEL"] = min_odg_channel
		elif prop == MAX_ENC_DIFF:
		search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse)
		max_enc_diff = 0
		if search_result:
		max_enc_diff, max_enc_diff_ratio = search_result.groups(0)
		if max_enc_diff:
		max_enc_diff = float(max_enc_diff)
		props[MAX_ENC_DIFF] = max_enc_diff

		return props

tests/constants.py

+15 −6

Original line number	Diff line number	Diff line
		@@ -5,6 +5,15 @@ HERE = pathlib.Path(__file__).parent.absolute()
		SCRIPTS_DIR = HERE.parent.joinpath("scripts")
		TESTV_DIR = SCRIPTS_DIR.joinpath("testv")


		# Properties to record
		MLD = "MLD"
		MAX_ABS_DIFF = "MAXIMUM ABS DIFF"
		SSNR = "SSNR"
		ODG = "ODG"
		MAX_ENC_DIFF = "MAXIMUM ENC DIFF"


		# regex patterns for parsing the output from comparisons -> mainly for BASOP ci
		MLD_PATTERN = r"MLD: ([\d\.]*)"
		MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)"
		@@ -13,12 +22,12 @@ ODG_PATTERN = r"ODG: (-\d\.\d*)"
		SSNR_PATTERN = r"Channel \d* SSNR: (nan\|[+-]inf\|[\d\.])"
		MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)"

		MIN_ENC_FILE_LENGTH_DIFF = (
		0.1 # minimum difference between ref and dut encoder file lengths
		)
		MIN_ENC_STATS_DIFF = (
		0.1 # minimum difference between the statistics of ref and dut encoder files
		)

		# minimum difference between ref and dut encoder file lengths
		MIN_ENC_FILE_LENGTH_DIFF = 0.1
		# minimum difference between the statistics of ref and dut encoder files
		MIN_ENC_STATS_DIFF = 0.1


		# list of encoder filename patterns with their data type and number of samples per frame
		# note: instead of specifying the number of samples per frame, you can use a formula incl. 'fs', e.g. 'fs/50'