Merge remote-tracking branch 'remotes/origin/main' into... (724b802b) · Commits · IVAS Codec Public Collaboration / IVAS Codec

ci/complexity_measurements/getWmops.sh

+6 −2

Original line number	Diff line number	Diff line
		@@ -73,7 +73,8 @@ config_file="scripts/config/ci_linux_ltv.json"
		# get wmops newsletter
		ivas_format_for_filename=${ivas_format// /-}
		wmopsFilenameFlcLast=wmops_newsletter_${ivas_format_for_filename}_to_${output_format}__${commit_sha}_${date}
		wmopsFilenameFlc=${destDir}/wmops/logs/${wmopsFilenameFlcLast}
		wmopsFilenameFlcDir=${destDir}/wmops/logs
		wmopsFilenameFlc=${wmopsFilenameFlcDir}/${wmopsFilenameFlcLast}

		ret_val=0

		@@ -92,6 +93,9 @@ ret_val=$?
		### WMOPS
		${scriptDir}/parseNewsletterWmops.py ${wmopsFilenameFlc}_WMOPS.csv ${wmopsFilenameFlcLast}_WMOPS.csv ${commit_sha} ${shortDate} ${fullDate} >>${destDir}/wmops/log_wmops_all.txt

		# create copy of WMOPS csv file with special name to easily get it from artifacts when comparing BASOP to float numbers
		cp ${wmopsFilenameFlc}_WMOPS.csv ${wmopsFilenameFlcDir}/latest_WMOPS.csv

		# get memory info for webpage
		### RAM
		${scriptDir}/mergeNewsletterRam.py ${wmopsFilenameFlc}_HEAP.csv ${wmopsFilenameFlc}_STACK.csv >${wmopsFilenameFlc}_RAM.csv

tests/cmp_stats_files.py

+10 −8

Original line number	Diff line number	Diff line
		@@ -18,7 +18,7 @@ def str2num(s):


		def cmp_stats_files(
		ref_stats_file, dut_stats_file, min_enc_file_length_diff=0.1, min_enc_stats_diff=0.1
		ref_stats_file, dut_stats_file, max_enc_file_length_diff, max_enc_stats_diff
		) -> (int, str):
		"""
		Compare two .stats files containing encoder statistics (extracted from binary files)
		@@ -41,6 +41,7 @@ def cmp_stats_files(
		enc_test_result_msg = ""
		max_total_num_diff = 0
		max_total_num_diff_ratio = 0
		max_diff_name = ""
		for name in ref_stats_names:
		if name in dut_stats_names:
		# retrieve the dictionaries
		@@ -59,7 +60,7 @@ def cmp_stats_files(
		if (
		abs(ref_stats_dict["length"] - dut_stats_dict["length"])
		/ file_length
		> min_enc_file_length_diff
		> max_enc_file_length_diff
		):
		result_len_check = 1

		@@ -97,7 +98,7 @@ def cmp_stats_files(
		)

		msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)"
		if total_num_diff_ratio > min_enc_stats_diff:
		if total_num_diff_ratio > max_enc_stats_diff:
		result_diff_check = 1
		msg += "! "
		else:
		@@ -108,6 +109,7 @@ def cmp_stats_files(
		if total_num_diff_ratio > max_total_num_diff_ratio:
		max_total_num_diff = total_num_diff
		max_total_num_diff_ratio = total_num_diff_ratio
		max_diff_name = name

		# update test result
		if result_len_check or result_diff_check:
		@@ -116,8 +118,8 @@ def cmp_stats_files(

		print(msg)

		if enc_test_result and max_total_num_diff > 0:
		msg = f"MAXIMUM ENC DIFF: {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%) "
		if enc_test_result or max_total_num_diff > 0:
		msg = f"MAXIMUM ENC DIFF: {max_diff_name} {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%)"
		enc_test_result_msg += msg
		print(msg)

		@@ -130,13 +132,13 @@ if __name__ == "__main__":
		parser.add_argument("ref_stats_file", type=str)
		parser.add_argument("dut_stats_file", type=str)
		parser.add_argument(
		"--min_enc_file_length_diff",
		"--max_enc_file_length_diff",
		type=float,
		default=0.1,
		dest="min_enc_file_length_diff",
		dest="max_enc_file_length_diff",
		)
		parser.add_argument(
		"--min_enc_stats_diff", type=float, default=0.1, dest="min_enc_stats_diff"
		"--max_enc_stats_diff", type=float, default=0.1, dest="max_enc_stats_diff"
		)
		args = parser.parse_args()

tests/codec_be_on_mr_nonselection/test_param_file.py

+9 −7

Original line number	Diff line number	Diff line
		@@ -47,8 +47,8 @@ from tests.cmp_stats_files import cmp_stats_files
		from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties
		from tests.testconfig import PARAM_FILE
		from tests.constants import (
		MIN_ENC_FILE_LENGTH_DIFF,
		MIN_ENC_STATS_DIFF,
		MAX_ENC_FILE_LENGTH_DIFF,
		MAX_ENC_STATS_DIFF,
		SCRIPTS_DIR,
		MAX_ENC_DIFF,
		)
		@@ -251,13 +251,15 @@ def test_param_file_tests(
		enc_test_result, enc_test_result_msg = cmp_stats_files(
		ref_stats_file,
		dut_stats_file,
		min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF,
		min_enc_stats_diff=MIN_ENC_STATS_DIFF,
		max_enc_file_length_diff=MAX_ENC_FILE_LENGTH_DIFF,
		max_enc_stats_diff=MAX_ENC_STATS_DIFF,
		)

		print("")
		cmp_result_msg += enc_test_result_msg

		# avoid double recording of the encoder diff
		if encoder_only:
		props = parse_properties(cmp_result_msg, False, [MAX_ENC_DIFF])
		for k, v in props.items():
		dut_encoder_frontend.record_property(k, v)

tests/codec_be_on_mr_nonselection/test_sba.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -42,7 +42,7 @@ from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend
		from ..conftest import parse_properties
		from ..cmp_stats_files import cmp_stats_files
		from ..constants import TESTV_DIR, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF
		from ..constants import TESTV_DIR, MAX_ENC_FILE_LENGTH_DIFF, MAX_ENC_STATS_DIFF
		from tests.testconfig import use_ltv


		@@ -309,8 +309,8 @@ def test_sba_enc_system(
		enc_test_result, enc_test_result_msg = cmp_stats_files(
		ref_stats_file,
		dut_stats_file,
		min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF,
		min_enc_stats_diff=MIN_ENC_STATS_DIFF,
		max_enc_file_length_diff=MAX_ENC_FILE_LENGTH_DIFF,
		max_enc_stats_diff=MAX_ENC_STATS_DIFF,
		)

		print("")

tests/conftest.py

+59 −24

Original line number	Diff line number	Diff line
		@@ -46,6 +46,8 @@ import tempfile
		from typing import Optional, Union
		import numpy as np
		from .constants import (
		# MAX_ENC_DIFF_NAME_PATTERN,
		MAX_ENC_DIFF_PARAM_NAME,
		MLD_PATTERN,
		MAX_DIFF_PATTERN,
		SSNR_PATTERN,
		@@ -273,6 +275,7 @@ def pytest_addoption(parser):
		default=False,
		)


		@pytest.fixture(scope="session", autouse=True)
		def update_ref(request):
		"""
		@@ -332,6 +335,7 @@ def get_odg(request):
		"""
		return request.config.option.odg


		@pytest.fixture(scope="session")
		def abs_tol(request) -> int:
		"""
		@@ -408,7 +412,9 @@ class EncoderFrontend:
		hist_dicts = []

		if not os.path.exists(dbg_tweak_folder):
		print(f"No statistics have been extracted from the res/ folder to the {stats_file} file!")
		print(
		f"No statistics have been extracted from the res/ folder to the {stats_file} file!"
		)
		else:
		for f in ENC_AUX_FILES:
		filename = f[0]
		@@ -543,7 +549,11 @@ class EncoderFrontend:
		self.stdout = result.stdout.decode("ascii")

		# Record core encoder clipping
		for (prop, pattern) in [(ENC_CORE_OVL,ENC_CORE_OVL_PATTERN),(MAX_OVL,MAX_OVL_PATTERN),(MIN_OVL,MIN_OVL_PATTERN)]:
		for prop, pattern in [
		(ENC_CORE_OVL, ENC_CORE_OVL_PATTERN),
		(MAX_OVL, MAX_OVL_PATTERN),
		(MIN_OVL, MIN_OVL_PATTERN),
		]:
		val = 0
		search_result = re.search(pattern, self.stdout)
		if search_result:
		@@ -576,7 +586,9 @@ class EncoderFrontend:


		@pytest.fixture(scope="function")
		def dut_encoder_frontend(dut_encoder_path, request, record_property) -> Union[None, EncoderFrontend]:
		def dut_encoder_frontend(
		dut_encoder_path, request, record_property
		) -> Union[None, EncoderFrontend]:
		"""
		Return a :class:`conftest.EncoderFrontend` instance as DUT for the test session.
		"""
		@@ -584,7 +596,9 @@ def dut_encoder_frontend(dut_encoder_path, request, record_property) -> Union[No

		if dut_encoder_path:
		timeout = request.config.getoption("--testcase_timeout")
		encoder = EncoderFrontend(dut_encoder_path, "DUT", record_property, timeout=timeout)
		encoder = EncoderFrontend(
		dut_encoder_path, "DUT", record_property, timeout=timeout
		)
		yield encoder

		if encoder is not None:
		@@ -623,7 +637,9 @@ def ref_encoder_path(request) -> str:


		@pytest.fixture(scope="function")
		def ref_encoder_frontend(ref_encoder_path, request, record_property) -> Union[None, EncoderFrontend]:
		def ref_encoder_frontend(
		ref_encoder_path, request, record_property
		) -> Union[None, EncoderFrontend]:
		"""
		Return a :class:`conftest.EncoderFrontend` instance as REF for the test session.
		"""
		@@ -631,7 +647,9 @@ def ref_encoder_frontend(ref_encoder_path, request, record_property) -> Union[No

		if ref_encoder_path:
		timeout = request.config.getoption("--testcase_timeout")
		encoder = EncoderFrontend(ref_encoder_path, "REF", record_property, timeout=timeout)
		encoder = EncoderFrontend(
		ref_encoder_path, "REF", record_property, timeout=timeout
		)

		yield encoder

		@@ -846,7 +864,9 @@ class DecoderFrontend:


		@pytest.fixture(scope="function")
		def dut_decoder_frontend(dut_decoder_path, request, record_property) -> Union[None, DecoderFrontend]:
		def dut_decoder_frontend(
		dut_decoder_path, request, record_property
		) -> Union[None, DecoderFrontend]:
		"""
		Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session.
		"""
		@@ -854,7 +874,13 @@ def dut_decoder_frontend(dut_decoder_path, request, record_property) -> Union[No

		if dut_decoder_path:
		timeout = request.config.getoption("--testcase_timeout")
		decoder = DecoderFrontend(dut_decoder_path, "DUT", record_property, timeout=timeout, fr=request.config.option.dut_fr)
		decoder = DecoderFrontend(
		dut_decoder_path,
		"DUT",
		record_property,
		timeout=timeout,
		fr=request.config.option.dut_fr,
		)

		yield decoder

		@@ -894,7 +920,9 @@ def ref_decoder_path(request) -> str:


		@pytest.fixture(scope="function")
		def ref_decoder_frontend(ref_decoder_path, request, record_property) -> Union[None, DecoderFrontend]:
		def ref_decoder_frontend(
		ref_decoder_path, request, record_property
		) -> Union[None, DecoderFrontend]:
		"""
		Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session.
		"""
		@@ -902,7 +930,9 @@ def ref_decoder_frontend(ref_decoder_path, request, record_property) -> Union[No

		if ref_decoder_path:
		timeout = request.config.getoption("--testcase_timeout")
		decoder = DecoderFrontend(ref_decoder_path, "REF", record_property, timeout=timeout)
		decoder = DecoderFrontend(
		ref_decoder_path, "REF", record_property, timeout=timeout
		)

		yield decoder

		@@ -959,7 +989,10 @@ def reference_path(request) -> str:

		path = str(path.resolve())

		if request.config.option.update_ref == "0" and not request.config.option.compare_to_input:
		if (
		request.config.option.update_ref == "0"
		and not request.config.option.compare_to_input
		):
		if not os.path.isdir(path):
		raise FileNotFoundError(
		f"REF path {path} not found!\nPlease generate the references, first!\n!"
		@@ -1000,6 +1033,7 @@ def encoder_only(request) -> bool:
		"""
		return request.config.getoption("--encoder_only")


		@pytest.fixture(scope="session", autouse=True)
		def compare_to_input(request) -> bool:
		"""
		@@ -1007,6 +1041,7 @@ def compare_to_input(request) -> bool:
		"""
		return request.config.getoption("--compare_to_input")


		def pytest_configure(config):
		config.addinivalue_line("markers", "serial: mark test to run only in serial")
		if config.option.param_file:
		@@ -1079,12 +1114,12 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record:
		props["MIN_ODG_CHANNEL"] = min_odg_channel
		elif prop == MAX_ENC_DIFF:
		search_result = re.search(MAX_ENC_DIFF_PATTERN, text_to_parse)
		max_enc_diff = 0
		max_enc_diff_ratio = 0.0
		max_enc_diff_param_name = ""
		if search_result:
		max_enc_diff, max_enc_diff_ratio = search_result.groups(0)
		if max_enc_diff:
		max_enc_diff = float(max_enc_diff)
		props[MAX_ENC_DIFF] = max_enc_diff
		max_enc_diff_param_name, _, max_enc_diff_ratio = search_result.groups(0)
		props[MAX_ENC_DIFF] = float(max_enc_diff_ratio)
		props[MAX_ENC_DIFF_PARAM_NAME] = max_enc_diff_param_name
		return props