port encoder logging from basop repo (4bb8605f) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/codec_be_on_mr_nonselection/test_param_file.py

+138 −2

Original line number	Diff line number	Diff line
		@@ -35,6 +35,7 @@ Execute tests specified via a parameter file.
		import errno
		import filecmp
		import os
		import json
		import platform
		from pathlib import Path
		from subprocess import run
		@@ -44,6 +45,7 @@ import numpy as np
		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties
		from tests.testconfig import PARAM_FILE
		from tests.constants import MIN_ENC_AUX_FILE_LENGTH_DIFF, MIN_ENC_AUX_FILE_DIFF_THR


		VALID_DEC_OUTPUT_CONF = [
		@@ -129,6 +131,16 @@ def convert_test_string_to_tag(test_string):
		return tag_str


		def num(s):
		"""
		Convert string either to integer or float
		"""
		try:
		return int(s)
		except ValueError:
		return float(s)


		@pytest.mark.create_ref
		@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys()))
		# hack to have stv/ltv/evs in the test name
		@@ -136,6 +148,7 @@ def convert_test_string_to_tag(test_string):
		def test_param_file_tests(
		record_property,
		props_to_record,
		encoder_only,
		decoder_only,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		@@ -199,8 +212,118 @@ def test_param_file_tests(
		bitstream_file,
		enc_split,
		update_ref,
		encoder_only,
		)

		# compare binary files extracted from the encoder
		if encoder_only:
		print("Comparing encoder auxiliary files")
		print("=================================\n")

		stats_file = bitstream_file.replace(".192", ".stats")
		ref_stats_file = f"{reference_path}/param_file/enc/{stats_file}"
		dut_stats_file = f"{dut_base_path}/param_file/enc/{stats_file}"

		# open and read the .stats file
		with open(ref_stats_file, "r") as f_aux:
		ref_stats = json.load(f_aux)
		# create dictionary to map "name" to the corresponding dictionaries
		ref_stats_names = {d["name"]: d for d in ref_stats}

		with open(dut_stats_file, "r") as f_aux:
		dut_stats = json.load(f_aux)
		# create dictionary to map "name" to the corresponding dictionaries
		dut_stats_names = {d["name"]: d for d in dut_stats}

		# loop over all common aux files
		enc_test_result = 0
		enc_test_result_msg = ""
		max_enc_diff = 0
		for name in ref_stats_names:
		if name in dut_stats_names:
		# retrieve the dictionaries
		ref_stats_dict = ref_stats_names[name]
		dut_stats_dict = dut_stats_names[name]

		msg = f"File {name}"

		# compare the file lengths
		result_len_check = 0
		file_length = max(
		ref_stats_dict["length"], dut_stats_dict["length"]
		)
		if ref_stats_dict["length"] != dut_stats_dict["length"]:
		msg += f" has different length between Ref {ref_stats_dict['length']} and DuT {dut_stats_dict['length']}"

		# check if threshold has been exceeded
		if (
		abs(ref_stats_dict["length"] - dut_stats_dict["length"])
		/ file_length
		> MIN_ENC_AUX_FILE_LENGTH_DIFF
		):
		result_len_check = 1

		msg += ", "

		# remove the "name" and "length" keys for further processing
		del ref_stats_dict["name"]
		del dut_stats_dict["name"]
		del ref_stats_dict["length"]
		del dut_stats_dict["length"]

		# convert keys and values from string to float
		ref_hist = {num(i): num(j) for i, j in ref_stats_dict.items()}
		cut_hist = {num(i): num(j) for i, j in dut_stats_dict.items()}
		delta_ref = set(cut_hist) - set(ref_hist)
		delta_cut = set(ref_hist) - set(cut_hist)

		# append missing keys
		for item in delta_cut:
		cut_hist[item] = 0

		for item in delta_ref:
		ref_hist[item] = 0

		ref_hist = dict(sorted(ref_hist.items()))
		cut_hist = dict(sorted(cut_hist.items()))

		# caculate difference of statistics
		diff_hist = {k: cut_hist[k] - ref_hist[k] for k in ref_hist.keys()}

		# calculate the total number of differences
		total_num_diff = sum(np.abs(list(diff_hist.values())))
		total_num_diff_ratio = total_num_diff / (
		sum(ref_hist.values()) + sum(cut_hist.values())
		)

		msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)"
		if total_num_diff_ratio > MIN_ENC_AUX_FILE_DIFF_THR:
		result_diff_check = 1
		msg += "! "
		else:
		result_diff_check = 0
		msg += ". "

		# check if the maximum difference has been exceeded
		if total_num_diff_ratio > max_enc_diff:
		max_enc_diff = total_num_diff_ratio

		# update test result
		if result_len_check or result_diff_check:
		enc_test_result = 1
		enc_test_result_msg += msg

		print(msg)

		print("")

		if enc_test_result:
		record_property("MAXIMUM ENC DIFF", max_enc_diff)
		pytest.fail(enc_test_result_msg)

		if encoder_only:
		return

		# check for networkSimulator_g192 command line
		if sim_opts != "":
		sim_split = sim_opts.split()
		@@ -423,6 +546,7 @@ def encode(
		bitstream_file,
		enc_opts_list,
		update_ref,
		encoder_only,
		):
		"""
		Call REF and/or DUT encoder.
		@@ -434,8 +558,17 @@ def encode(
		ref_out_file = f"{ref_out_dir}/{bitstream_file}"
		dut_out_file = f"{dut_out_dir}/{bitstream_file}"

		if update_ref == 1 or update_ref == 2 and not os.path.exists(ref_out_file):
		if encoder_only:
		stats_file = bitstream_file.replace(".192", ".stats")
		ref_stats_file = f"{ref_out_dir}/{stats_file}"
		dut_stats_file = f"{dut_out_dir}/{stats_file}"
		else:
		ref_stats_file = None
		dut_stats_file = None

		if (update_ref in [1, 2] and not os.path.exists(ref_out_file)) or encoder_only:
		check_and_makedir(ref_out_dir)

		# call REF encoder
		ref_encoder_frontend.run(
		bitrate,
		@@ -443,10 +576,12 @@ def encode(
		testv_file,
		ref_out_file,
		add_option_list=enc_opts_list,
		stats_file=ref_stats_file,
		)

		if update_ref in [0, 2]:
		if update_ref in [0, 2] or encoder_only:
		check_and_makedir(dut_out_dir)

		# call DUT encoder
		dut_encoder_frontend.run(
		bitrate,
		@@ -454,6 +589,7 @@ def encode(
		testv_file,
		dut_out_file,
		add_option_list=enc_opts_list,
		stats_file=dut_stats_file,
		)

tests/conftest.py

+106 −11

Original line number	Diff line number	Diff line
		@@ -35,6 +35,7 @@ Pytest customization (configuration and fixtures) for the IVAS codec test suite.
		import logging
		import os
		import re
		import json
		from tests import testconfig
		import pytest
		import platform
		@@ -43,7 +44,8 @@ from pathlib import Path
		from subprocess import TimeoutExpired, run
		import tempfile
		from typing import Optional, Union
		from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN
		import numpy as np
		from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN, ENC_AUX_FILES

		logger = logging.getLogger(__name__)
		USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature
		@@ -201,6 +203,13 @@ def pytest_addoption(parser):
		default=20,
		)

		parser.addoption(
		"--encoder_only",
		help="Only run encoder parts of tests in 'codec_be_on_mr_nonselection'.",
		action="store_true",
		default=False,
		)

		parser.addoption(
		"--decoder_only",
		help="Only run decoder parts of tests in 'codec_be_on_mr_nonselection'. Use ref encoder output bitstreams.",
		@@ -312,13 +321,83 @@ def dut_encoder_path(request) -> str:

		class EncoderFrontend:
		def __init__(self, path, enc_type, timeout=None) -> None:
		self._path = str(Path(path).absolute())
		self._path = Path(path).absolute()
		self._type = enc_type
		self.returncode = None
		self.stdout = None
		self.stderr = None
		self.timeout = timeout

		def extract_enc_stats(
		self,
		dbg_tweak_folder,
		stats_file,
		sampling_rate,
		):
		"""
		Extract statistics from auxiliary encoder files generated by running the encoder with DEBUG_MODE_INFO.
		Write the statistics to a text file
		"""

		hist_dicts = []
		for f in ENC_AUX_FILES:
		filename = f[0]
		dtype = f[1]
		fs = int(sampling_rate) * 1000
		if isinstance(f[2], str):
		nsamples_per_frame = np.int16(eval(f[2]))
		else:
		nsamples_per_frame = np.int16(f[2])

		# aux_files = glob.glob(os.path.join(dbg_tweak_folder, filename + '\.*'))
		aux_files = [
		f
		for f in os.listdir(dbg_tweak_folder)
		if re.search(rf"^{filename}(\..*)?$", f)
		]
		# aux_files = [os.path.basename(f) for f in aux_files]

		for aux_file in aux_files:
		# extract statistics from the aux file based on histogram of values
		print(
		f"Extracting statistics from {os.path.basename(aux_file)} ... ",
		end="",
		)

		# read the aux file
		with open(os.path.join(dbg_tweak_folder, aux_file), "r") as f_aux:
		data = np.fromfile(f_aux, dtype=dtype)

		# get file length
		data_len = data.shape[0]

		# remove the duplicates of each value per frame
		if nsamples_per_frame > 1:
		data = data[::nsamples_per_frame]

		# calculate histogram from data
		unique_values = np.sort(np.unique(data))
		hist, _ = np.histogram(
		data, bins=np.append(unique_values, unique_values[-1] + 10)
		)

		# convert to dict and sort by absolute value of difference
		hist_dict = {"name": os.path.basename(aux_file), "length": data_len}
		dict_values = {
		str(unique_values[i]): str(hist[i])
		for i in range(len(unique_values))
		}
		hist_dict.update(dict_values)
		hist_dicts.append(hist_dict)

		print("DONE")

		print("")

		with open(stats_file, "w") as f_stats:
		# append the statistics to the output file in text format
		f_stats.write(json.dumps(hist_dicts, indent=2))

		def run(
		self,
		bitrate: Union[int, Path],
		@@ -332,8 +411,9 @@ class EncoderFrontend:
		quiet_mode: Optional[bool] = True,
		add_option_list: Optional[list] = None,
		run_dir: Optional[Path] = None,
		stats_file: Optional[Path] = None,
		) -> None:
		command = [self._path]
		command = [str(self._path)]

		# add optional parameters
		if sba_order is not None:
		@@ -368,9 +448,10 @@ class EncoderFrontend:
		try:
		with tempfile.TemporaryDirectory() as tmp_dir:
		if run_dir is None:
		cwd = tmp_dir
		cwd = Path(tmp_dir).absolute()
		else:
		cwd = run_dir
		cwd = Path(run_dir).absolute()

		result = run(
		command,
		capture_output=True,
		@@ -378,6 +459,12 @@ class EncoderFrontend:
		timeout=self.timeout,
		cwd=cwd,
		)

		if stats_file is not None:
		self.extract_enc_stats(
		cwd.joinpath("res"), stats_file, input_sampling_rate
		)

		except TimeoutExpired:
		pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.")

		@@ -554,9 +641,9 @@ class DecoderFrontend:
		if not os.path.exists(str(input_bitstream_path) + eid_output_suffix):
		with tempfile.TemporaryDirectory() as tmp_dir:
		if run_dir is None:
		cwd = tmp_dir
		cwd = Path(tmp_dir).absolute()
		else:
		cwd = run_dir
		cwd = Path(run_dir).absolute()
		result = run(eid_command, check=True, cwd=cwd)
		except Exception as e:
		pytest.fail(f"eid-xor operation failed! - {e}")
		@@ -595,9 +682,9 @@ class DecoderFrontend:
		try:
		with tempfile.TemporaryDirectory() as tmp_dir:
		if run_dir is None:
		cwd = tmp_dir
		cwd = Path(tmp_dir).absolute()
		else:
		cwd = run_dir
		cwd = Path(run_dir).absolute()
		run(netsim_command, check=True, cwd=cwd)
		except Exception as e:
		pytest.fail(f"netsim operation failed! - {e}")
		@@ -628,9 +715,9 @@ class DecoderFrontend:
		try:
		with tempfile.TemporaryDirectory() as tmp_dir:
		if run_dir is None:
		cwd = tmp_dir
		cwd = Path(tmp_dir).absolute()
		else:
		cwd = run_dir
		cwd = Path(run_dir).absolute()
		result = run(
		command,
		capture_output=True,
		@@ -816,6 +903,14 @@ def decoder_only(request) -> bool:
		return request.config.getoption("--decoder_only")


		@pytest.fixture(scope="session", autouse=True)
		def encoder_only(request) -> bool:
		"""
		Return value of cmdl param --encoder_only
		"""
		return request.config.getoption("--encoder_only")


		def pytest_configure(config):
		config.addinivalue_line("markers", "serial: mark test to run only in serial")
		config.addinivalue_line(

tests/constants.py

+35 −1

Original line number	Diff line number	Diff line
		import pathlib
		import numpy as np

		HERE = pathlib.Path(__file__).parent.absolute()
		SCRIPTS_DIR = HERE.parent.joinpath("scripts")
		TESTV_DIR = SCRIPTS_DIR.joinpath("testv")

		# regex patterns for parsing the output from cmp_pcm -> mainly for BASOP ci
		# regex patterns for parsing the output from comparisons -> mainly for BASOP ci
		MLD_PATTERN = r"MLD: ([\d\.]*)"
		MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)"
		SSNR_PATTERN = r"Channel \d* SSNR: (nan\|[+-]inf\|[\d\.])"
		MAX_ENC_DIFF_PATTERN = r"total number of differences is \d+ $(\d+\.\d+)%$"

		MIN_ENC_AUX_FILE_DIFF_THR = (
		0.1 # minimum ratio of total number of differences in encoder aux file
		)
		MIN_ENC_AUX_FILE_LENGTH_DIFF = 0.1 # minimum difference of encoder aux file length

		# list of encoder filename patterns with their data type and number of samples per frame
		# note: instead of specifying the number of samples per frame, you can use a formula incl. 'fs', e.g. 'fs/50'
		ENC_AUX_FILES = [
		["bits_nominal", np.int16, "fs/50"],
		["bwidth", np.int16, "fs/50"],
		["clas", np.int16, "fs/50"],
		["cng_type", np.int16, "fs/50"],
		["coder_type", np.int16, "fs/50"],
		["core", np.int16, "fs/50"],
		["core_brate", np.float32, "fs/50"],
		["count_SWB", np.int16, "fs/50"],
		["count_WB", np.int16, "fs/50"],
		["element_brate", np.float32, "fs/50"],
		["element_mode", np.int16, "fs/50"],
		["extl", np.int16, "fs/50"],
		["extl_brate", np.float32, "fs/50"],
		["ivas_total_brate", np.float32, "fs/50"],
		["L_frame", np.int16, "fs/50"],
		["localVAD", np.int16, "fs/50"],
		["sp_aud_decision0", np.int16, "fs/50"],
		["sp_aud_decision1", np.int16, "fs/50"],
		["sp_aud_decision2", np.int16, "fs/50"],
		["tdm_LRTD_flag", np.int16, "fs/50"],
		["total_brate", np.float32, "fs/50"],
		["vad_flag", np.int16, "fs/50"],
		]