add wrapper function for comparing .stats files (0242d399) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/cmp_stats_files.py

0 → 100644

+145 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import argparse
		import os
		import sys
		import json

		import pdb

		THIS_PATH = os.path.join(os.getcwd(), __file__)
		sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts"))

		import numpy as np
		import pyaudio3dtools
		import pyivastest
		from collections import OrderedDict



		def str2num(s):
		"""
		Convert string either to integer or float
		"""
		try:
		return int(s)
		except ValueError:
		return float(s)

		def cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = 0.1, min_enc_stats_diff = 0.1) -> (int, str):
		"""
		Compare two .stats files containing encoder statistics (extracted from binary files)
		"""
		print(f"Comparing {os.path.basename(ref_stats_file)} between Ref and Dut ... \n")

		# open and read the .stats files
		with open(ref_stats_file, "r") as f_aux:
		ref_stats = json.load(f_aux)
		# create dictionary to map "name" to the corresponding dictionaries
		ref_stats_names = {d["name"]: d for d in ref_stats}

		with open(dut_stats_file, "r") as f_aux:
		dut_stats = json.load(f_aux)
		# create dictionary to map "name" to the corresponding dictionaries
		dut_stats_names = {d["name"]: d for d in dut_stats}

		# loop over all common aux files
		enc_test_result = 0
		enc_test_result_msg = ""
		max_total_num_diff = 0
		max_total_num_diff_ratio = 0
		for name in ref_stats_names:
		if name in dut_stats_names:
		# retrieve the dictionaries
		ref_stats_dict = ref_stats_names[name]
		dut_stats_dict = dut_stats_names[name]

		msg = f"File {name}"

		# compare the file lengths
		result_len_check = 0
		file_length = max(ref_stats_dict["length"], dut_stats_dict["length"])
		if ref_stats_dict["length"] != dut_stats_dict["length"]:
		msg += f" has different length between Ref {ref_stats_dict['length']} and DuT {dut_stats_dict['length']}"

		# check if threshold has been exceeded
		if abs(ref_stats_dict["length"] - dut_stats_dict["length"]) / file_length > min_enc_file_length_diff:
		result_len_check = 1

		msg += f", "

		# remove the "name" and "length" keys for further processing
		del ref_stats_dict["name"]
		del dut_stats_dict["name"]
		del ref_stats_dict["length"]
		del dut_stats_dict["length"]

		# convert keys and values from string to float
		ref_hist = {str2num(i) : str2num(j) for i,j in ref_stats_dict.items()}
		cut_hist = {str2num(i) : str2num(j) for i,j in dut_stats_dict.items()}
		delta_ref = set(cut_hist) - set(ref_hist)
		delta_cut = set(ref_hist) - set(cut_hist)

		# append missing keys
		for item in delta_cut:
		cut_hist[item] = 0

		for item in delta_ref:
		ref_hist[item] = 0

		ref_hist = dict(sorted(ref_hist.items()))
		cut_hist = dict(sorted(cut_hist.items()))

		# caculate difference of statistics
		diff_hist = { k : cut_hist[k] - ref_hist[k] for k in ref_hist.keys()}

		# calculate the total number of differences
		total_num_diff = sum(np.abs(list(diff_hist.values())))
		total_num_diff_ratio = total_num_diff / (sum(ref_hist.values()) + sum(cut_hist.values()))

		msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)"
		if total_num_diff_ratio > min_enc_stats_diff:
		result_diff_check = 1
		msg += "! "
		else:
		result_diff_check = 0
		msg += ". "

		# check if the maximum difference has been exceeded
		if total_num_diff_ratio > max_total_num_diff_ratio:
		max_total_num_diff = total_num_diff
		max_total_num_diff_ratio = total_num_diff_ratio

		# update test result
		if result_len_check or result_diff_check:
		enc_test_result = 1
		enc_test_result_msg += msg

		print(msg)

		if enc_test_result and max_total_num_diff > 0:
		msg = f"MAXIMUM ENC DIFF: {max_total_num_diff} ({(max_total_num_diff_ratio*100):.2f}%) "
		enc_test_result_msg += msg
		print(msg)

		return enc_test_result, enc_test_result_msg



		if __name__ == "__main__":
		parser = argparse.ArgumentParser()
		parser.add_argument("ref_file", type=str)
		parser.add_argument("cut_file", type=str)
		parser.add_argument("--data_type", type=str, default='int16', dest="dtype")
		parser.add_argument("--nsamples_per_frame", type=str, default=960, dest="nsamples_per_frame")
		parser.add_argument("--len_check", type=int, default=0, dest="len_check")
		parser.add_argument("--min_diff_thr", type=float, default=0.0, dest="min_diff_thr")
		args = parser.parse_args()

		# convert dtype from str
		if isinstance(args.dtype, str):
		args.dtype = np.dtype(getattr(np, args.dtype))

		result, msg = cmp_bin_files(**vars(args))
		print(msg)
		sys.exit(result)

tests/codec_be_on_mr_nonselection/test_param_file.py

+18 −106

Original line number	Diff line number	Diff line
		@@ -41,12 +41,16 @@ from pathlib import Path
		from subprocess import run
		import pytest
		import numpy as np
		import re


		from tests.cmp_pcm import cmp_pcm
		from tests.cmp_stats_files import cmp_stats_files
		from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties
		from tests.testconfig import PARAM_FILE
		from tests.constants import MIN_ENC_AUX_FILE_LENGTH_DIFF, MIN_ENC_AUX_FILE_DIFF_THR
		from tests.constants import ENC_AUX_FILES, MAX_ENC_DIFF_PATTERN, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF

		import pdb

		VALID_DEC_OUTPUT_CONF = [
		"MONO",
		@@ -131,15 +135,6 @@ def convert_test_string_to_tag(test_string):
		return tag_str


		def num(s):
		"""
		Convert string either to integer or float
		"""
		try:
		return int(s)
		except ValueError:
		return float(s)


		@pytest.mark.create_ref
		@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys()))
		@@ -172,6 +167,7 @@ def test_param_file_tests(

		tag_str = convert_test_string_to_tag(test_tag)


		# evaluate encoder options
		enc_split = enc_opts.split()
		assert len(enc_split) >= 4
		@@ -225,100 +221,16 @@ def test_param_file_tests(
		ref_stats_file = f"{reference_path}/param_file/enc/{stats_file}"
		dut_stats_file = f"{dut_base_path}/param_file/enc/{stats_file}"

		# open and read the .stats file
		with open(ref_stats_file, "r") as f_aux:
		ref_stats = json.load(f_aux)
		# create dictionary to map "name" to the corresponding dictionaries
		ref_stats_names = {d["name"]: d for d in ref_stats}

		with open(dut_stats_file, "r") as f_aux:
		dut_stats = json.load(f_aux)
		# create dictionary to map "name" to the corresponding dictionaries
		dut_stats_names = {d["name"]: d for d in dut_stats}

		# loop over all common aux files
		enc_test_result = 0
		enc_test_result_msg = ""
		max_enc_diff = 0
		for name in ref_stats_names:
		if name in dut_stats_names:
		# retrieve the dictionaries
		ref_stats_dict = ref_stats_names[name]
		dut_stats_dict = dut_stats_names[name]

		msg = f"File {name}"

		# compare the file lengths
		result_len_check = 0
		file_length = max(
		ref_stats_dict["length"], dut_stats_dict["length"]
		)
		if ref_stats_dict["length"] != dut_stats_dict["length"]:
		msg += f" has different length between Ref {ref_stats_dict['length']} and DuT {dut_stats_dict['length']}"

		# check if threshold has been exceeded
		if (
		abs(ref_stats_dict["length"] - dut_stats_dict["length"])
		/ file_length
		> MIN_ENC_AUX_FILE_LENGTH_DIFF
		):
		result_len_check = 1

		msg += ", "

		# remove the "name" and "length" keys for further processing
		del ref_stats_dict["name"]
		del dut_stats_dict["name"]
		del ref_stats_dict["length"]
		del dut_stats_dict["length"]

		# convert keys and values from string to float
		ref_hist = {num(i): num(j) for i, j in ref_stats_dict.items()}
		cut_hist = {num(i): num(j) for i, j in dut_stats_dict.items()}
		delta_ref = set(cut_hist) - set(ref_hist)
		delta_cut = set(ref_hist) - set(cut_hist)

		# append missing keys
		for item in delta_cut:
		cut_hist[item] = 0

		for item in delta_ref:
		ref_hist[item] = 0

		ref_hist = dict(sorted(ref_hist.items()))
		cut_hist = dict(sorted(cut_hist.items()))

		# caculate difference of statistics
		diff_hist = {k: cut_hist[k] - ref_hist[k] for k in ref_hist.keys()}

		# calculate the total number of differences
		total_num_diff = sum(np.abs(list(diff_hist.values())))
		total_num_diff_ratio = total_num_diff / (
		sum(ref_hist.values()) + sum(cut_hist.values())
		)

		msg += f"the total number of differences is {total_num_diff} ({(total_num_diff_ratio*100):.2f}%)"
		if total_num_diff_ratio > MIN_ENC_AUX_FILE_DIFF_THR:
		result_diff_check = 1
		msg += "! "
		else:
		result_diff_check = 0
		msg += ". "

		# check if the maximum difference has been exceeded
		if total_num_diff_ratio > max_enc_diff:
		max_enc_diff = total_num_diff_ratio

		# update test result
		if result_len_check or result_diff_check:
		enc_test_result = 1
		enc_test_result_msg += msg

		print(msg)

		print("")
		# compare ref and dut .stats files
		enc_test_result, enc_test_result_msg = cmp_stats_files(ref_stats_file, dut_stats_file, min_enc_file_length_diff = MIN_ENC_FILE_LENGTH_DIFF, min_enc_stats_diff = MIN_ENC_STATS_DIFF)

		if enc_test_result:
		# find the maximum number of differences in the test result message
		search_result = re.search(MAX_ENC_DIFF_PATTERN, enc_test_result_msg)
		if search_result:
		max_enc_diff, max_enc_diff_ratio = search_result.groups(0)
		if max_enc_diff:
		max_enc_diff = float(max_enc_diff)
		record_property("MAXIMUM ENC DIFF", max_enc_diff)
		pytest.fail(enc_test_result_msg)

tests/conftest.py

+10 −4

Original line number	Diff line number	Diff line
		@@ -230,6 +230,13 @@ def pytest_addoption(parser):
		default=0,
		)

		parser.addoption(
		"--get_enc_stats",
		help="Generate .stats file containing statistics about encoder parameters",
		action="store_true",
		default=False,
		)


		@pytest.fixture(scope="session", autouse=True)
		def update_ref(request):
		@@ -276,12 +283,11 @@ def get_ssnr(request):


		@pytest.fixture(scope="session", autouse=True)
		def get_enc_stats(request):
		def get_enc_stats(request) -> bool:
		"""
		Return indication to compare statistics of values logged from encoder.
		Return value of cmdl param --get_enc_stats
		"""
		return request.config.option.enc_stats

		return request.config.getoption("--get_enc_stats")

		@pytest.fixture(scope="session")
		def abs_tol(request) -> int:

tests/constants.py

+3 −5

Original line number	Diff line number	Diff line
		@@ -9,12 +9,10 @@ TESTV_DIR = SCRIPTS_DIR.joinpath("testv")
		MLD_PATTERN = r"MLD: ([\d\.]*)"
		MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)"
		SSNR_PATTERN = r"Channel \d* SSNR: (nan\|[+-]inf\|[\d\.])"
		MAX_ENC_DIFF_PATTERN = r"total number of differences is \d+ \((\d+\.\d+)%\)"
		MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)"

		MIN_ENC_AUX_FILE_DIFF_THR = (
		0.1 # minimum ratio of total number of differences in encoder aux file
		)
		MIN_ENC_AUX_FILE_LENGTH_DIFF = 0.1 # minimum difference of encoder aux file length
		MIN_ENC_FILE_LENGTH_DIFF = 0.1 # minimum difference between ref and dut encoder file lengths
		MIN_ENC_STATS_DIFF = 0.1 # minimum difference between the statistics of ref and dut encoder files

		# list of encoder filename patterns with their data type and number of samples per frame
		# note: instead of specifying the number of samples per frame, you can use a formula incl. 'fs', e.g. 'fs/50'