Merge branch '266-improve-speed-of-comparison-in-pytest-tests' into 'main' (0f6c6a6a) · Commits · IVAS Codec Public Collaboration / IVAS Codec

ci/ivas_voip_be_test.sh

+2 −2

Original line number	Diff line number	Diff line
		@@ -121,7 +121,7 @@ done
		# Assert BE between non-VoIP and VoIP modes
		all_be=1

		cmp_custom_path=$(pwd)/tests/cmp_custom.py
		cmp_tool_path=$(pwd)/tests/cmp_pcm.py

		for ref in "$output_dir_default_dec_pcm"/*; do
		cut=${ref/$output_dir_default_dec_pcm/$output_dir_voip_dec_trimmed}
		@@ -129,7 +129,7 @@ for ref in "$output_dir_default_dec_pcm"/*; do

		# Print paths of compared files, since the script doesn't do it
		printf "\nComparing %s and %s\n" "$ref" "$cut" \| tee -a voip_be_test_output.txt
		printout=$($cmp_custom_path "$ref" "$cut" 2 0)
		printout=$($cmp_tool_path "$ref" "$cut")
		if [ $? -ne 0 ]; then
		all_be=0
		fi

scripts/pyaudio3dtools/audioarray.py

+31 −11

Original line number	Diff line number	Diff line
		@@ -221,7 +221,7 @@ def cut(x: np.ndarray, limits: Tuple[int, int]) -> np.ndarray:
		return y


		def compare(ref: np.ndarray, test: np.ndarray, fs: int) -> dict:
		def compare(ref: np.ndarray, test: np.ndarray, fs: int, per_frame: bool=True) -> dict:
		"""Compare two audio arrays

		Parameters
		@@ -246,12 +246,17 @@ def compare(ref: np.ndarray, test: np.ndarray, fs: int) -> dict:
		"max_abs_diff": 0,
		"max_abs_diff_pos_sample": 0,
		"max_abs_diff_pos_channel": 0,
		"max_abs_diff_pos_frame": 0,
		"nsamples_diff": 0,
		"nsamples_diff_percentage": 0.0,
		"nframes_diff": 0,
		"nframes_diff_percentage": 0.0,
		"first_diff_pos_sample": -1,
		"first_diff_pos_channel": -1,
		"first_diff_pos_frame": -1
		}
		if per_frame:
		result["max_abs_diff_pos_frame"] = 0
		result["nframes_diff"] = 0
		result["nframes_diff_percentage"] = 0.0

		if max_diff != 0:
		if diff.ndim == 1:
		nsamples_total = diff.shape
		@@ -264,25 +269,40 @@ def compare(ref: np.ndarray, test: np.ndarray, fs: int) -> dict:
		max_diff_pos[0][0] // framesize,
		max_diff_pos[1][0],
		]

		first_diff_pos = np.nonzero(diff)
		first_diff_pos = [
		first_diff_pos[0][0],
		first_diff_pos[0][0] // framesize,
		first_diff_pos[1][0],
		]

		nsamples_diff = np.nonzero(diff)[0].size
		nsamples_diff_percentage = nsamples_diff / (nsamples_total * nchannels) * 100.0
		nframes = nsamples_total // framesize
		nframes_diff = 0
		for fr in range(nframes):
		diff_fr = diff[fr * framesize : ((fr + 1) * framesize), :]
		nframes_diff += 1 if diff_fr.nonzero()[0].size > 0 else 0
		nframes_diff_percentage = nframes_diff / nframes * 100.0

		result = {
		"bitexact": False,
		"max_abs_diff": max_diff,
		"max_abs_diff_pos_sample": max_diff_pos[0],
		"max_abs_diff_pos_channel": max_diff_pos[2],
		"max_abs_diff_pos_frame": max_diff_pos[1],
		"nsamples_diff": nsamples_diff,
		"nsamples_diff_percentage": nsamples_diff_percentage,
		"nframes_diff": nframes_diff,
		"nframes_diff_percentage": nframes_diff_percentage,
		"first_diff_pos_sample": first_diff_pos[0],
		"first_diff_pos_channel": first_diff_pos[2],
		"first_diff_pos_frame": first_diff_pos[1],
		}

		if per_frame:
		for fr in range(nframes):
		diff_fr = diff[fr * framesize : ((fr + 1) * framesize), :]
		nframes_diff += 1 if diff_fr.nonzero()[0].size > 0 else 0
		nframes_diff_percentage = nframes_diff / nframes * 100.0
		result["max_abs_diff_pos_frame"] = max_diff_pos[1]
		result["nframes_diff"] = nframes_diff
		result["nframes_diff_percentage"] = nframes_diff_percentage

		return result

tests/cmp_custom.py

deleted100755 → 0

+0 −201

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		__copyright__ = \
		"""
		(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository. All Rights Reserved.

		This software is protected by copyright law and by international treaties.
		The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository retain full ownership rights in their respective contributions in
		the software. This notice grants no license of any kind, including but not limited to patent
		license, nor is any license granted by implication, estoppel or otherwise.

		Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
		contributions.

		This software is provided "AS IS", without any express or implied warranties. The software is in the
		development stage. It is intended exclusively for experts who have experience with such software and
		solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
		and fitness for a particular purpose are hereby disclaimed and excluded.

		Any dispute, controversy or claim arising under or in relation to providing this software shall be
		submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		__doc__ = \
		"""
		Script to compare samples in 2 PCM files.

		USAGE : cmp_custom.py file_1 file_2 sample_size_in_bytes tolerance [end_samples_to_skip]
		file_1, file_2 : files to compare
		sample_size_in_bytes : 1, 2, 4, 8, these many bytes will be compared in single iteration
		tolerance : abs error tolerance, will be computed based on sample_size_in_bytes
		end_samples_to_skip : num of samples to be skipped at the end
		"""

		import sys
		import platform


		class CompareSamples:
		"""
		A class to compare PCM samples.
		"""

		def __init__(
		self,
		filename_1: str,
		filename_2: str,
		sample_size_in_bytes: int,
		tolerance: int,
		end_samples_to_skip: int,
		):
		self.file_1 = open(filename_1, "rb")
		self.file_2 = open(filename_2, "rb")
		self.sample_size = sample_size_in_bytes
		self.tolerance = tolerance
		self.end_samples_to_skip = end_samples_to_skip
		self.samples = 0
		self.max_diff = 0
		self.max_diff_sample_num = 0
		self.diff_present = False
		self.first_diff_sample_num = 0
		self.first_diff = 0
		self.file_samples_to_read = 0
		self.file_size_1_samples = 0
		self.file_size_2_samples = 0

		def get_file_sizes(self):
		"""
		Determine the file sizes in samples of the 2 PCM files.
		"""
		self.file_1.seek(0, 2)
		self.file_2.seek(0, 2)
		self.file_size_1_samples = self.file_1.tell() / self.sample_size
		self.file_size_2_samples = self.file_2.tell() / self.sample_size
		self.file_samples_to_read = (
		min(self.file_size_1_samples, self.file_size_2_samples)
		- self.end_samples_to_skip
		)
		self.file_1.seek(0)
		self.file_2.seek(0)

		def print_summary(self) -> (int, str):
		"""
		Print the summary of the comparison.
		"""
		print("Compare Custom Report")
		print("=====================")
		print(
		f"file size in samples: file 1 = {self.file_size_1_samples},",
		f"file 2 = {self.file_size_2_samples}",
		)
		if self.file_size_1_samples != self.file_size_2_samples:
		print("WARNING !!!! file size different")
		print(f"Total number of samples compared = {self.samples}")
		if not self.diff_present:
		print("Comparison success")
		print("")
		return 0, "Comparison success"

		# comparison failed
		print(
		f"First unmatched diff ==> {self.first_diff}",
		f"at sample num {self.first_diff_sample_num}",
		)
		diff_msg = f"MAXIMUM ABS DIFF ==> {self.max_diff} at sample num {self.max_diff_sample_num}"
		print(diff_msg)
		print("Comparison failed")
		print("")
		return 1, f"Comparison failed, {diff_msg}"

		def compare_next_sample(self):
		"""
		Compare the next input sample from both files.
		"""
		if self.samples == self.file_samples_to_read:
		return 1
		val1_c = self.file_1.read(self.sample_size)
		val2_c = self.file_2.read(self.sample_size)
		if (len(val1_c) != self.sample_size) or (len(val2_c) != self.sample_size):
		return 1

		val1 = int.from_bytes(val1_c, byteorder="little", signed=True)
		val2 = int.from_bytes(val2_c, byteorder="little", signed=True)

		self.samples = self.samples + 1
		abs_diff = (val1 - val2) if (val1 > val2) else (val2 - val1)
		if abs_diff > self.tolerance:
		if abs_diff > self.max_diff:
		self.max_diff = abs_diff
		self.max_diff_sample_num = self.samples
		if not self.diff_present:
		self.first_diff = abs_diff
		self.first_diff_sample_num = self.samples
		self.diff_present = True
		return 0


		def usage():
		print(__doc__)
		return 1, ""


		def cmp_custom(
		file_1_name,
		file_2_name,
		sample_size_in_bytes_str,
		tolerance_str,
		end_samples_to_skip_str="0",
		) -> (int, str):
		"""
		Function to compare the samples in 2 PCM files.
		"""

		# check for python >= 3.7
		if sys.version_info[0] < 3 or sys.version_info[1] < 7:
		sys.exit(
		"This script is written for Python >= 3.7. Found: "
		+ platform.python_version()
		)

		sample_size_in_bytes = int(sample_size_in_bytes_str)
		if sample_size_in_bytes not in [1, 2, 4, 8]:
		print(f"Error: unsupported sample size ({sample_size_in_bytes})")
		return usage()

		cmp_samples = CompareSamples(
		file_1_name,
		file_2_name,
		sample_size_in_bytes,
		int(tolerance_str),
		int(end_samples_to_skip_str),
		)

		cmp_samples.get_file_sizes()

		result = 0
		while result == 0:
		result = cmp_samples.compare_next_sample()

		return cmp_samples.print_summary()


		def main(argv) -> int:
		if len(argv) < 5:
		return usage()
		retval, _reason = cmp_custom(*argv[1:])
		return retval


		if __name__ == "__main__":
		sys.exit(main(sys.argv))

tests/cmp_pcm.py

0 → 100755

+72 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import os
		import sys
		import argparse

		THIS_PATH = os.path.join(os.getcwd(), __file__)
		sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts"))

		import pyaudio3dtools
		import pyivastest
		import numpy as np


		def cmp_pcm(file1, file2, out_config, fs) -> (int, str):
		"""
		Compare 2 PCM files for bitexactness
		"""
		print("Cmp PCM Report")
		print("=====================")

		out_config = "MONO" if out_config == "" else out_config
		if out_config.upper() not in pyivastest.constants.OC_TO_NCHANNELS:
		out_config_in_file_names = os.path.splitext(os.path.basename(out_config))[0]
		nchannels = (
		pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout(
		out_config
		)
		)
		else:
		out_config_in_file_names = out_config
		nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()]

		s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs, outdtype=np.int16)
		s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs, outdtype=np.int16)

		if s1.shape != s2.shape:
		print(
		f"file size in samples: file 1 = {s1.shape[0]},",
		f"file 2 = {s2.shape[0]}",
		)
		return 1, "FAIL: File lengths differ"

		cmp_result = pyaudio3dtools.audioarray.compare(s1, s2, fs, per_frame=False)

		if cmp_result["bitexact"]:
		return 0, "SUCCESS: Files are bitexact"
		else:
		diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)"
		first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)"
		print(diff_msg)
		print(first_msg)
		return 1, "FAIL: Files have different content"


		if __name__ == "__main__":
		parser = argparse.ArgumentParser()
		parser.add_argument("file1", type=str)
		parser.add_argument("file2", type=str)
		parser.add_argument(
		"-o",
		"--out_config",
		type=str.upper,
		default="MONO",
		choices=pyivastest.constants.OC_TO_NCHANNELS.keys(),
		)
		parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs")
		args = parser.parse_args()

		result, msg = cmp_pcm(**vars(args))
		print(msg)
		sys.exit(result)

tests/test_param_file.py

+6 −22

Original line number	Diff line number	Diff line
		@@ -39,8 +39,8 @@ import errno
		import platform
		from subprocess import run
		import pytest
		from cmp_pcm import cmp_pcm
		from cut_pcm import cut_samples
		from cmp_custom import cmp_custom
		from conftest import EncoderFrontend, DecoderFrontend
		from testconfig import PARAM_FILE

		@@ -286,12 +286,12 @@ def test_param_file_tests(
		tracefile_dec,
		)

		# compare
		if update_ref in [0, 2]:
		compare(
		f"{dut_base_path}/param_file/dec/{output_file}",
		f"{reference_path}/param_file/dec/{output_file}",
		)
		dut_file = f"{dut_base_path}/param_file/dec/{output_file}"
		ref_file = f"{reference_path}/param_file/dec/{output_file}"
		fs = int(sampling_rate) * 1000
		cmp_result, reason = cmp_pcm(dut_file, ref_file, output_config, fs)
		assert cmp_result == 0, reason

		# remove DUT output files when test result is OK (to save disk space)
		if not keep_files:
		@@ -470,19 +470,3 @@ def decode(
		dut_out_file,
		add_option_list=add_option_list,
		)


		def compare(
		pcm_file_1,
		pcm_file_2,
		):
		"""
		Compare two PCM files.
		Currently, both PCM files are treated like mono files.
		This is just fine when checking for bit-exactness.
		More advanced comparisons are possible and might come with a future update.
		"""
		sample_size = "2" # 16-bit samples
		tolerance = "0" # zero tolerance for BE testing
		cmp_result, reason = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance)
		assert cmp_result == 0, reason