diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh index 8ff091cf190ffa28ad632c12969e4d49b4c1fddb..4d780ef6c7fc982d421f1e190ce9d908e06afef4 100755 --- a/ci/ivas_voip_be_test.sh +++ b/ci/ivas_voip_be_test.sh @@ -121,7 +121,7 @@ done # Assert BE between non-VoIP and VoIP modes all_be=1 -cmp_custom_path=$(pwd)/tests/cmp_custom.py +cmp_tool_path=$(pwd)/tests/cmp_pcm.py for ref in "$output_dir_default_dec_pcm"/*; do cut=${ref/$output_dir_default_dec_pcm/$output_dir_voip_dec_trimmed} @@ -129,7 +129,7 @@ for ref in "$output_dir_default_dec_pcm"/*; do # Print paths of compared files, since the script doesn't do it printf "\nComparing %s and %s\n" "$ref" "$cut" | tee -a voip_be_test_output.txt - printout=$($cmp_custom_path "$ref" "$cut" 2 0) + printout=$($cmp_tool_path "$ref" "$cut") if [ $? -ne 0 ]; then all_be=0 fi diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 917cdf59c6537a30153bb42ba468ed1f62e8fc36..8263e8048bc8de26102fd45db97fc88e4f43c165 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -221,7 +221,7 @@ def cut(x: np.ndarray, limits: Tuple[int, int]) -> np.ndarray: return y -def compare(ref: np.ndarray, test: np.ndarray, fs: int) -> dict: +def compare(ref: np.ndarray, test: np.ndarray, fs: int, per_frame: bool=True) -> dict: """Compare two audio arrays Parameters @@ -246,12 +246,17 @@ def compare(ref: np.ndarray, test: np.ndarray, fs: int) -> dict: "max_abs_diff": 0, "max_abs_diff_pos_sample": 0, "max_abs_diff_pos_channel": 0, - "max_abs_diff_pos_frame": 0, "nsamples_diff": 0, "nsamples_diff_percentage": 0.0, - "nframes_diff": 0, - "nframes_diff_percentage": 0.0, + "first_diff_pos_sample": -1, + "first_diff_pos_channel": -1, + "first_diff_pos_frame": -1 } + if per_frame: + result["max_abs_diff_pos_frame"] = 0 + result["nframes_diff"] = 0 + result["nframes_diff_percentage"] = 0.0 + if max_diff != 0: if diff.ndim == 1: nsamples_total = diff.shape @@ -264,25 +269,40 @@ def compare(ref: np.ndarray, test: np.ndarray, fs: int) -> dict: max_diff_pos[0][0] // framesize, max_diff_pos[1][0], ] + + first_diff_pos = np.nonzero(diff) + first_diff_pos = [ + first_diff_pos[0][0], + first_diff_pos[0][0] // framesize, + first_diff_pos[1][0], + ] + nsamples_diff = np.nonzero(diff)[0].size nsamples_diff_percentage = nsamples_diff / (nsamples_total * nchannels) * 100.0 nframes = nsamples_total // framesize nframes_diff = 0 - for fr in range(nframes): - diff_fr = diff[fr * framesize : ((fr + 1) * framesize), :] - nframes_diff += 1 if diff_fr.nonzero()[0].size > 0 else 0 - nframes_diff_percentage = nframes_diff / nframes * 100.0 + result = { "bitexact": False, "max_abs_diff": max_diff, "max_abs_diff_pos_sample": max_diff_pos[0], "max_abs_diff_pos_channel": max_diff_pos[2], - "max_abs_diff_pos_frame": max_diff_pos[1], "nsamples_diff": nsamples_diff, "nsamples_diff_percentage": nsamples_diff_percentage, - "nframes_diff": nframes_diff, - "nframes_diff_percentage": nframes_diff_percentage, + "first_diff_pos_sample": first_diff_pos[0], + "first_diff_pos_channel": first_diff_pos[2], + "first_diff_pos_frame": first_diff_pos[1], } + + if per_frame: + for fr in range(nframes): + diff_fr = diff[fr * framesize : ((fr + 1) * framesize), :] + nframes_diff += 1 if diff_fr.nonzero()[0].size > 0 else 0 + nframes_diff_percentage = nframes_diff / nframes * 100.0 + result["max_abs_diff_pos_frame"] = max_diff_pos[1] + result["nframes_diff"] = nframes_diff + result["nframes_diff_percentage"] = nframes_diff_percentage + return result diff --git a/tests/cmp_custom.py b/tests/cmp_custom.py deleted file mode 100755 index ab22bc0ceb9793dc81166f29089fbd00fe75f513..0000000000000000000000000000000000000000 --- a/tests/cmp_custom.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python3 - -__copyright__ = \ -""" -(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, -Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -contributors to this repository. All Rights Reserved. - -This software is protected by copyright law and by international treaties. -The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, -Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -contributors to this repository retain full ownership rights in their respective contributions in -the software. This notice grants no license of any kind, including but not limited to patent -license, nor is any license granted by implication, estoppel or otherwise. - -Contributors are required to enter into the IVAS codec Public Collaboration agreement before making -contributions. - -This software is provided "AS IS", without any express or implied warranties. The software is in the -development stage. It is intended exclusively for experts who have experience with such software and -solely for the purpose of inspection. All implied warranties of non-infringement, merchantability -and fitness for a particular purpose are hereby disclaimed and excluded. - -Any dispute, controversy or claim arising under or in relation to providing this software shall be -submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in -accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and -the United Nations Convention on Contracts on the International Sales of Goods. -""" - -__doc__ = \ -""" -Script to compare samples in 2 PCM files. - -USAGE : cmp_custom.py file_1 file_2 sample_size_in_bytes tolerance [end_samples_to_skip] -file_1, file_2 : files to compare -sample_size_in_bytes : 1, 2, 4, 8, these many bytes will be compared in single iteration -tolerance : abs error tolerance, will be computed based on sample_size_in_bytes -end_samples_to_skip : num of samples to be skipped at the end -""" - -import sys -import platform - - -class CompareSamples: - """ - A class to compare PCM samples. - """ - - def __init__( - self, - filename_1: str, - filename_2: str, - sample_size_in_bytes: int, - tolerance: int, - end_samples_to_skip: int, - ): - self.file_1 = open(filename_1, "rb") - self.file_2 = open(filename_2, "rb") - self.sample_size = sample_size_in_bytes - self.tolerance = tolerance - self.end_samples_to_skip = end_samples_to_skip - self.samples = 0 - self.max_diff = 0 - self.max_diff_sample_num = 0 - self.diff_present = False - self.first_diff_sample_num = 0 - self.first_diff = 0 - self.file_samples_to_read = 0 - self.file_size_1_samples = 0 - self.file_size_2_samples = 0 - - def get_file_sizes(self): - """ - Determine the file sizes in samples of the 2 PCM files. - """ - self.file_1.seek(0, 2) - self.file_2.seek(0, 2) - self.file_size_1_samples = self.file_1.tell() / self.sample_size - self.file_size_2_samples = self.file_2.tell() / self.sample_size - self.file_samples_to_read = ( - min(self.file_size_1_samples, self.file_size_2_samples) - - self.end_samples_to_skip - ) - self.file_1.seek(0) - self.file_2.seek(0) - - def print_summary(self) -> (int, str): - """ - Print the summary of the comparison. - """ - print("Compare Custom Report") - print("=====================") - print( - f"file size in samples: file 1 = {self.file_size_1_samples},", - f"file 2 = {self.file_size_2_samples}", - ) - if self.file_size_1_samples != self.file_size_2_samples: - print("WARNING !!!! file size different") - print(f"Total number of samples compared = {self.samples}") - if not self.diff_present: - print("Comparison success") - print("") - return 0, "Comparison success" - - # comparison failed - print( - f"First unmatched diff ==> {self.first_diff}", - f"at sample num {self.first_diff_sample_num}", - ) - diff_msg = f"MAXIMUM ABS DIFF ==> {self.max_diff} at sample num {self.max_diff_sample_num}" - print(diff_msg) - print("Comparison failed") - print("") - return 1, f"Comparison failed, {diff_msg}" - - def compare_next_sample(self): - """ - Compare the next input sample from both files. - """ - if self.samples == self.file_samples_to_read: - return 1 - val1_c = self.file_1.read(self.sample_size) - val2_c = self.file_2.read(self.sample_size) - if (len(val1_c) != self.sample_size) or (len(val2_c) != self.sample_size): - return 1 - - val1 = int.from_bytes(val1_c, byteorder="little", signed=True) - val2 = int.from_bytes(val2_c, byteorder="little", signed=True) - - self.samples = self.samples + 1 - abs_diff = (val1 - val2) if (val1 > val2) else (val2 - val1) - if abs_diff > self.tolerance: - if abs_diff > self.max_diff: - self.max_diff = abs_diff - self.max_diff_sample_num = self.samples - if not self.diff_present: - self.first_diff = abs_diff - self.first_diff_sample_num = self.samples - self.diff_present = True - return 0 - - -def usage(): - print(__doc__) - return 1, "" - - -def cmp_custom( - file_1_name, - file_2_name, - sample_size_in_bytes_str, - tolerance_str, - end_samples_to_skip_str="0", -) -> (int, str): - """ - Function to compare the samples in 2 PCM files. - """ - - # check for python >= 3.7 - if sys.version_info[0] < 3 or sys.version_info[1] < 7: - sys.exit( - "This script is written for Python >= 3.7. Found: " - + platform.python_version() - ) - - sample_size_in_bytes = int(sample_size_in_bytes_str) - if sample_size_in_bytes not in [1, 2, 4, 8]: - print(f"Error: unsupported sample size ({sample_size_in_bytes})") - return usage() - - cmp_samples = CompareSamples( - file_1_name, - file_2_name, - sample_size_in_bytes, - int(tolerance_str), - int(end_samples_to_skip_str), - ) - - cmp_samples.get_file_sizes() - - result = 0 - while result == 0: - result = cmp_samples.compare_next_sample() - - return cmp_samples.print_summary() - - -def main(argv) -> int: - if len(argv) < 5: - return usage() - retval, _reason = cmp_custom(*argv[1:]) - return retval - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py new file mode 100755 index 0000000000000000000000000000000000000000..a54aa2cf11fc56b6b5650adbe2b19855281f5227 --- /dev/null +++ b/tests/cmp_pcm.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse + +THIS_PATH = os.path.join(os.getcwd(), __file__) +sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) + +import pyaudio3dtools +import pyivastest +import numpy as np + + +def cmp_pcm(file1, file2, out_config, fs) -> (int, str): + """ + Compare 2 PCM files for bitexactness + """ + print("Cmp PCM Report") + print("=====================") + + out_config = "MONO" if out_config == "" else out_config + if out_config.upper() not in pyivastest.constants.OC_TO_NCHANNELS: + out_config_in_file_names = os.path.splitext(os.path.basename(out_config))[0] + nchannels = ( + pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout( + out_config + ) + ) + else: + out_config_in_file_names = out_config + nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] + + s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs, outdtype=np.int16) + s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs, outdtype=np.int16) + + if s1.shape != s2.shape: + print( + f"file size in samples: file 1 = {s1.shape[0]},", + f"file 2 = {s2.shape[0]}", + ) + return 1, "FAIL: File lengths differ" + + cmp_result = pyaudio3dtools.audioarray.compare(s1, s2, fs, per_frame=False) + + if cmp_result["bitexact"]: + return 0, "SUCCESS: Files are bitexact" + else: + diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" + first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" + print(diff_msg) + print(first_msg) + return 1, "FAIL: Files have different content" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("file1", type=str) + parser.add_argument("file2", type=str) + parser.add_argument( + "-o", + "--out_config", + type=str.upper, + default="MONO", + choices=pyivastest.constants.OC_TO_NCHANNELS.keys(), + ) + parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs") + args = parser.parse_args() + + result, msg = cmp_pcm(**vars(args)) + print(msg) + sys.exit(result) diff --git a/tests/test_param_file.py b/tests/test_param_file.py index 8f28d2f6d60fb05927b975bea510eb3b01642d8d..af63df11318fe6f6f2d22b09185f625233f97da8 100644 --- a/tests/test_param_file.py +++ b/tests/test_param_file.py @@ -39,8 +39,8 @@ import errno import platform from subprocess import run import pytest +from cmp_pcm import cmp_pcm from cut_pcm import cut_samples -from cmp_custom import cmp_custom from conftest import EncoderFrontend, DecoderFrontend from testconfig import PARAM_FILE @@ -286,12 +286,12 @@ def test_param_file_tests( tracefile_dec, ) - # compare if update_ref in [0, 2]: - compare( - f"{dut_base_path}/param_file/dec/{output_file}", - f"{reference_path}/param_file/dec/{output_file}", - ) + dut_file = f"{dut_base_path}/param_file/dec/{output_file}" + ref_file = f"{reference_path}/param_file/dec/{output_file}" + fs = int(sampling_rate) * 1000 + cmp_result, reason = cmp_pcm(dut_file, ref_file, output_config, fs) + assert cmp_result == 0, reason # remove DUT output files when test result is OK (to save disk space) if not keep_files: @@ -470,19 +470,3 @@ def decode( dut_out_file, add_option_list=add_option_list, ) - - -def compare( - pcm_file_1, - pcm_file_2, -): - """ - Compare two PCM files. - Currently, both PCM files are treated like mono files. - This is just fine when checking for bit-exactness. - More advanced comparisons are possible and might come with a future update. - """ - sample_size = "2" # 16-bit samples - tolerance = "0" # zero tolerance for BE testing - cmp_result, reason = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance) - assert cmp_result == 0, reason diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index 4ac4f761e631b9807931fb5ae4f240d90f4629b5..eb7ad72a36f1c7ed228c259271bff1256f7e4ced 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -38,7 +38,7 @@ import os import errno import pytest -from cmp_custom import cmp_custom +from cmp_pcm import cmp_pcm from conftest import DecoderFrontend # params @@ -151,12 +151,13 @@ def sba_dec_plc( dut_out_raw = f"{dut_out_dir}/{plc_tag_out}.raw" ref_out_raw = f"{ref_out_dir}/{plc_tag_out}.raw" + output_config = "FOA" if ref_decoder_path: ref_decoder = DecoderFrontend(ref_decoder_path, "REF") # call REF decoder ref_decoder.run( - "FOA", + output_config, sampling_rate, ref_in_pkt, ref_out_raw, @@ -166,7 +167,7 @@ def sba_dec_plc( if update_ref == 0: # call DUT decoder decoder_frontend.run( - "FOA", + output_config, sampling_rate, ref_in_pkt_dutenc, dut_out_raw, @@ -174,16 +175,8 @@ def sba_dec_plc( ) # -------------- compare cmd -------------- - - end_skip_samples = '0' - - cmp_result, reason = cmp_custom( - dut_out_raw, - ref_out_raw, - "2", - AbsTol, - end_skip_samples - ) + fs = int(sampling_rate) * 1000 + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs) # report compare result assert cmp_result == 0, reason diff --git a/tests/test_sba_bs_enc.py b/tests/test_sba_bs_enc.py index 5fba38131314da2dc207f7ab492df4bcb37eb520..6f088f0e457592766b21d18219113f08f268ab9e 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/test_sba_bs_enc.py @@ -39,7 +39,7 @@ import os import errno import pytest -from cmp_custom import cmp_custom +from cmp_pcm import cmp_pcm from cut_pcm import cut_samples from conftest import EncoderFrontend, DecoderFrontend @@ -593,17 +593,8 @@ def sba_dec( dut_out_raw, ) - # -------------- compare cmd -------------- - - end_skip_samples = '0' - - cmp_result, reason = cmp_custom( - dut_out_raw, - ref_out_raw, - "2", - AbsTol, - end_skip_samples - ) + fs = int(sampling_rate) * 1000 + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs) # report compare result assert cmp_result == 0, reason