From dc847d1f2b902f07936c739b9e8f1a608af15302 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 9 Jan 2023 10:02:08 +0100 Subject: [PATCH 01/12] add new pcm comparison code --- tests/cmp_pcm.py | 34 ++++++++++++++++++++++++++++++++++ tests/test_sba_bs_enc.py | 16 ++++------------ tests/testconfig.py | 21 +++++++++++++++++++++ 3 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 tests/cmp_pcm.py diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py new file mode 100644 index 0000000000..f2d5c63b4f --- /dev/null +++ b/tests/cmp_pcm.py @@ -0,0 +1,34 @@ +import os +import sys + +THIS_PATH = os.path.join(os.getcwd(), __file__) +sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) + +import pyaudio3dtools + + +def cmp_pcm(file1, file2, nchannels, fs) -> (int, str): + """ + Compare 2 PCM files for bitexactness + """ + print("Cmp PCM Report") + print("=====================") + + s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs) + s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs) + + if s1.shape != s2.shape: + print( + f"file size in samples: file 1 = {s1.shape[0]},", + f"file 2 = {s2.shape[0]}", + ) + return 1, "FAIL: File lengths differ" + + cmp_result = pyaudio3dtools.audioarray.compare(s1, s2, fs) + + if cmp_result["bitexact"]: + return 0, "SUCCESS: Files are bitexact" + else: + diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']}" + print(diff_msg) + return 1, "FAIL: Files have different content" diff --git a/tests/test_sba_bs_enc.py b/tests/test_sba_bs_enc.py index 5fba381313..6fc0c71ad0 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/test_sba_bs_enc.py @@ -39,9 +39,10 @@ import os import errno import pytest -from cmp_custom import cmp_custom +from cmp_pcm import cmp_pcm from cut_pcm import cut_samples from conftest import EncoderFrontend, DecoderFrontend +from testconfig import OC_TO_NCHANNELS # params tag_list = ['stvFOA'] @@ -593,17 +594,8 @@ def sba_dec( dut_out_raw, ) - # -------------- compare cmd -------------- - - end_skip_samples = '0' - - cmp_result, reason = cmp_custom( - dut_out_raw, - ref_out_raw, - "2", - AbsTol, - end_skip_samples - ) + fs = int(sampling_rate) * 1000 + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, OC_TO_NCHANNELS[output_config], fs) # report compare result assert cmp_result == 0, reason diff --git a/tests/testconfig.py b/tests/testconfig.py index f4827a004f..0d84555366 100644 --- a/tests/testconfig.py +++ b/tests/testconfig.py @@ -35,3 +35,24 @@ To configure test modules. """ PARAM_FILE = "scripts/config/self_test.prm" +OC_TO_NCHANNELS = { + "MONO": 1, + "STEREO": 2, + "BINAURAL": 2, + "BINAURAL_ROOM": 2, + "5_1": 6, + "7_1": 8, + "5_1_2": 8, + "5_1_4": 10, + "7_1_4": 12, + "FOA": 4, + "HOA2": 9, + "HOA3": 16, + "EXT": 1, + "ISM1": 1, + "ISM2": 2, + "ISM3": 3, + "ISM4": 4, + "MASA1TC": 1, + "MASA2TC": 2, +} -- GitLab From 74122691dcf501fad75d8bd06776934685c072c5 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 9 Jan 2023 10:31:51 +0100 Subject: [PATCH 02/12] add new pcm comparison to all files --- tests/cmp_custom.py | 201 ----------------------------------- tests/test_param_file.py | 30 ++---- tests/test_sba_bs_dec_plc.py | 14 +-- 3 files changed, 10 insertions(+), 235 deletions(-) delete mode 100755 tests/cmp_custom.py diff --git a/tests/cmp_custom.py b/tests/cmp_custom.py deleted file mode 100755 index ab22bc0ceb..0000000000 --- a/tests/cmp_custom.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python3 - -__copyright__ = \ -""" -(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, -Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -contributors to this repository. All Rights Reserved. - -This software is protected by copyright law and by international treaties. -The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, -Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -contributors to this repository retain full ownership rights in their respective contributions in -the software. This notice grants no license of any kind, including but not limited to patent -license, nor is any license granted by implication, estoppel or otherwise. - -Contributors are required to enter into the IVAS codec Public Collaboration agreement before making -contributions. - -This software is provided "AS IS", without any express or implied warranties. The software is in the -development stage. It is intended exclusively for experts who have experience with such software and -solely for the purpose of inspection. All implied warranties of non-infringement, merchantability -and fitness for a particular purpose are hereby disclaimed and excluded. - -Any dispute, controversy or claim arising under or in relation to providing this software shall be -submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in -accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and -the United Nations Convention on Contracts on the International Sales of Goods. -""" - -__doc__ = \ -""" -Script to compare samples in 2 PCM files. - -USAGE : cmp_custom.py file_1 file_2 sample_size_in_bytes tolerance [end_samples_to_skip] -file_1, file_2 : files to compare -sample_size_in_bytes : 1, 2, 4, 8, these many bytes will be compared in single iteration -tolerance : abs error tolerance, will be computed based on sample_size_in_bytes -end_samples_to_skip : num of samples to be skipped at the end -""" - -import sys -import platform - - -class CompareSamples: - """ - A class to compare PCM samples. - """ - - def __init__( - self, - filename_1: str, - filename_2: str, - sample_size_in_bytes: int, - tolerance: int, - end_samples_to_skip: int, - ): - self.file_1 = open(filename_1, "rb") - self.file_2 = open(filename_2, "rb") - self.sample_size = sample_size_in_bytes - self.tolerance = tolerance - self.end_samples_to_skip = end_samples_to_skip - self.samples = 0 - self.max_diff = 0 - self.max_diff_sample_num = 0 - self.diff_present = False - self.first_diff_sample_num = 0 - self.first_diff = 0 - self.file_samples_to_read = 0 - self.file_size_1_samples = 0 - self.file_size_2_samples = 0 - - def get_file_sizes(self): - """ - Determine the file sizes in samples of the 2 PCM files. - """ - self.file_1.seek(0, 2) - self.file_2.seek(0, 2) - self.file_size_1_samples = self.file_1.tell() / self.sample_size - self.file_size_2_samples = self.file_2.tell() / self.sample_size - self.file_samples_to_read = ( - min(self.file_size_1_samples, self.file_size_2_samples) - - self.end_samples_to_skip - ) - self.file_1.seek(0) - self.file_2.seek(0) - - def print_summary(self) -> (int, str): - """ - Print the summary of the comparison. - """ - print("Compare Custom Report") - print("=====================") - print( - f"file size in samples: file 1 = {self.file_size_1_samples},", - f"file 2 = {self.file_size_2_samples}", - ) - if self.file_size_1_samples != self.file_size_2_samples: - print("WARNING !!!! file size different") - print(f"Total number of samples compared = {self.samples}") - if not self.diff_present: - print("Comparison success") - print("") - return 0, "Comparison success" - - # comparison failed - print( - f"First unmatched diff ==> {self.first_diff}", - f"at sample num {self.first_diff_sample_num}", - ) - diff_msg = f"MAXIMUM ABS DIFF ==> {self.max_diff} at sample num {self.max_diff_sample_num}" - print(diff_msg) - print("Comparison failed") - print("") - return 1, f"Comparison failed, {diff_msg}" - - def compare_next_sample(self): - """ - Compare the next input sample from both files. - """ - if self.samples == self.file_samples_to_read: - return 1 - val1_c = self.file_1.read(self.sample_size) - val2_c = self.file_2.read(self.sample_size) - if (len(val1_c) != self.sample_size) or (len(val2_c) != self.sample_size): - return 1 - - val1 = int.from_bytes(val1_c, byteorder="little", signed=True) - val2 = int.from_bytes(val2_c, byteorder="little", signed=True) - - self.samples = self.samples + 1 - abs_diff = (val1 - val2) if (val1 > val2) else (val2 - val1) - if abs_diff > self.tolerance: - if abs_diff > self.max_diff: - self.max_diff = abs_diff - self.max_diff_sample_num = self.samples - if not self.diff_present: - self.first_diff = abs_diff - self.first_diff_sample_num = self.samples - self.diff_present = True - return 0 - - -def usage(): - print(__doc__) - return 1, "" - - -def cmp_custom( - file_1_name, - file_2_name, - sample_size_in_bytes_str, - tolerance_str, - end_samples_to_skip_str="0", -) -> (int, str): - """ - Function to compare the samples in 2 PCM files. - """ - - # check for python >= 3.7 - if sys.version_info[0] < 3 or sys.version_info[1] < 7: - sys.exit( - "This script is written for Python >= 3.7. Found: " - + platform.python_version() - ) - - sample_size_in_bytes = int(sample_size_in_bytes_str) - if sample_size_in_bytes not in [1, 2, 4, 8]: - print(f"Error: unsupported sample size ({sample_size_in_bytes})") - return usage() - - cmp_samples = CompareSamples( - file_1_name, - file_2_name, - sample_size_in_bytes, - int(tolerance_str), - int(end_samples_to_skip_str), - ) - - cmp_samples.get_file_sizes() - - result = 0 - while result == 0: - result = cmp_samples.compare_next_sample() - - return cmp_samples.print_summary() - - -def main(argv) -> int: - if len(argv) < 5: - return usage() - retval, _reason = cmp_custom(*argv[1:]) - return retval - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/tests/test_param_file.py b/tests/test_param_file.py index 4031fd36bf..d92cd820b7 100644 --- a/tests/test_param_file.py +++ b/tests/test_param_file.py @@ -39,9 +39,9 @@ import errno import platform from subprocess import run import pytest -from cmp_custom import cmp_custom +from cmp_pcm import cmp_pcm from conftest import EncoderFrontend, DecoderFrontend -from testconfig import PARAM_FILE +from testconfig import PARAM_FILE, OC_TO_NCHANNELS VALID_DEC_OUTPUT_CONF = [ @@ -271,12 +271,12 @@ def test_param_file_tests( tracefile_dec, ) - # compare if update_ref in [0, 2]: - compare( - f"{dut_base_path}/param_file/dec/{output_file}", - f"{reference_path}/param_file/dec/{output_file}", - ) + dut_file = f"{dut_base_path}/param_file/dec/{output_file}" + ref_file = f"{reference_path}/param_file/dec/{output_file}" + fs = int(sampling_rate) * 1000 + cmp_result, reason = cmp_pcm(dut_file, ref_file, OC_TO_NCHANNELS[output_config], fs) + assert cmp_result == 0, reason # remove DUT output files when test result is OK (to save disk space) if not keep_files: @@ -438,19 +438,3 @@ def decode( dut_out_file, add_option_list=add_option_list, ) - - -def compare( - pcm_file_1, - pcm_file_2, -): - """ - Compare two PCM files. - Currently, both PCM files are treated like mono files. - This is just fine when checking for bit-exactness. - More advanced comparisons are possible and might come with a future update. - """ - sample_size = "2" # 16-bit samples - tolerance = "0" # zero tolerance for BE testing - cmp_result, reason = cmp_custom(pcm_file_1, pcm_file_2, sample_size, tolerance) - assert cmp_result == 0, reason diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index 4ac4f761e6..c9289ec863 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -38,7 +38,7 @@ import os import errno import pytest -from cmp_custom import cmp_custom +from cmp_pcm import cmp_pcm from conftest import DecoderFrontend # params @@ -174,16 +174,8 @@ def sba_dec_plc( ) # -------------- compare cmd -------------- - - end_skip_samples = '0' - - cmp_result, reason = cmp_custom( - dut_out_raw, - ref_out_raw, - "2", - AbsTol, - end_skip_samples - ) + fs = int(sampling_rate) * 1000 + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, OC_TO_NCHANNELS[output_config], fs) # report compare result assert cmp_result == 0, reason -- GitLab From 26af0823c350ebf8c5f1014997c13678440623db Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 9 Jan 2023 10:52:04 +0100 Subject: [PATCH 03/12] add missing import --- tests/test_sba_bs_dec_plc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index c9289ec863..340600dd7a 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -40,6 +40,7 @@ import pytest from cmp_pcm import cmp_pcm from conftest import DecoderFrontend +from testconfig import OC_TO_NCHANNELS # params tag_list = ['stvFOA'] -- GitLab From b08c69a37b8181d64cc58735e56f72c9aa1f1ab1 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 9 Jan 2023 11:23:27 +0100 Subject: [PATCH 04/12] fix undeclared variable error --- tests/test_sba_bs_dec_plc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index 340600dd7a..61db443bd3 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -152,12 +152,13 @@ def sba_dec_plc( dut_out_raw = f"{dut_out_dir}/{plc_tag_out}.raw" ref_out_raw = f"{ref_out_dir}/{plc_tag_out}.raw" + output_config = "FOA" if ref_decoder_path: ref_decoder = DecoderFrontend(ref_decoder_path, "REF") # call REF decoder ref_decoder.run( - "FOA", + output_config, sampling_rate, ref_in_pkt, ref_out_raw, @@ -167,7 +168,7 @@ def sba_dec_plc( if update_ref == 0: # call DUT decoder decoder_frontend.run( - "FOA", + output_config, sampling_rate, ref_in_pkt_dutenc, dut_out_raw, -- GitLab From d6d6812b8b47832564cdc503fefa9adb2fd391dd Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 9 Jan 2023 12:06:31 +0100 Subject: [PATCH 05/12] fix out_config->nchannels parsing --- tests/cmp_pcm.py | 11 ++++++++++- tests/test_param_file.py | 4 ++-- tests/test_sba_bs_dec_plc.py | 3 +-- tests/test_sba_bs_enc.py | 3 +-- tests/testconfig.py | 21 --------------------- 5 files changed, 14 insertions(+), 28 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index f2d5c63b4f..9d62cd6114 100644 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -5,15 +5,24 @@ THIS_PATH = os.path.join(os.getcwd(), __file__) sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) import pyaudio3dtools +import pyivastest -def cmp_pcm(file1, file2, nchannels, fs) -> (int, str): +def cmp_pcm(file1, file2, oc, fs) -> (int, str): """ Compare 2 PCM files for bitexactness """ print("Cmp PCM Report") print("=====================") + oc = "MONO" if oc == "" else oc + if oc.upper() not in pyivastest.constants.OC_TO_NCHANNELS: + oc_in_file_names = os.path.splitext(os.path.basename(oc))[0] + nchannels = pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout(oc) + else: + oc_in_file_names = oc + nchannels = pyivastest.constants.OC_TO_NCHANNELS[oc.upper()] + s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs) s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs) diff --git a/tests/test_param_file.py b/tests/test_param_file.py index d92cd820b7..1cc4b2946a 100644 --- a/tests/test_param_file.py +++ b/tests/test_param_file.py @@ -41,7 +41,7 @@ from subprocess import run import pytest from cmp_pcm import cmp_pcm from conftest import EncoderFrontend, DecoderFrontend -from testconfig import PARAM_FILE, OC_TO_NCHANNELS +from testconfig import PARAM_FILE VALID_DEC_OUTPUT_CONF = [ @@ -275,7 +275,7 @@ def test_param_file_tests( dut_file = f"{dut_base_path}/param_file/dec/{output_file}" ref_file = f"{reference_path}/param_file/dec/{output_file}" fs = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm(dut_file, ref_file, OC_TO_NCHANNELS[output_config], fs) + cmp_result, reason = cmp_pcm(dut_file, ref_file, output_config, fs) assert cmp_result == 0, reason # remove DUT output files when test result is OK (to save disk space) diff --git a/tests/test_sba_bs_dec_plc.py b/tests/test_sba_bs_dec_plc.py index 61db443bd3..eb7ad72a36 100644 --- a/tests/test_sba_bs_dec_plc.py +++ b/tests/test_sba_bs_dec_plc.py @@ -40,7 +40,6 @@ import pytest from cmp_pcm import cmp_pcm from conftest import DecoderFrontend -from testconfig import OC_TO_NCHANNELS # params tag_list = ['stvFOA'] @@ -177,7 +176,7 @@ def sba_dec_plc( # -------------- compare cmd -------------- fs = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, OC_TO_NCHANNELS[output_config], fs) + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs) # report compare result assert cmp_result == 0, reason diff --git a/tests/test_sba_bs_enc.py b/tests/test_sba_bs_enc.py index 6fc0c71ad0..6f088f0e45 100644 --- a/tests/test_sba_bs_enc.py +++ b/tests/test_sba_bs_enc.py @@ -42,7 +42,6 @@ import pytest from cmp_pcm import cmp_pcm from cut_pcm import cut_samples from conftest import EncoderFrontend, DecoderFrontend -from testconfig import OC_TO_NCHANNELS # params tag_list = ['stvFOA'] @@ -595,7 +594,7 @@ def sba_dec( ) fs = int(sampling_rate) * 1000 - cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, OC_TO_NCHANNELS[output_config], fs) + cmp_result, reason = cmp_pcm(dut_out_raw, ref_out_raw, output_config, fs) # report compare result assert cmp_result == 0, reason diff --git a/tests/testconfig.py b/tests/testconfig.py index 0d84555366..f4827a004f 100644 --- a/tests/testconfig.py +++ b/tests/testconfig.py @@ -35,24 +35,3 @@ To configure test modules. """ PARAM_FILE = "scripts/config/self_test.prm" -OC_TO_NCHANNELS = { - "MONO": 1, - "STEREO": 2, - "BINAURAL": 2, - "BINAURAL_ROOM": 2, - "5_1": 6, - "7_1": 8, - "5_1_2": 8, - "5_1_4": 10, - "7_1_4": 12, - "FOA": 4, - "HOA2": 9, - "HOA3": 16, - "EXT": 1, - "ISM1": 1, - "ISM2": 2, - "ISM3": 3, - "ISM4": 4, - "MASA1TC": 1, - "MASA2TC": 2, -} -- GitLab From a678ab5fc93150befc6221ccced9d920ff86b02a Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 9 Jan 2023 16:44:05 +0100 Subject: [PATCH 06/12] add argument for skipping frame-wise diff --- scripts/pyaudio3dtools/audioarray.py | 28 +++++++++++++++++----------- tests/cmp_pcm.py | 2 +- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 917cdf59c6..1221e25960 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -221,7 +221,7 @@ def cut(x: np.ndarray, limits: Tuple[int, int]) -> np.ndarray: return y -def compare(ref: np.ndarray, test: np.ndarray, fs: int) -> dict: +def compare(ref: np.ndarray, test: np.ndarray, fs: int, per_frame: bool=True) -> dict: """Compare two audio arrays Parameters @@ -246,12 +246,14 @@ def compare(ref: np.ndarray, test: np.ndarray, fs: int) -> dict: "max_abs_diff": 0, "max_abs_diff_pos_sample": 0, "max_abs_diff_pos_channel": 0, - "max_abs_diff_pos_frame": 0, "nsamples_diff": 0, "nsamples_diff_percentage": 0.0, - "nframes_diff": 0, - "nframes_diff_percentage": 0.0, } + if per_frame: + result["max_abs_diff_pos_frame"] = 0 + result["nframes_diff"] = 0 + result["nframes_diff_percentage"] = 0.0 + if max_diff != 0: if diff.ndim == 1: nsamples_total = diff.shape @@ -268,21 +270,25 @@ def compare(ref: np.ndarray, test: np.ndarray, fs: int) -> dict: nsamples_diff_percentage = nsamples_diff / (nsamples_total * nchannels) * 100.0 nframes = nsamples_total // framesize nframes_diff = 0 - for fr in range(nframes): - diff_fr = diff[fr * framesize : ((fr + 1) * framesize), :] - nframes_diff += 1 if diff_fr.nonzero()[0].size > 0 else 0 - nframes_diff_percentage = nframes_diff / nframes * 100.0 + result = { "bitexact": False, "max_abs_diff": max_diff, "max_abs_diff_pos_sample": max_diff_pos[0], "max_abs_diff_pos_channel": max_diff_pos[2], - "max_abs_diff_pos_frame": max_diff_pos[1], "nsamples_diff": nsamples_diff, "nsamples_diff_percentage": nsamples_diff_percentage, - "nframes_diff": nframes_diff, - "nframes_diff_percentage": nframes_diff_percentage, } + + if per_frame: + for fr in range(nframes): + diff_fr = diff[fr * framesize : ((fr + 1) * framesize), :] + nframes_diff += 1 if diff_fr.nonzero()[0].size > 0 else 0 + nframes_diff_percentage = nframes_diff / nframes * 100.0 + result["max_abs_diff_pos_frame"] = max_diff_pos[1] + result["nframes_diff"] = nframes_diff + result["nframes_diff_percentage"] = nframes_diff_percentage + return result diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 9d62cd6114..e8afb46cbc 100644 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -33,7 +33,7 @@ def cmp_pcm(file1, file2, oc, fs) -> (int, str): ) return 1, "FAIL: File lengths differ" - cmp_result = pyaudio3dtools.audioarray.compare(s1, s2, fs) + cmp_result = pyaudio3dtools.audioarray.compare(s1, s2, fs, per_frame=False) if cmp_result["bitexact"]: return 0, "SUCCESS: Files are bitexact" -- GitLab From 871dbfe006147de36d65bd94478dec619552b558 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 10 Jan 2023 16:12:24 +0100 Subject: [PATCH 07/12] use cmp_pcm in voip be test --- ci/ivas_voip_be_test.sh | 4 ++-- tests/cmp_pcm.py | 28 ++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/ci/ivas_voip_be_test.sh b/ci/ivas_voip_be_test.sh index d65677b498..67e78bbd8f 100755 --- a/ci/ivas_voip_be_test.sh +++ b/ci/ivas_voip_be_test.sh @@ -88,7 +88,7 @@ done # Assert BE between non-VoIP and VoIP modes all_be=1 -cmp_custom_path=$(pwd)/tests/cmp_custom.py +cmp_tool_path=$(pwd)/tests/cmp_pcm.py for ref in "$output_dir_default_dec_pcm"/*; do cut=${ref/$output_dir_default_dec_pcm/$output_dir_voip_dec_trimmed} @@ -96,7 +96,7 @@ for ref in "$output_dir_default_dec_pcm"/*; do # Print paths of compared files, since the script doesn't do it printf "\nComparing %s and %s\n" "$ref" "$cut" | tee -a voip_be_test_output.txt - printout=$($cmp_custom_path "$ref" "$cut" 2 0) + printout=$($cmp_tool_path "$ref" "$cut") if [ $? -ne 0 ]; then all_be=0 fi diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index e8afb46cbc..40d165c040 100644 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -1,5 +1,6 @@ import os import sys +import argparse THIS_PATH = os.path.join(os.getcwd(), __file__) sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) @@ -8,20 +9,20 @@ import pyaudio3dtools import pyivastest -def cmp_pcm(file1, file2, oc, fs) -> (int, str): +def cmp_pcm(file1, file2, out_config, fs) -> (int, str): """ Compare 2 PCM files for bitexactness """ print("Cmp PCM Report") print("=====================") - oc = "MONO" if oc == "" else oc - if oc.upper() not in pyivastest.constants.OC_TO_NCHANNELS: - oc_in_file_names = os.path.splitext(os.path.basename(oc))[0] - nchannels = pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout(oc) + out_config = "MONO" if out_config == "" else out_config + if out_config.upper() not in pyivastest.constants.OC_TO_NCHANNELS: + out_config_in_file_names = os.path.splitext(os.path.basename(out_config))[0] + nchannels = pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout(out_config) else: - oc_in_file_names = oc - nchannels = pyivastest.constants.OC_TO_NCHANNELS[oc.upper()] + out_config_in_file_names = out_config + nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs) s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs) @@ -38,6 +39,17 @@ def cmp_pcm(file1, file2, oc, fs) -> (int, str): if cmp_result["bitexact"]: return 0, "SUCCESS: Files are bitexact" else: - diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']}" + diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" print(diff_msg) return 1, "FAIL: Files have different content" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("file1", type=str) + parser.add_argument("file2", type=str) + parser.add_argument("-o", "--out_config", type=str.upper, default="MONO", choices=pyivastest.constants.OC_TO_NCHANNELS.keys()) + parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs") + + args = parser.parse_args() + sys.exit(cmp_pcm(**vars(args))) -- GitLab From 3725b2dbb8c609e0f9186c830dd76119747d39ba Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 10 Jan 2023 16:31:08 +0100 Subject: [PATCH 08/12] make script executable --- tests/cmp_pcm.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/cmp_pcm.py diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py old mode 100644 new mode 100755 -- GitLab From 28137917327ca994ae3bb808b77f54cd3f42dae6 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 10 Jan 2023 16:41:07 +0100 Subject: [PATCH 09/12] add shebang to script --- tests/cmp_pcm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 40d165c040..0804ece286 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import os import sys import argparse -- GitLab From 1ec57e1f2c7a2a06393fbe690d36791678e46762 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 10 Jan 2023 16:57:45 +0100 Subject: [PATCH 10/12] fix return value --- tests/cmp_pcm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 0804ece286..45ca7d4fc4 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -52,6 +52,7 @@ if __name__ == "__main__": parser.add_argument("file2", type=str) parser.add_argument("-o", "--out_config", type=str.upper, default="MONO", choices=pyivastest.constants.OC_TO_NCHANNELS.keys()) parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs") - args = parser.parse_args() - sys.exit(cmp_pcm(**vars(args))) + + result, _ = cmp_pcm(**vars(args)) + sys.exit(result) -- GitLab From 015a779b4d97ad4aa99db24e6b406598268721ff Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 11 Jan 2023 11:01:03 +0100 Subject: [PATCH 11/12] fix printout and use integer data type --- tests/cmp_pcm.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 45ca7d4fc4..2320018acb 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -9,6 +9,7 @@ sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) import pyaudio3dtools import pyivastest +import numpy as np def cmp_pcm(file1, file2, out_config, fs) -> (int, str): @@ -26,8 +27,8 @@ def cmp_pcm(file1, file2, out_config, fs) -> (int, str): out_config_in_file_names = out_config nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] - s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs) - s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs) + s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs, outdtype=np.int16) + s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs, outdtype=np.int16) if s1.shape != s2.shape: print( @@ -54,5 +55,6 @@ if __name__ == "__main__": parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs") args = parser.parse_args() - result, _ = cmp_pcm(**vars(args)) + result, msg = cmp_pcm(**vars(args)) + print(msg) sys.exit(result) -- GitLab From d6fcf004a2557de4672cee25c477e767ac6b5c28 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 16 Jan 2023 13:59:32 +0100 Subject: [PATCH 12/12] add printout of first diff pos in samples and frames --- scripts/pyaudio3dtools/audioarray.py | 14 ++++++++++++++ tests/cmp_pcm.py | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 1221e25960..8263e8048b 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -248,6 +248,9 @@ def compare(ref: np.ndarray, test: np.ndarray, fs: int, per_frame: bool=True) -> "max_abs_diff_pos_channel": 0, "nsamples_diff": 0, "nsamples_diff_percentage": 0.0, + "first_diff_pos_sample": -1, + "first_diff_pos_channel": -1, + "first_diff_pos_frame": -1 } if per_frame: result["max_abs_diff_pos_frame"] = 0 @@ -266,6 +269,14 @@ def compare(ref: np.ndarray, test: np.ndarray, fs: int, per_frame: bool=True) -> max_diff_pos[0][0] // framesize, max_diff_pos[1][0], ] + + first_diff_pos = np.nonzero(diff) + first_diff_pos = [ + first_diff_pos[0][0], + first_diff_pos[0][0] // framesize, + first_diff_pos[1][0], + ] + nsamples_diff = np.nonzero(diff)[0].size nsamples_diff_percentage = nsamples_diff / (nsamples_total * nchannels) * 100.0 nframes = nsamples_total // framesize @@ -278,6 +289,9 @@ def compare(ref: np.ndarray, test: np.ndarray, fs: int, per_frame: bool=True) -> "max_abs_diff_pos_channel": max_diff_pos[2], "nsamples_diff": nsamples_diff, "nsamples_diff_percentage": nsamples_diff_percentage, + "first_diff_pos_sample": first_diff_pos[0], + "first_diff_pos_channel": first_diff_pos[2], + "first_diff_pos_frame": first_diff_pos[1], } if per_frame: diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 2320018acb..a54aa2cf11 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -22,7 +22,11 @@ def cmp_pcm(file1, file2, out_config, fs) -> (int, str): out_config = "MONO" if out_config == "" else out_config if out_config.upper() not in pyivastest.constants.OC_TO_NCHANNELS: out_config_in_file_names = os.path.splitext(os.path.basename(out_config))[0] - nchannels = pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout(out_config) + nchannels = ( + pyivastest.IvasScriptsCommon.IvasScript.get_n_channels_from_ls_layout( + out_config + ) + ) else: out_config_in_file_names = out_config nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] @@ -43,7 +47,9 @@ def cmp_pcm(file1, file2, out_config, fs) -> (int, str): return 0, "SUCCESS: Files are bitexact" else: diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" + first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" print(diff_msg) + print(first_msg) return 1, "FAIL: Files have different content" @@ -51,7 +57,13 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("file1", type=str) parser.add_argument("file2", type=str) - parser.add_argument("-o", "--out_config", type=str.upper, default="MONO", choices=pyivastest.constants.OC_TO_NCHANNELS.keys()) + parser.add_argument( + "-o", + "--out_config", + type=str.upper, + default="MONO", + choices=pyivastest.constants.OC_TO_NCHANNELS.keys(), + ) parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs") args = parser.parse_args() -- GitLab