From 46783e2663e8dae47f7f8cd4deda0571a42ffd97 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 12 Jun 2024 15:30:19 +0200 Subject: [PATCH 01/25] add ssnr script --- tests/ssnr.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/ssnr.py diff --git a/tests/ssnr.py b/tests/ssnr.py new file mode 100644 index 0000000000..01785c21e1 --- /dev/null +++ b/tests/ssnr.py @@ -0,0 +1,80 @@ +import argparse +import sys +import pathlib +import numpy as np + +HERE = pathlib.Path(__file__).parent.absolute() +SCRIPT_DIR = HERE.parent.joinpath("scripts") + +sys.path.append(str(SCRIPT_DIR)) +from pyaudio3dtools import audiofile, audioarray + + +THRESH_LOW = -50 +THRESH_HIGH = -15 + + +def get_ssnr( + ref_sig: np.ndarray, + test_sig: np.ndarray, + len_seg: int, + thresh_low: float = -200, + thresh_high: float = 0, +): + """ + Calculate Segmental SNR for test_sig to ref_sig as defined in ISO/IEC 14496-4 + """ + ss = list() + + denom_add = 10**-13 * len_seg + n = 0 + for ref_seg, test_seg in zip( + audioarray.get_framewise(ref_sig, len_seg, zero_pad=True), + audioarray.get_framewise(test_sig, len_seg, zero_pad=True), + ): + nrg_ref = np.sum(ref_seg**2) + + ref_power = 10 * np.log10(nrg_ref / len_seg) + if ref_power < thresh_low or ref_power > thresh_high: + continue + + diff_seg = ref_seg - test_seg + nrg_diff = np.sum(diff_seg**2) + + ss_seg = np.log10(1 + nrg_ref / (denom_add + nrg_diff)) + ss.append(ss_seg) + n += 1 + + ssnr = 10 * np.log10(10 ** (np.sum(ss) / n) - 1) + return ssnr + + +def main(args): + ref_sig, fs_ref = audiofile.readfile(args.ref_file) + test_sig, fs_test = audiofile.readfile(args.test_file) + + if fs_ref != fs_test: + print("Files need to have same sampling rate!") + return -1 + + # normalize 16Bit wav signals to range of [-1, 1] + ref_sig /= -np.iinfo(np.int16).min + test_sig /= -np.iinfo(np.int16).min + + len_seg = int(20 * fs_ref / 1000) + ssnr = get_ssnr( + ref_sig, test_sig, len_seg, thresh_low=THRESH_LOW, thresh_high=THRESH_HIGH + ) + print(ssnr) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("ref_file", type=pathlib.Path, help="Reference signal wav file") + parser.add_argument( + "test_file", type=pathlib.Path, help="Signal under test wav file" + ) + + args = parser.parse_args() + + main(args) -- GitLab From d7d223a07e57f4addd6c884b1bfe0b2b630c7117 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 12 Jun 2024 17:10:18 +0200 Subject: [PATCH 02/25] extend for more than one channel --- tests/ssnr.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/ssnr.py b/tests/ssnr.py index 01785c21e1..bbea5826ba 100644 --- a/tests/ssnr.py +++ b/tests/ssnr.py @@ -20,32 +20,36 @@ def get_ssnr( len_seg: int, thresh_low: float = -200, thresh_high: float = 0, -): +) -> np.ndarray: """ Calculate Segmental SNR for test_sig to ref_sig as defined in ISO/IEC 14496-4 """ ss = list() denom_add = 10**-13 * len_seg - n = 0 + segment_counter = np.zeros(ref_sig.shape[1]) for ref_seg, test_seg in zip( audioarray.get_framewise(ref_sig, len_seg, zero_pad=True), audioarray.get_framewise(test_sig, len_seg, zero_pad=True), ): - nrg_ref = np.sum(ref_seg**2) - - ref_power = 10 * np.log10(nrg_ref / len_seg) - if ref_power < thresh_low or ref_power > thresh_high: - continue + nrg_ref = np.sum(ref_seg**2, axis=0) diff_seg = ref_seg - test_seg - nrg_diff = np.sum(diff_seg**2) + nrg_diff = np.sum(diff_seg**2, axis=0) ss_seg = np.log10(1 + nrg_ref / (denom_add + nrg_diff)) + + # only sum up segments that fall inside the thresholds + ref_power = 10 * np.log10(nrg_ref / len_seg) + zero_mask = np.logical_or(ref_power < thresh_low, ref_power > thresh_high) + + ss_seg[zero_mask] = 0 + # increase segment counter only for channels that were not zeroed + segment_counter += np.logical_not(zero_mask) + ss.append(ss_seg) - n += 1 - ssnr = 10 * np.log10(10 ** (np.sum(ss) / n) - 1) + ssnr = 10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1) return ssnr @@ -65,7 +69,11 @@ def main(args): ssnr = get_ssnr( ref_sig, test_sig, len_seg, thresh_low=THRESH_LOW, thresh_high=THRESH_HIGH ) - print(ssnr) + + for i, s in enumerate(ssnr, start=1): + print(f"Channel {i}: {s}") + + return 0 if __name__ == "__main__": @@ -77,4 +85,4 @@ if __name__ == "__main__": args = parser.parse_args() - main(args) + sys.exit(main(args)) -- GitLab From defa3721e638393c92059bddfe0c325f8577333a Mon Sep 17 00:00:00 2001 From: kiene Date: Thu, 13 Jun 2024 12:22:00 +0200 Subject: [PATCH 03/25] add small constant to ref_power to avoid zero in log --- tests/ssnr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ssnr.py b/tests/ssnr.py index bbea5826ba..6711831876 100644 --- a/tests/ssnr.py +++ b/tests/ssnr.py @@ -40,7 +40,8 @@ def get_ssnr( ss_seg = np.log10(1 + nrg_ref / (denom_add + nrg_diff)) # only sum up segments that fall inside the thresholds - ref_power = 10 * np.log10(nrg_ref / len_seg) + # add smapp eps to nrg_ref to prevent RuntimeWarnings from numpy + ref_power = 10 * np.log10((nrg_ref + 10**-7) / len_seg) zero_mask = np.logical_or(ref_power < thresh_low, ref_power > thresh_high) ss_seg[zero_mask] = 0 -- GitLab From 1ec14d374096807f7761714010cb05cc8351f661 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Jun 2024 12:33:56 +0200 Subject: [PATCH 04/25] correct small typo --- tests/ssnr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ssnr.py b/tests/ssnr.py index 6711831876..15ec538950 100644 --- a/tests/ssnr.py +++ b/tests/ssnr.py @@ -40,7 +40,7 @@ def get_ssnr( ss_seg = np.log10(1 + nrg_ref / (denom_add + nrg_diff)) # only sum up segments that fall inside the thresholds - # add smapp eps to nrg_ref to prevent RuntimeWarnings from numpy + # add small eps to nrg_ref to prevent RuntimeWarnings from numpy ref_power = 10 * np.log10((nrg_ref + 10**-7) / len_seg) zero_mask = np.logical_or(ref_power < thresh_low, ref_power > thresh_high) -- GitLab From a7b7c46274798110df93715dec9f59b67ed553a7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Jun 2024 12:47:43 +0200 Subject: [PATCH 05/25] move ssnr script and functionality to scripts folder --- scripts/pyaudio3dtools/audioarray.py | 40 +++++++++++++ scripts/ssnr.py | 44 ++++++++++++++ tests/ssnr.py | 89 ---------------------------- 3 files changed, 84 insertions(+), 89 deletions(-) create mode 100644 scripts/ssnr.py delete mode 100644 tests/ssnr.py diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index d89c079d89..ef233d2db0 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -513,3 +513,43 @@ def process_async(files: Iterable, func: Callable, **kwargs): for r in results: r.get() return results + + +def ssnr( + ref_sig: np.ndarray, + test_sig: np.ndarray, + len_seg: int, + thresh_low: float = -200, + thresh_high: float = 0, +) -> np.ndarray: + """ + Calculate Segmental SNR for test_sig to ref_sig as defined in ISO/IEC 14496-4 + """ + ss = list() + + denom_add = 10**-13 * len_seg + segment_counter = np.zeros(ref_sig.shape[1]) + for ref_seg, test_seg in zip( + get_framewise(ref_sig, len_seg, zero_pad=True), + get_framewise(test_sig, len_seg, zero_pad=True), + ): + nrg_ref = np.sum(ref_seg**2, axis=0) + + diff_seg = ref_seg - test_seg + nrg_diff = np.sum(diff_seg**2, axis=0) + + ss_seg = np.log10(1 + nrg_ref / (denom_add + nrg_diff)) + + # only sum up segments that fall inside the thresholds + # add small eps to nrg_ref to prevent RuntimeWarnings from numpy + ref_power = 10 * np.log10((nrg_ref + 10**-7) / len_seg) + zero_mask = np.logical_or(ref_power < thresh_low, ref_power > thresh_high) + + ss_seg[zero_mask] = 0 + # increase segment counter only for channels that were not zeroed + segment_counter += np.logical_not(zero_mask) + + ss.append(ss_seg) + + ssnr = 10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1) + return ssnr diff --git a/scripts/ssnr.py b/scripts/ssnr.py new file mode 100644 index 0000000000..2235b7bf8c --- /dev/null +++ b/scripts/ssnr.py @@ -0,0 +1,44 @@ +import argparse +import sys +import pathlib +import numpy as np +from pyaudio3dtools import audiofile, audioarray + + +THRESH_LOW = -50 +THRESH_HIGH = -15 + + +def main(args): + ref_sig, fs_ref = audiofile.readfile(args.ref_file) + test_sig, fs_test = audiofile.readfile(args.test_file) + + if fs_ref != fs_test: + print("Files need to have same sampling rate!") + return -1 + + # normalize 16Bit wav signals to range of [-1, 1] + ref_sig /= -np.iinfo(np.int16).min + test_sig /= -np.iinfo(np.int16).min + + len_seg = int(20 * fs_ref / 1000) + ssnr = audioarray.ssnr( + ref_sig, test_sig, len_seg, thresh_low=THRESH_LOW, thresh_high=THRESH_HIGH + ) + + for i, s in enumerate(ssnr, start=1): + print(f"Channel {i}: {s}") + + return 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("ref_file", type=pathlib.Path, help="Reference signal wav file") + parser.add_argument( + "test_file", type=pathlib.Path, help="Signal under test wav file" + ) + + args = parser.parse_args() + + sys.exit(main(args)) diff --git a/tests/ssnr.py b/tests/ssnr.py deleted file mode 100644 index 15ec538950..0000000000 --- a/tests/ssnr.py +++ /dev/null @@ -1,89 +0,0 @@ -import argparse -import sys -import pathlib -import numpy as np - -HERE = pathlib.Path(__file__).parent.absolute() -SCRIPT_DIR = HERE.parent.joinpath("scripts") - -sys.path.append(str(SCRIPT_DIR)) -from pyaudio3dtools import audiofile, audioarray - - -THRESH_LOW = -50 -THRESH_HIGH = -15 - - -def get_ssnr( - ref_sig: np.ndarray, - test_sig: np.ndarray, - len_seg: int, - thresh_low: float = -200, - thresh_high: float = 0, -) -> np.ndarray: - """ - Calculate Segmental SNR for test_sig to ref_sig as defined in ISO/IEC 14496-4 - """ - ss = list() - - denom_add = 10**-13 * len_seg - segment_counter = np.zeros(ref_sig.shape[1]) - for ref_seg, test_seg in zip( - audioarray.get_framewise(ref_sig, len_seg, zero_pad=True), - audioarray.get_framewise(test_sig, len_seg, zero_pad=True), - ): - nrg_ref = np.sum(ref_seg**2, axis=0) - - diff_seg = ref_seg - test_seg - nrg_diff = np.sum(diff_seg**2, axis=0) - - ss_seg = np.log10(1 + nrg_ref / (denom_add + nrg_diff)) - - # only sum up segments that fall inside the thresholds - # add small eps to nrg_ref to prevent RuntimeWarnings from numpy - ref_power = 10 * np.log10((nrg_ref + 10**-7) / len_seg) - zero_mask = np.logical_or(ref_power < thresh_low, ref_power > thresh_high) - - ss_seg[zero_mask] = 0 - # increase segment counter only for channels that were not zeroed - segment_counter += np.logical_not(zero_mask) - - ss.append(ss_seg) - - ssnr = 10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1) - return ssnr - - -def main(args): - ref_sig, fs_ref = audiofile.readfile(args.ref_file) - test_sig, fs_test = audiofile.readfile(args.test_file) - - if fs_ref != fs_test: - print("Files need to have same sampling rate!") - return -1 - - # normalize 16Bit wav signals to range of [-1, 1] - ref_sig /= -np.iinfo(np.int16).min - test_sig /= -np.iinfo(np.int16).min - - len_seg = int(20 * fs_ref / 1000) - ssnr = get_ssnr( - ref_sig, test_sig, len_seg, thresh_low=THRESH_LOW, thresh_high=THRESH_HIGH - ) - - for i, s in enumerate(ssnr, start=1): - print(f"Channel {i}: {s}") - - return 0 - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("ref_file", type=pathlib.Path, help="Reference signal wav file") - parser.add_argument( - "test_file", type=pathlib.Path, help="Signal under test wav file" - ) - - args = parser.parse_args() - - sys.exit(main(args)) -- GitLab From ead310a4097ba08aa4090b52565070ee1baed701 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Jun 2024 14:19:49 +0200 Subject: [PATCH 06/25] add ssnr to test_param_fle tests --- scripts/pyaudio3dtools/audioarray.py | 25 +++++++++++++++---- scripts/ssnr.py | 14 ++--------- tests/cmp_pcm.py | 19 ++++++++------ .../test_param_file.py | 4 ++- tests/conftest.py | 15 +++++++++++ 5 files changed, 52 insertions(+), 25 deletions(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index ef233d2db0..4f153e9e9b 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -232,6 +232,7 @@ def compare( fs: int, per_frame: bool = True, get_mld: bool = False, + get_ssnr: bool = False, ) -> dict: """Compare two audio arrays @@ -266,8 +267,13 @@ def compare( "first_diff_pos_sample": -1, "first_diff_pos_channel": -1, "first_diff_pos_frame": -1, - "MLD": 0 if get_mld else None, } + + if get_mld: + result["MLD"] = 0 + if get_ssnr: + result["SSNR"] = np.zeros(ref.shape[1]) + if per_frame: result["max_abs_diff_pos_frame"] = 0 result["nframes_diff"] = 0 @@ -320,7 +326,6 @@ def compare( result["nframes_diff_percentage"] = nframes_diff_percentage if get_mld: - mld_max = 0 toolsdir = Path(__file__).parent.parent.joinpath("tools") if platform.system() == "Windows": @@ -343,6 +348,12 @@ def compare( result["MLD"] = mld_max + if get_ssnr: + # length of segment is always 20ms + len_seg = int(0.02 * fs) + print(len_seg, ref.shape, test.shape) + result["SSNR"] = ssnr(ref, test, len_seg, thresh_low=-50, thresh_high=-15) + return result @@ -527,11 +538,14 @@ def ssnr( """ ss = list() + ref_sig_norm = ref_sig / -np.iinfo(np.int16).min + test_sig_norm = test_sig / -np.iinfo(np.int16).min + denom_add = 10**-13 * len_seg segment_counter = np.zeros(ref_sig.shape[1]) for ref_seg, test_seg in zip( - get_framewise(ref_sig, len_seg, zero_pad=True), - get_framewise(test_sig, len_seg, zero_pad=True), + get_framewise(ref_sig_norm, len_seg, zero_pad=True), + get_framewise(test_sig_norm, len_seg, zero_pad=True), ): nrg_ref = np.sum(ref_seg**2, axis=0) @@ -551,5 +565,6 @@ def ssnr( ss.append(ss_seg) - ssnr = 10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1) + # round to 2 decimals. this is just to be in line with the meg conformance tool + ssnr = np.round(10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2) return ssnr diff --git a/scripts/ssnr.py b/scripts/ssnr.py index 2235b7bf8c..6141bb0ed0 100644 --- a/scripts/ssnr.py +++ b/scripts/ssnr.py @@ -1,14 +1,9 @@ import argparse import sys import pathlib -import numpy as np from pyaudio3dtools import audiofile, audioarray -THRESH_LOW = -50 -THRESH_HIGH = -15 - - def main(args): ref_sig, fs_ref = audiofile.readfile(args.ref_file) test_sig, fs_test = audiofile.readfile(args.test_file) @@ -17,14 +12,9 @@ def main(args): print("Files need to have same sampling rate!") return -1 - # normalize 16Bit wav signals to range of [-1, 1] - ref_sig /= -np.iinfo(np.int16).min - test_sig /= -np.iinfo(np.int16).min - len_seg = int(20 * fs_ref / 1000) - ssnr = audioarray.ssnr( - ref_sig, test_sig, len_seg, thresh_low=THRESH_LOW, thresh_high=THRESH_HIGH - ) + print(len_seg, ref_sig.shape, test_sig.shape) + ssnr = audioarray.ssnr(ref_sig, test_sig, len_seg, thresh_low=-50, thresh_high=-15) for i, s in enumerate(ssnr, start=1): print(f"Channel {i}: {s}") diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 9783e8b561..b7f260ea7f 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -13,14 +13,15 @@ import pyivastest def cmp_pcm( - file1, - file2, + ref_file, + cmp_file, out_config, fs, get_mld=False, allow_differing_lengths=False, mld_lim=0, abs_tol=0, + get_ssnr=False, ) -> (int, str): """ Compare 2 PCM files for bitexactness @@ -39,8 +40,8 @@ def cmp_pcm( else: nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] - s1, _ = pyaudio3dtools.audiofile.readfile(file1, nchannels, fs, outdtype=np.int16) - s2, _ = pyaudio3dtools.audiofile.readfile(file2, nchannels, fs, outdtype=np.int16) + s1, _ = pyaudio3dtools.audiofile.readfile(ref_file, nchannels, fs, outdtype=np.int16) + s2, _ = pyaudio3dtools.audiofile.readfile(cmp_file, nchannels, fs, outdtype=np.int16) # In case of wav input, override the nchannels with the one from the wav header nchannels = s1.shape[1] @@ -62,7 +63,7 @@ def cmp_pcm( return 1, reason cmp_result = pyaudio3dtools.audioarray.compare( - s1, s2, fs, per_frame=False, get_mld=get_mld + s1, s2, fs, per_frame=False, get_mld=get_mld, get_ssnr=get_ssnr, ) output_differs = 0 @@ -90,13 +91,17 @@ def cmp_pcm( else: reason += f" > {mld_lim}" + if get_ssnr: + for i, s in enumerate(cmp_result["SSNR"], start=1): + print(f"Channel {i} SSNR: {s}") + return output_differs, reason if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("file1", type=str) - parser.add_argument("file2", type=str) + parser.add_argument("ref_file", type=str) + parser.add_argument("cmp_file", type=str) parser.add_argument( "-o", "--out_config", diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 619e252767..0e4b6e6035 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -152,6 +152,7 @@ def test_param_file_tests( get_mld, get_mld_lim, abs_tol, + get_ssnr, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -341,14 +342,15 @@ def test_param_file_tests( fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( - dut_output_file, ref_output_file, + dut_output_file, output_config, fs, get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, allow_differing_lengths=allow_differing_lengths, + get_ssnr=get_ssnr, ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) diff --git a/tests/conftest.py b/tests/conftest.py index cec06c2061..304651846d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -171,6 +171,13 @@ def pytest_addoption(parser): help="MLD limit for comparison (default: 0)", default="0", ) + + parser.addoption( + "--ssnr", + action="store_true", + help="Compute Segmental SNR (SSNR) between ref and dut output instead of just comparing for bitexactness", + ) + parser.addoption( "--create_ref", action="store_true", @@ -242,6 +249,14 @@ def get_mld_lim(request): return float(request.config.getoption("--mld-lim")) +@pytest.fixture(scope="session", autouse=True) +def get_ssnr(request): + """ + Return indication to compute ssnr during ref/dut comparison. + """ + return request.config.option.ssnr + + @pytest.fixture(scope="session") def abs_tol(request) -> int: """ -- GitLab From fdf2af967c718e77656659ab950bcfa75e0b4623 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Jun 2024 14:25:29 +0200 Subject: [PATCH 07/25] add ssnr possibility in other tests --- .../codec_be_on_mr_nonselection/test_masa_enc_dec.py | 3 +++ .../test_sba_bs_dec_plc.py | 4 ++++ tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py | 12 ++++++++++++ 3 files changed, 19 insertions(+) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 34fa40e278..afbc5b1196 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -114,6 +114,7 @@ def test_masa_enc_dec( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): # Input parameters in_fs = 48 @@ -219,6 +220,7 @@ def test_masa_enc_dec( get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr ) if get_mld: mld = re.search(MLD_PATTERN, reason).groups(1)[0] @@ -256,6 +258,7 @@ def test_masa_enc_dec( get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr ) if get_mld: mld = re.search(MLD_PATTERN, reason).groups(1)[0] diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 289d10fae1..fd90446513 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -93,6 +93,7 @@ def test_sba_plc_system( get_mld, get_mld_lim, abs_tol, + get_ssnr, ): SID = 0 if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: @@ -132,6 +133,7 @@ def test_sba_plc_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -156,6 +158,7 @@ def sba_dec_plc( get_mld=False, get_mld_lim=0, abs_tol=0, + get_ssnr=False, ): # ------------ run cmd ------------ @@ -213,6 +216,7 @@ def sba_dec_plc( get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) if get_mld: mld = re.search(MLD_PATTERN, reason).groups(1)[0] diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 2415d9d9a9..e9e566d854 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -107,6 +107,7 @@ def test_pca_enc( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): pca = True tag = tag + fs + "c" @@ -162,6 +163,7 @@ def test_pca_enc( get_mld_lim=get_mld_lim, pca=pca, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -194,6 +196,7 @@ def test_sba_enc_system( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -269,6 +272,7 @@ def test_sba_enc_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -292,6 +296,7 @@ def test_spar_hoa2_enc_system( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): fs = "48" dtx = "0" @@ -344,6 +349,7 @@ def test_spar_hoa2_enc_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -367,6 +373,7 @@ def test_spar_hoa3_enc_system( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): fs = "48" dtx = "0" @@ -419,6 +426,7 @@ def test_spar_hoa3_enc_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -446,6 +454,7 @@ def test_sba_enc_BWforce_system( get_mld_lim, decoder_only, abs_tol, + get_ssnr, ): if dtx == "1" and ivas_br not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -504,6 +513,7 @@ def test_sba_enc_BWforce_system( get_mld=get_mld, get_mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) @@ -665,6 +675,7 @@ def sba_dec( get_mld_lim=0, pca=False, abs_tol=0, + get_ssnr=False, ): # -------- run cmd ------------ # sampling rate to BW mapping @@ -729,6 +740,7 @@ def sba_dec( get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, + get_ssnr=get_ssnr, ) if get_mld: mld = re.search(MLD_PATTERN, reason).groups(1)[0] -- GitLab From fccf1155b1326b88b0d712af4bb130858ed4cb4e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Jun 2024 15:40:31 +0200 Subject: [PATCH 08/25] record SSNR as property --- tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py | 9 ++++++++- tests/codec_be_on_mr_nonselection/test_param_file.py | 9 ++++++++- tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py | 9 ++++++++- tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py | 9 ++++++++- tests/constants.py | 1 + 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index afbc5b1196..9078141f51 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -44,7 +44,7 @@ import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN +from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN # params # output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] @@ -226,6 +226,13 @@ def test_masa_enc_dec( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + if get_ssnr: + ssnrs = re.findall(SSNR_PATTERN, reason) + min_ssnr = min(ssnrs) + min_ssnr_channel = ssnrs.index(min_ssnr) + record_property("MIN_SSNR", min_ssnr) + record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) + max_diff = 0 if pcmcmp_res: search_result = re.search(MAX_DIFF_PATTERN, reason) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 0e4b6e6035..b3a8986860 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -45,7 +45,7 @@ import numpy as np from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend from tests.testconfig import PARAM_FILE -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN +from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN VALID_DEC_OUTPUT_CONF = [ "MONO", @@ -358,6 +358,13 @@ def test_param_file_tests( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + if get_ssnr: + ssnrs = re.findall(SSNR_PATTERN, reason) + min_ssnr = min(ssnrs) + min_ssnr_channel = ssnrs.index(min_ssnr) + record_property("MIN_SSNR", min_ssnr) + record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) + max_diff = 0 if output_differs: search_result = re.search(MAX_DIFF_PATTERN, reason) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index fd90446513..e1bbe37f08 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -40,7 +40,7 @@ import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN +from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN # params tag_list = ["stvFOA"] @@ -222,6 +222,13 @@ def sba_dec_plc( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + if get_ssnr: + ssnrs = re.findall(SSNR_PATTERN, reason) + min_ssnr = min(ssnrs) + min_ssnr_channel = ssnrs.index(min_ssnr) + record_property("MIN_SSNR", min_ssnr) + record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) + max_diff = 0 if cmp_result: search_result = re.search(MAX_DIFF_PATTERN, reason) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index e9e566d854..b76b0b6386 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -42,7 +42,7 @@ from cut_bs import cut_from_start from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN +from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN # params @@ -746,6 +746,13 @@ def sba_dec( mld = re.search(MLD_PATTERN, reason).groups(1)[0] record_property("MLD", mld) + if get_ssnr: + ssnrs = re.findall(SSNR_PATTERN, reason) + min_ssnr = min(ssnrs) + min_ssnr_channel = ssnrs.index(min_ssnr) + record_property("MIN_SSNR", min_ssnr) + record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) + max_diff = 0 if cmp_result: search_result = re.search(MAX_DIFF_PATTERN, reason) diff --git a/tests/constants.py b/tests/constants.py index 453c07dca8..27b10e52f6 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,3 +1,4 @@ # regex patterns for parsing the output from cmp_pcm -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" +SSNR_PATTERN = r"Channel \d* SSNR: (\d*\.\d*)" -- GitLab From 8aa62d0311c2d768912fef1056d2b6c4bc881f65 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Jun 2024 15:47:00 +0200 Subject: [PATCH 09/25] add ssnr printouts to return string --- tests/cmp_pcm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index b7f260ea7f..3c16e526c9 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -92,8 +92,11 @@ def cmp_pcm( reason += f" > {mld_lim}" if get_ssnr: + reason += "\n" for i, s in enumerate(cmp_result["SSNR"], start=1): - print(f"Channel {i} SSNR: {s}") + msg = f"Channel {i} SSNR: {s}" + reason += msg + "\n" + print(msg) return output_differs, reason -- GitLab From 214691e843bc3f09334887bf7a6b11e3504abb5e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Jun 2024 09:21:01 +0200 Subject: [PATCH 10/25] return inf on exact same signal --- scripts/pyaudio3dtools/audioarray.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 4f153e9e9b..266bf4ca5c 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -541,6 +541,10 @@ def ssnr( ref_sig_norm = ref_sig / -np.iinfo(np.int16).min test_sig_norm = test_sig / -np.iinfo(np.int16).min + diff_norm = ref_sig_norm - test_sig_norm + if np.all(diff_norm == 0): + return np.asarray([np.inf] * ref_sig_norm.shape[1]) + denom_add = 10**-13 * len_seg segment_counter = np.zeros(ref_sig.shape[1]) for ref_seg, test_seg in zip( -- GitLab From 4e4ebbd1710c0ae3d821f3c1d364a356cd586010 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Jun 2024 10:44:53 +0200 Subject: [PATCH 11/25] add MIN_SSNR to report parsing script --- scripts/parse_mld_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index 7370cfe7ff..a7f10f94a4 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -7,7 +7,7 @@ from xml.etree import ElementTree Parse a junit report and create an MLD summary report. """ -PROPERTIES = ["MLD", "MAXIMUM ABS DIFF"] +PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] # Main routine -- GitLab From cf01bc182e2076ff6fc35489486dfbcf9528a031 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Jun 2024 12:52:56 +0200 Subject: [PATCH 12/25] return inf correctly from audioarray.compare if signals are equal --- scripts/pyaudio3dtools/audioarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 266bf4ca5c..86cf3481da 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -272,7 +272,7 @@ def compare( if get_mld: result["MLD"] = 0 if get_ssnr: - result["SSNR"] = np.zeros(ref.shape[1]) + result["SSNR"] = np.asarray([np.inf] * ref.shape[1]) if per_frame: result["max_abs_diff_pos_frame"] = 0 -- GitLab From e7d8cde577c2c952dcf0c5f4cc842cfd3219f902 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jun 2024 13:21:18 +0200 Subject: [PATCH 13/25] return neg inf if ref outside of thresh everywhere --- scripts/pyaudio3dtools/audioarray.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 86cf3481da..8b18d93479 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -569,6 +569,12 @@ def ssnr( ss.append(ss_seg) - # round to 2 decimals. this is just to be in line with the meg conformance tool - ssnr = np.round(10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2) + if np.all(segment_counter == 0): + # the reference signal was outside of [thresh_lowm thresh_high] for all segments -> return -inf + # note that this is not in line with the mpeg conf tool which gives +/-0.0 in this case + ssnr = np.asarray([-np.inf] * len(segment_counter)) + else: + # round to 2 decimals. this is just to be in line with the mpeg conformance tool + ssnr = np.round(10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2) + return ssnr -- GitLab From ef16ea02dcdb4a9d5cf9e92371b33c0f0bda6a9f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jun 2024 14:31:10 +0200 Subject: [PATCH 14/25] unify property recording --- .../test_masa_enc_dec.py | 97 +++++-------------- .../test_param_file.py | 28 +----- .../test_sba_bs_dec_plc.py | 24 ++--- .../test_sba_bs_enc.py | 32 +++--- tests/conftest.py | 42 ++++++++ 5 files changed, 92 insertions(+), 131 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 9078141f51..94f5e70f92 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -35,16 +35,13 @@ __doc__ = """ import errno import os -import re from filecmp import cmp from typing import Optional import pytest from tests.cmp_pcm import cmp_pcm -from tests.conftest import DecoderFrontend, EncoderFrontend - -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN +from tests.conftest import DecoderFrontend, EncoderFrontend, record_properties # params # output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] @@ -98,6 +95,7 @@ def check_and_makedir(dir_path): ) def test_masa_enc_dec( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, ref_encoder_frontend: EncoderFrontend, @@ -207,80 +205,37 @@ def test_masa_enc_dec( ) # Compare outputs. For EXT output, also compare metadata. + metacmp_res = True if output_mode == "EXT": # Compare metadata as binary blob metacmp_res = cmp(dec_met_output_ref, dec_met_output_dut) - # Compare audio outputs - pcmcmp_res, reason = cmp_pcm( - dec_output_dut, - dec_output_ref, - output_mode, - int(out_fs * 1000), - get_mld=get_mld, - mld_lim=get_mld_lim, - abs_tol=abs_tol, - get_ssnr=get_ssnr - ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - if get_ssnr: - ssnrs = re.findall(SSNR_PATTERN, reason) - min_ssnr = min(ssnrs) - min_ssnr_channel = ssnrs.index(min_ssnr) - record_property("MIN_SSNR", min_ssnr) - record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) - - max_diff = 0 - if pcmcmp_res: - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) - - if get_mld and get_mld_lim > 0: - if pcmcmp_res != 0: - pytest.fail(reason) - else: - # Fail if compare fails compare result - if metacmp_res == False and pcmcmp_res != 0: - pytest.fail("Metadata and transport output difference detected") - elif metacmp_res == False: - pytest.fail("Metadata output difference detected") - elif pcmcmp_res != 0: - pytest.fail("Transport output difference detected") - else: - print("Comparison bit exact") - - else: - # Compare audio outputs - filecmp_res = cmp(dec_output_ref, dec_output_dut) - if filecmp_res == False: - cmp_result, reason = cmp_pcm( - dec_output_dut, - dec_output_ref, - output_mode, - int(out_fs * 1000), - get_mld=get_mld, - mld_lim=get_mld_lim, - abs_tol=abs_tol, - get_ssnr=get_ssnr - ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) + # Compare audio outputs + pcmcmp_res, reason = cmp_pcm( + dec_output_dut, + dec_output_ref, + output_mode, + int(out_fs * 1000), + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr + ) - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) + record_properties(text_to_parse=reason, output_differs=pcmcmp_res != 0, record_property=record_property, props_to_record=props_to_record) - # Report compare result - if cmp_result != 0: - pytest.fail(reason) + if get_mld and get_mld_lim > 0: + if pcmcmp_res != 0: + pytest.fail(reason) + else: + # Fail if compare fails compare result + if not metacmp_res and pcmcmp_res != 0: + pytest.fail("Metadata and transport output difference detected") + elif not metacmp_res: + pytest.fail("Metadata output difference detected") + elif pcmcmp_res != 0: + pytest.fail("Transport output difference detected") else: - if get_mld: - record_property("MLD", "0") print("Comparison bit exact") diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 9fd688471c..325bad4da7 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -35,7 +35,6 @@ Execute tests specified via a parameter file. import errno import filecmp import os -import re import platform from pathlib import Path from subprocess import run @@ -43,9 +42,9 @@ import pytest import numpy as np from tests.cmp_pcm import cmp_pcm -from tests.conftest import DecoderFrontend, EncoderFrontend +from tests.conftest import DecoderFrontend, EncoderFrontend, record_properties from tests.testconfig import PARAM_FILE -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN + VALID_DEC_OUTPUT_CONF = [ "MONO", @@ -136,6 +135,7 @@ def convert_test_string_to_tag(test_string): @pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID]) def test_param_file_tests( record_property, + props_to_record, decoder_only, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -354,27 +354,7 @@ def test_param_file_tests( ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - if get_ssnr: - ssnrs = re.findall(SSNR_PATTERN, reason) - min_ssnr = min(ssnrs) - min_ssnr_channel = ssnrs.index(min_ssnr) - record_property("MIN_SSNR", min_ssnr) - record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) - - max_diff = 0 - if output_differs: - search_result = re.search(MAX_DIFF_PATTERN, reason) - if search_result: - max_diff = search_result.groups(1)[0] - else: - msg = "Error " + MAX_DIFF_PATTERN + " not found" - print(msg) - pytest.fail(msg) - record_property("MAXIMUM ABS DIFF", max_diff) + record_properties(text_to_parse=reason, output_differs=output_differs, record_property=record_property, props_to_record=props_to_record) metadata_differs = False for md_file in md_out_files: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index e1bbe37f08..7df7d24dd3 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -34,13 +34,12 @@ __doc__ = """ import errno import os -import re import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN +from ..conftest import record_properties # params tag_list = ["stvFOA"] @@ -77,6 +76,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("gain_flag", gain_list) def test_sba_plc_system( record_property, + props_to_record, dut_decoder_frontend: DecoderFrontend, test_vector_path, reference_path, @@ -116,6 +116,7 @@ def test_sba_plc_system( # dec sba_dec_plc( record_property, + props_to_record, dut_decoder_frontend, test_vector_path, reference_path, @@ -141,6 +142,7 @@ def test_sba_plc_system( # -------------------- test function -------------------- def sba_dec_plc( record_property, + props_to_record, decoder_frontend, test_vector_path, reference_path, @@ -218,22 +220,8 @@ def sba_dec_plc( abs_tol=abs_tol, get_ssnr=get_ssnr, ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - if get_ssnr: - ssnrs = re.findall(SSNR_PATTERN, reason) - min_ssnr = min(ssnrs) - min_ssnr_channel = ssnrs.index(min_ssnr) - record_property("MIN_SSNR", min_ssnr) - record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) - - max_diff = 0 - if cmp_result: - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) + + record_properties(text_to_parse=reason, output_differs=cmp_result!=0, record_property=record_property, props_to_record=props_to_record) # report compare result if cmp_result != 0: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index b76b0b6386..b49c69a615 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -35,14 +35,13 @@ __doc__ = """ import errno import os -import re import pytest from cut_bs import cut_from_start from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN +from ..conftest import record_properties # params @@ -92,6 +91,7 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("fs", sample_rate_list) def test_pca_enc( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -144,6 +144,7 @@ def test_pca_enc( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -176,6 +177,7 @@ def test_pca_enc( @pytest.mark.parametrize("SID", SID_list) def test_sba_enc_system( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -254,6 +256,7 @@ def test_sba_enc_system( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -281,6 +284,7 @@ def test_sba_enc_system( @pytest.mark.parametrize("tag", tag_list_HOA2) def test_spar_hoa2_enc_system( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -331,6 +335,7 @@ def test_spar_hoa2_enc_system( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -358,6 +363,7 @@ def test_spar_hoa2_enc_system( @pytest.mark.parametrize("tag", tag_list_HOA3) def test_spar_hoa3_enc_system( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -408,6 +414,7 @@ def test_spar_hoa3_enc_system( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -437,6 +444,7 @@ def test_spar_hoa3_enc_system( @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) def test_sba_enc_BWforce_system( record_property, + props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, test_vector_path, @@ -495,6 +503,7 @@ def test_sba_enc_BWforce_system( # dec sba_dec( record_property, + props_to_record, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -656,6 +665,7 @@ def sba_enc( def sba_dec( record_property, + props_to_record, decoder_frontend, ref_decoder_frontend, reference_path, @@ -742,22 +752,8 @@ def sba_dec( abs_tol=abs_tol, get_ssnr=get_ssnr, ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - if get_ssnr: - ssnrs = re.findall(SSNR_PATTERN, reason) - min_ssnr = min(ssnrs) - min_ssnr_channel = ssnrs.index(min_ssnr) - record_property("MIN_SSNR", min_ssnr) - record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) - - max_diff = 0 - if cmp_result: - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) + + record_properties(text_to_parse=reason, output_differs=cmp_result!=0, record_property=record_property, props_to_record=props_to_record) # report compare result if cmp_result != 0: diff --git a/tests/conftest.py b/tests/conftest.py index 304651846d..4000bc2fed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,6 +34,7 @@ Pytest customization (configuration and fixtures) for the IVAS codec test suite. import logging import os +import re from tests import testconfig import pytest import platform @@ -41,6 +42,7 @@ import textwrap from pathlib import Path from subprocess import TimeoutExpired, run from typing import Optional, Union +from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN logger = logging.getLogger(__name__) USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature @@ -764,3 +766,43 @@ def pytest_configure(config): testconfig.MD5_REF_DICT = { line.split()[0]: line.split()[1] for line in f.readlines() } + + +@pytest.fixture(scope="session") +def props_to_record(request, get_mld, get_ssnr) -> str: + props = ["MAXIMUM ABS DIFF"] + if get_mld: + props.append("MLD") + if get_ssnr: + props.append("SSNR") + + return props + + +def record_properties(text_to_parse: str, output_differs: bool, record_property, props_to_record: list): + """ + Record the given properties in the report by parsing their values from the text. + """ + + for prop in props_to_record: + if prop == "MLD": + mld = re.search(MLD_PATTERN, text_to_parse).groups(1)[0] + record_property("MLD", mld) + elif prop == "MAXIMUM ABS DIFF": + max_diff = 0 + if output_differs: + if (match := re.search(MAX_DIFF_PATTERN, text_to_parse)) is not None: + max_diff = match.groups(1)[0] + else: + raise MaxDiffPatternNotFound() + record_property("MAXIMUM ABS DIFF", max_diff) + elif prop == "SSNR": + ssnrs = re.findall(SSNR_PATTERN, text_to_parse) + min_ssnr = min(ssnrs) + min_ssnr_channel = ssnrs.index(min_ssnr) + record_property("MIN_SSNR", min_ssnr) + record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) + + +class MaxDiffPatternNotFound(Exception): + pass -- GitLab From b6fc5dd5bb90dc045c5115e7b4b70fb7692d540d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jun 2024 14:45:13 +0200 Subject: [PATCH 15/25] modify SSNR pattern to match also inf's and nan's --- tests/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/constants.py b/tests/constants.py index 27b10e52f6..5bce5ac4e3 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,4 +1,4 @@ # regex patterns for parsing the output from cmp_pcm -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" -SSNR_PATTERN = r"Channel \d* SSNR: (\d*\.\d*)" +SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)" -- GitLab From f3eb9cc3b6f2fed98069f772ad7c3a3298d7cee1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jun 2024 14:54:32 +0200 Subject: [PATCH 16/25] refactor prop recording a bit --- .../test_masa_enc_dec.py | 6 ++++-- .../test_param_file.py | 6 ++++-- .../test_sba_bs_dec_plc.py | 6 ++++-- .../test_sba_bs_enc.py | 6 ++++-- tests/conftest.py | 16 ++++++++++------ 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 94f5e70f92..0a2ef6c30b 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -41,7 +41,7 @@ from typing import Optional import pytest from tests.cmp_pcm import cmp_pcm -from tests.conftest import DecoderFrontend, EncoderFrontend, record_properties +from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties # params # output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] @@ -222,7 +222,9 @@ def test_masa_enc_dec( get_ssnr=get_ssnr ) - record_properties(text_to_parse=reason, output_differs=pcmcmp_res != 0, record_property=record_property, props_to_record=props_to_record) + props = parse_properties(reason, pcmcmp_res != 0, props_to_record) + for k, v in props.items(): + record_property(k, v) if get_mld and get_mld_lim > 0: if pcmcmp_res != 0: diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 325bad4da7..814a6441df 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -42,7 +42,7 @@ import pytest import numpy as np from tests.cmp_pcm import cmp_pcm -from tests.conftest import DecoderFrontend, EncoderFrontend, record_properties +from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties from tests.testconfig import PARAM_FILE @@ -354,7 +354,9 @@ def test_param_file_tests( ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) - record_properties(text_to_parse=reason, output_differs=output_differs, record_property=record_property, props_to_record=props_to_record) + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) metadata_differs = False for md_file in md_out_files: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 7df7d24dd3..e188d4a114 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -39,7 +39,7 @@ import pytest from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend -from ..conftest import record_properties +from ..conftest import parse_properties # params tag_list = ["stvFOA"] @@ -221,7 +221,9 @@ def sba_dec_plc( get_ssnr=get_ssnr, ) - record_properties(text_to_parse=reason, output_differs=cmp_result!=0, record_property=record_property, props_to_record=props_to_record) + props = parse_properties(reason, cmp_result!=0, props_to_record) + for k, v in props.items(): + record_property(k, v) # report compare result if cmp_result != 0: diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index b49c69a615..ce99c55a27 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -41,7 +41,7 @@ from cut_bs import cut_from_start from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend -from ..conftest import record_properties +from ..conftest import parse_properties # params @@ -753,7 +753,9 @@ def sba_dec( get_ssnr=get_ssnr, ) - record_properties(text_to_parse=reason, output_differs=cmp_result!=0, record_property=record_property, props_to_record=props_to_record) + props = parse_properties(reason, cmp_result!=0, props_to_record) + for k, v in props.items(): + record_property(k, v) # report compare result if cmp_result != 0: diff --git a/tests/conftest.py b/tests/conftest.py index 4000bc2fed..68a591a18b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -779,15 +779,17 @@ def props_to_record(request, get_mld, get_ssnr) -> str: return props -def record_properties(text_to_parse: str, output_differs: bool, record_property, props_to_record: list): +def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: list): """ Record the given properties in the report by parsing their values from the text. """ + props = dict() + for prop in props_to_record: if prop == "MLD": - mld = re.search(MLD_PATTERN, text_to_parse).groups(1)[0] - record_property("MLD", mld) + mld = float(re.search(MLD_PATTERN, text_to_parse).groups(1)[0]) + props[prop] = mld elif prop == "MAXIMUM ABS DIFF": max_diff = 0 if output_differs: @@ -795,13 +797,15 @@ def record_properties(text_to_parse: str, output_differs: bool, record_property, max_diff = match.groups(1)[0] else: raise MaxDiffPatternNotFound() - record_property("MAXIMUM ABS DIFF", max_diff) + props[prop] = max_diff elif prop == "SSNR": ssnrs = re.findall(SSNR_PATTERN, text_to_parse) min_ssnr = min(ssnrs) min_ssnr_channel = ssnrs.index(min_ssnr) - record_property("MIN_SSNR", min_ssnr) - record_property("MIN_SSNR_CHANNEL", min_ssnr_channel) + props["MIN_SSNR"] = min_ssnr + props["MIN_SSNR_CHANNEL"] = min_ssnr_channel + + return props class MaxDiffPatternNotFound(Exception): -- GitLab From 5ee6c3255936fa8ca7380bf686bf639cdbb5d0cb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jun 2024 15:49:24 +0200 Subject: [PATCH 17/25] also handle out-of-threshold reference in single channels only --- scripts/pyaudio3dtools/audioarray.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 8b18d93479..2b662db538 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -31,6 +31,7 @@ """ import logging +import warnings import math import multiprocessing as mp import platform @@ -569,12 +570,11 @@ def ssnr( ss.append(ss_seg) - if np.all(segment_counter == 0): - # the reference signal was outside of [thresh_lowm thresh_high] for all segments -> return -inf - # note that this is not in line with the mpeg conf tool which gives +/-0.0 in this case - ssnr = np.asarray([-np.inf] * len(segment_counter)) - else: - # round to 2 decimals. this is just to be in line with the mpeg conformance tool + # if the reference signal was outside the thresholds for all segments in a channel, segment_counter is zero + # for that channel and the division here would trigger a warning. We supress the warning and later + # set the SSNr for those channels to -inf manually instead (overwriting later is simply easier than adding ifs here) + with warnings.catch_warnings(): ssnr = np.round(10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2) + ssnr[segment_counter == 0] = -np.inf return ssnr -- GitLab From 9d43d5369d7b506b331a423e9281682c07f1deeb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jun 2024 17:20:46 +0200 Subject: [PATCH 18/25] fix for no diff, but out of threshold cases --- scripts/pyaudio3dtools/audioarray.py | 11 +++++++++-- scripts/ssnr.py | 8 +++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 2b662db538..73f4130aef 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -234,6 +234,8 @@ def compare( per_frame: bool = True, get_mld: bool = False, get_ssnr: bool = False, + ssnr_thresh_low: float = -np.inf, + ssnr_thresh_high: float = np.inf, ) -> dict: """Compare two audio arrays @@ -353,7 +355,7 @@ def compare( # length of segment is always 20ms len_seg = int(0.02 * fs) print(len_seg, ref.shape, test.shape) - result["SSNR"] = ssnr(ref, test, len_seg, thresh_low=-50, thresh_high=-15) + result["SSNR"] = ssnr(ref, test, len_seg, thresh_low=ssnr_thresh_low, thresh_high=ssnr_thresh_high) return result @@ -542,6 +544,7 @@ def ssnr( ref_sig_norm = ref_sig / -np.iinfo(np.int16).min test_sig_norm = test_sig / -np.iinfo(np.int16).min + # check if diff of signal is zero already, then SNR is infinite, since no noise diff_norm = ref_sig_norm - test_sig_norm if np.all(diff_norm == 0): return np.asarray([np.inf] * ref_sig_norm.shape[1]) @@ -572,9 +575,13 @@ def ssnr( # if the reference signal was outside the thresholds for all segments in a channel, segment_counter is zero # for that channel and the division here would trigger a warning. We supress the warning and later - # set the SSNr for those channels to -inf manually instead (overwriting later is simply easier than adding ifs here) + # set the SSNR for those channels to -inf manually instead (overwriting later is simply easier than adding ifs here) with warnings.catch_warnings(): ssnr = np.round(10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2) ssnr[segment_counter == 0] = -np.inf + # set to zero for channels with no diff (this handles e.g. the corner-case of an all-zero channel in both ref and dut) + zero_diff_mask = np.asarray([np.all(diff_norm[:, c] == 0) for c in range(ref_sig.shape[1])]) + ssnr[zero_diff_mask] = 0 + return ssnr diff --git a/scripts/ssnr.py b/scripts/ssnr.py index 6141bb0ed0..80a3d9dd85 100644 --- a/scripts/ssnr.py +++ b/scripts/ssnr.py @@ -14,7 +14,7 @@ def main(args): len_seg = int(20 * fs_ref / 1000) print(len_seg, ref_sig.shape, test_sig.shape) - ssnr = audioarray.ssnr(ref_sig, test_sig, len_seg, thresh_low=-50, thresh_high=-15) + ssnr = audioarray.ssnr(ref_sig, test_sig, len_seg, args.thresh_low, args.thresh_high) for i, s in enumerate(ssnr, start=1): print(f"Channel {i}: {s}") @@ -28,6 +28,12 @@ if __name__ == "__main__": parser.add_argument( "test_file", type=pathlib.Path, help="Signal under test wav file" ) + parser.add_argument( + "--thresh_low", type=float, default="-inf", + ) + parser.add_argument( + "--thresh_high", type=float, default="inf", + ) args = parser.parse_args() -- GitLab From fc25a1899f529c4b160e745184b47957a13884e2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jun 2024 18:05:27 +0200 Subject: [PATCH 19/25] small optimization --- scripts/pyaudio3dtools/audioarray.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 73f4130aef..96fe7417bb 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -545,19 +545,17 @@ def ssnr( test_sig_norm = test_sig / -np.iinfo(np.int16).min # check if diff of signal is zero already, then SNR is infinite, since no noise - diff_norm = ref_sig_norm - test_sig_norm - if np.all(diff_norm == 0): + diff_sig_norm = ref_sig_norm - test_sig_norm + if np.all(diff_sig_norm == 0): return np.asarray([np.inf] * ref_sig_norm.shape[1]) denom_add = 10**-13 * len_seg segment_counter = np.zeros(ref_sig.shape[1]) - for ref_seg, test_seg in zip( + for ref_seg, diff_seg in zip( get_framewise(ref_sig_norm, len_seg, zero_pad=True), - get_framewise(test_sig_norm, len_seg, zero_pad=True), + get_framewise(diff_sig_norm, len_seg, zero_pad=True), ): nrg_ref = np.sum(ref_seg**2, axis=0) - - diff_seg = ref_seg - test_seg nrg_diff = np.sum(diff_seg**2, axis=0) ss_seg = np.log10(1 + nrg_ref / (denom_add + nrg_diff)) @@ -581,7 +579,7 @@ def ssnr( ssnr[segment_counter == 0] = -np.inf # set to zero for channels with no diff (this handles e.g. the corner-case of an all-zero channel in both ref and dut) - zero_diff_mask = np.asarray([np.all(diff_norm[:, c] == 0) for c in range(ref_sig.shape[1])]) + zero_diff_mask = np.asarray([np.all(diff_sig_norm[:, c] == 0) for c in range(ref_sig.shape[1])]) ssnr[zero_diff_mask] = 0 return ssnr -- GitLab From 13a83a72943f75b623bc50253b02d2fe146dafe6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Jun 2024 18:17:41 +0200 Subject: [PATCH 20/25] do not set SSNR to zero for same channels, use inf instead --- scripts/pyaudio3dtools/audioarray.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 96fe7417bb..1175a9d374 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -578,8 +578,4 @@ def ssnr( ssnr = np.round(10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2) ssnr[segment_counter == 0] = -np.inf - # set to zero for channels with no diff (this handles e.g. the corner-case of an all-zero channel in both ref and dut) - zero_diff_mask = np.asarray([np.all(diff_sig_norm[:, c] == 0) for c in range(ref_sig.shape[1])]) - ssnr[zero_diff_mask] = 0 - return ssnr -- GitLab From 625d67484d5f9f751d37e46f8f858042a32f5754 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Jun 2024 08:26:56 +0200 Subject: [PATCH 21/25] add MIN_SSNR to create_report_pages.py --- ci/basop-pages/create_report_pages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index e3557e252b..4eaf1c10f1 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -92,11 +92,12 @@ ARROW_DOWN = '⬂' # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table -COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] +COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] COLUMNS_GLOBAL = COLUMNS[:1] COLUMNS_DIFFERENTIAL = COLUMNS[1:] COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:] + def create_subpage( html_out, csv_out, -- GitLab From acde5b1ae8e77c86085606ca76e781eadd839f7a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Jun 2024 08:29:13 +0200 Subject: [PATCH 22/25] run formatter --- ci/basop-pages/create_report_pages.py | 22 ++++++-- scripts/pyaudio3dtools/audioarray.py | 76 ++++++++++++++++----------- scripts/ssnr.py | 14 +++-- tests/cmp_pcm.py | 15 ++++-- tests/conftest.py | 12 ++++- 5 files changed, 94 insertions(+), 45 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 4eaf1c10f1..c849b160be 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -112,11 +112,18 @@ def create_subpage( ) write_out_csv(merged_reports, merged_reports[0].keys(), csv_out) - table_header_a = "".join([TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL]) + table_header_a = "".join( + [TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL] + ) table_header_b = list() for c in COLUMNS_DIFFERENTIAL: - table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Previous Run
ID: {id_previous}")) - table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Current Run
ID: {id_current}")) + table_header_b.append( + TH_TMPL_SECOND_ROW.format(f"Previous Run
ID: {id_previous}") + ) + table_header_b.append( + TH_TMPL_SECOND_ROW.format(f"Current Run
ID: {id_current}") + ) table_header_b = "".join(table_header_b) table_body = "\n".join( tr_from_row(row, id_current, id_previous) for row in merged_reports @@ -242,8 +249,13 @@ def merge_and_cleanup_mld_reports( return diff - other_col_pairs = [(f"{col}-{id_previous}", f"{col}-{id_current}") for col in COLUMNS_DIFFERENTIAL_NOT_MLD] - merged = sorted(merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True) + other_col_pairs = [ + (f"{col}-{id_previous}", f"{col}-{id_current}") + for col in COLUMNS_DIFFERENTIAL_NOT_MLD + ] + merged = sorted( + merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True + ) # remove the unecessary whole path from the testcase names for row in merged: diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 1175a9d374..efc2573043 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import logging @@ -342,8 +342,14 @@ def compare( for i in range(nchannels): tmpfile_ref = Path(tmpdir).joinpath(f"ref_ch{i+1}.wav") tmpfile_test = Path(tmpdir).joinpath(f"test_ch{i+1}.wav") - r48 = np.clip( resample(ref[:, i].astype(float), fs, 48000), -32768, 32767 ).astype(np.int16) # Convert to float for resample, then to int16 for wavfile.write - t48 = np.clip( resample(test[:, i].astype(float), fs, 48000), -32768, 32767 ).astype(np.int16) + r48 = np.clip( + resample(ref[:, i].astype(float), fs, 48000), -32768, 32767 + ).astype( + np.int16 + ) # Convert to float for resample, then to int16 for wavfile.write + t48 = np.clip( + resample(test[:, i].astype(float), fs, 48000), -32768, 32767 + ).astype(np.int16) wavfile.write(str(tmpfile_ref), 48000, r48) wavfile.write(str(tmpfile_test), 48000, t48) out = subprocess.check_output([mld, tmpfile_ref, tmpfile_test]) @@ -355,7 +361,13 @@ def compare( # length of segment is always 20ms len_seg = int(0.02 * fs) print(len_seg, ref.shape, test.shape) - result["SSNR"] = ssnr(ref, test, len_seg, thresh_low=ssnr_thresh_low, thresh_high=ssnr_thresh_high) + result["SSNR"] = ssnr( + ref, + test, + len_seg, + thresh_low=ssnr_thresh_low, + thresh_high=ssnr_thresh_high, + ) return result @@ -575,7 +587,9 @@ def ssnr( # for that channel and the division here would trigger a warning. We supress the warning and later # set the SSNR for those channels to -inf manually instead (overwriting later is simply easier than adding ifs here) with warnings.catch_warnings(): - ssnr = np.round(10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2) + ssnr = np.round( + 10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2 + ) ssnr[segment_counter == 0] = -np.inf return ssnr diff --git a/scripts/ssnr.py b/scripts/ssnr.py index 80a3d9dd85..e57a3eb047 100644 --- a/scripts/ssnr.py +++ b/scripts/ssnr.py @@ -14,7 +14,9 @@ def main(args): len_seg = int(20 * fs_ref / 1000) print(len_seg, ref_sig.shape, test_sig.shape) - ssnr = audioarray.ssnr(ref_sig, test_sig, len_seg, args.thresh_low, args.thresh_high) + ssnr = audioarray.ssnr( + ref_sig, test_sig, len_seg, args.thresh_low, args.thresh_high + ) for i, s in enumerate(ssnr, start=1): print(f"Channel {i}: {s}") @@ -29,11 +31,15 @@ if __name__ == "__main__": "test_file", type=pathlib.Path, help="Signal under test wav file" ) parser.add_argument( - "--thresh_low", type=float, default="-inf", + "--thresh_low", + type=float, + default="-inf", ) parser.add_argument( - "--thresh_high", type=float, default="inf", - ) + "--thresh_high", + type=float, + default="inf", + ) args = parser.parse_args() diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 3c16e526c9..b3ce4565a5 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -40,8 +40,12 @@ def cmp_pcm( else: nchannels = pyivastest.constants.OC_TO_NCHANNELS[out_config.upper()] - s1, _ = pyaudio3dtools.audiofile.readfile(ref_file, nchannels, fs, outdtype=np.int16) - s2, _ = pyaudio3dtools.audiofile.readfile(cmp_file, nchannels, fs, outdtype=np.int16) + s1, _ = pyaudio3dtools.audiofile.readfile( + ref_file, nchannels, fs, outdtype=np.int16 + ) + s2, _ = pyaudio3dtools.audiofile.readfile( + cmp_file, nchannels, fs, outdtype=np.int16 + ) # In case of wav input, override the nchannels with the one from the wav header nchannels = s1.shape[1] @@ -63,7 +67,12 @@ def cmp_pcm( return 1, reason cmp_result = pyaudio3dtools.audioarray.compare( - s1, s2, fs, per_frame=False, get_mld=get_mld, get_ssnr=get_ssnr, + s1, + s2, + fs, + per_frame=False, + get_mld=get_mld, + get_ssnr=get_ssnr, ) output_differs = 0 diff --git a/tests/conftest.py b/tests/conftest.py index 68a591a18b..52b32c7704 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -366,7 +366,11 @@ class EncoderFrontend: try: result = run( - command, capture_output=True, check=False, timeout=self.timeout, cwd=run_dir + command, + capture_output=True, + check=False, + timeout=self.timeout, + cwd=run_dir, ) except TimeoutExpired: pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.") @@ -565,7 +569,11 @@ class DecoderFrontend: try: result = run( - command, capture_output=True, check=False, timeout=self.timeout, cwd=run_dir + command, + capture_output=True, + check=False, + timeout=self.timeout, + cwd=run_dir, ) except TimeoutExpired: pytest.fail(f"{self._type} decoder run timed out after {self.timeout}s.") -- GitLab From 084a854c88425d889065b0a491d5f64e4c4a405c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Jun 2024 16:47:44 +0200 Subject: [PATCH 23/25] fix for all-zero in both signals --- scripts/pyaudio3dtools/audioarray.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index efc2573043..4d9c5ee2ef 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -561,6 +561,8 @@ def ssnr( if np.all(diff_sig_norm == 0): return np.asarray([np.inf] * ref_sig_norm.shape[1]) + channels_identical_idx = np.sum(np.abs(diff_sig_norm), axis=0) == 0 + denom_add = 10**-13 * len_seg segment_counter = np.zeros(ref_sig.shape[1]) for ref_seg, diff_seg in zip( @@ -592,4 +594,7 @@ def ssnr( ) ssnr[segment_counter == 0] = -np.inf + # this prevents all-zero channels in both signals to be reported as -inf + ssnr[channels_identical_idx] = np.inf + return ssnr -- GitLab From 33f7971a99039d3a88a80bdc4c275d50a276848d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Jun 2024 16:51:13 +0200 Subject: [PATCH 24/25] reintroduce low threshold --- tests/cmp_pcm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index b3ce4565a5..7bf1359f4e 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -73,6 +73,7 @@ def cmp_pcm( per_frame=False, get_mld=get_mld, get_ssnr=get_ssnr, + ssnr_thresh_low=-50, ) output_differs = 0 -- GitLab From eb50ffa6eff82ef2112e67fe1def24aaa91c351c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Jun 2024 12:43:46 +0200 Subject: [PATCH 25/25] apply thresholding on both reference and test signal --- scripts/pyaudio3dtools/audioarray.py | 33 +++++++++++++++++++++++++--- scripts/ssnr.py | 18 ++++++++++++++- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/scripts/pyaudio3dtools/audioarray.py b/scripts/pyaudio3dtools/audioarray.py index 4d9c5ee2ef..0c313c4c91 100644 --- a/scripts/pyaudio3dtools/audioarray.py +++ b/scripts/pyaudio3dtools/audioarray.py @@ -236,6 +236,7 @@ def compare( get_ssnr: bool = False, ssnr_thresh_low: float = -np.inf, ssnr_thresh_high: float = np.inf, + apply_thresholds_to_ref_only: bool = False, ) -> dict: """Compare two audio arrays @@ -251,6 +252,18 @@ def compare( Compute difference per frame (default True) get_mld: bool Run MLD tool if there is a difference between the signals (default False) + get_ssnr: bool + Compute Segmental SNR between signals + ssnr_thresh_low: float + Low threshold for including a segment in the SSNR computation. Per default, both + reference and test signal power are compared to this threshold, see below + ssnr_thresh_high: float + High threshold for including a segment in the SSNR computation. Per default, both + reference and test signal power are compared to this threshold, see below + apply_thresholds_to_ref_only: bool + Set to True to only apply the threshold comparison for the reference signal + for whether to include a segment in the ssnr computation. Use this to align + behaviour with the MPEG-D conformance specification. Returns ------- @@ -367,6 +380,7 @@ def compare( len_seg, thresh_low=ssnr_thresh_low, thresh_high=ssnr_thresh_high, + apply_thresholds_to_ref_only=apply_thresholds_to_ref_only, ) return result @@ -547,6 +561,7 @@ def ssnr( len_seg: int, thresh_low: float = -200, thresh_high: float = 0, + apply_thresholds_to_ref_only: bool = False, ) -> np.ndarray: """ Calculate Segmental SNR for test_sig to ref_sig as defined in ISO/IEC 14496-4 @@ -565,9 +580,12 @@ def ssnr( denom_add = 10**-13 * len_seg segment_counter = np.zeros(ref_sig.shape[1]) - for ref_seg, diff_seg in zip( + + # iterate over test signal too to allow power comparison to threshold + for ref_seg, diff_seg, test_seg in zip( get_framewise(ref_sig_norm, len_seg, zero_pad=True), get_framewise(diff_sig_norm, len_seg, zero_pad=True), + get_framewise(test_sig_norm, len_seg, zero_pad=True), ): nrg_ref = np.sum(ref_seg**2, axis=0) nrg_diff = np.sum(diff_seg**2, axis=0) @@ -579,6 +597,15 @@ def ssnr( ref_power = 10 * np.log10((nrg_ref + 10**-7) / len_seg) zero_mask = np.logical_or(ref_power < thresh_low, ref_power > thresh_high) + # create same mask for test signal + if not apply_thresholds_to_ref_only: + nrg_test = np.sum(test_seg**2, axis=0) + test_power = 10 * np.log10((nrg_test + 10**-7) / len_seg) + zero_mask_test = np.logical_or( + test_power < thresh_low, test_power > thresh_high + ) + zero_mask = np.logical_or(zero_mask, zero_mask_test) + ss_seg[zero_mask] = 0 # increase segment counter only for channels that were not zeroed segment_counter += np.logical_not(zero_mask) @@ -587,12 +614,12 @@ def ssnr( # if the reference signal was outside the thresholds for all segments in a channel, segment_counter is zero # for that channel and the division here would trigger a warning. We supress the warning and later - # set the SSNR for those channels to -inf manually instead (overwriting later is simply easier than adding ifs here) + # set the SSNR for those channels to nan manually instead (overwriting later is simply easier than adding ifs here) with warnings.catch_warnings(): ssnr = np.round( 10 * np.log10(10 ** (np.sum(ss, axis=0) / segment_counter) - 1), 2 ) - ssnr[segment_counter == 0] = -np.inf + ssnr[segment_counter == 0] = np.nan # this prevents all-zero channels in both signals to be reported as -inf ssnr[channels_identical_idx] = np.inf diff --git a/scripts/ssnr.py b/scripts/ssnr.py index e57a3eb047..89ac4c6bf9 100644 --- a/scripts/ssnr.py +++ b/scripts/ssnr.py @@ -15,7 +15,12 @@ def main(args): len_seg = int(20 * fs_ref / 1000) print(len_seg, ref_sig.shape, test_sig.shape) ssnr = audioarray.ssnr( - ref_sig, test_sig, len_seg, args.thresh_low, args.thresh_high + ref_sig, + test_sig, + len_seg, + args.thresh_low, + args.thresh_high, + args.apply_thresholds_on_ref_only, ) for i, s in enumerate(ssnr, start=1): @@ -34,11 +39,22 @@ if __name__ == "__main__": "--thresh_low", type=float, default="-inf", + help="Low threshold for signal power in a segment to be used in the SSNR calculation (default: -inf).\n" + "Applied to both signals per default (see apply_thresholds_on_ref_only argument).", ) parser.add_argument( "--thresh_high", type=float, default="inf", + help="High threshold for signal power in a segment to be used in the SSNR calculation (default: +inf).\n" + "Applied to both signals per default (see apply_thresholds_on_ref_only argument).", + ) + parser.add_argument( + "--apply_thresholds_on_ref_only", + action="store_true", + default=False, + help="Use this to apply the thresholding on signal power to the reference signal only.\n" + "This makes the implementation behaviour conform to the MPEG-D conformance spec.", ) args = parser.parse_args() -- GitLab