diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 33681ecbeb10f50b1020a33c91c180a71da120f0..fe0d01034b0257caa8fc3bbb400ab85219d1e6c0 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -105,6 +105,7 @@ COLUMNS = [ "MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", + "MIN_ODG", ] COLUMNS_GLOBAL = COLUMNS[:1] COLUMNS_DIFFERENTIAL = COLUMNS[3:] diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 3bc0b619ab19b9f9e97be55f0289797035c22ab5..366e02aba5ae14fc2ca203c5bda1db1699a751f2 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -10,7 +10,7 @@ from xml.etree import ElementTree Parse a junit report and create a summary report. """ -PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR"] +PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"] FORMATS = { "Stereo": r"stereo", diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 7bf1359f4e9fd7249278638e78138bbc64f008c8..3139a32dc91242216dbc2f6a5ac7c01b38cfee25 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -3,6 +3,11 @@ import argparse import os import sys +import tempfile +import re +import subprocess +from pathlib import Path +from typing import Optional THIS_PATH = os.path.join(os.getcwd(), __file__) sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) @@ -10,6 +15,7 @@ sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts")) import numpy as np import pyaudio3dtools import pyivastest +from .constants import ODG_PATTERN_PQEVALAUDIO def cmp_pcm( @@ -22,6 +28,7 @@ def cmp_pcm( mld_lim=0, abs_tol=0, get_ssnr=False, + get_odg=False, ) -> (int, str): """ Compare 2 PCM files for bitexactness @@ -102,15 +109,50 @@ def cmp_pcm( reason += f" > {mld_lim}" if get_ssnr: - reason += "\n" for i, s in enumerate(cmp_result["SSNR"], start=1): msg = f"Channel {i} SSNR: {s}" - reason += msg + "\n" + reason += " - " + msg + print(msg) + + if get_odg: + for n in range(nchannels): + pqeval_output = pqevalaudio_wrapper(s1[:, n], s2[:, n], fs) + + match_odg = re.search(ODG_PATTERN_PQEVALAUDIO, pqeval_output) + odg = float(match_odg.groups()[0]) + msg = f"Channel {n} ODG: {odg}" + reason += " - " + msg print(msg) return output_differs, reason +def pqevalaudio_wrapper( + ref_sig: np.ndarray, + eval_sig: np.ndarray, + fs: int, + ) -> str: + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir = Path(tmp_dir) + tmp_file_ref = str(tmp_dir.joinpath("ref.wav")) + tmp_file_eval = str(tmp_dir.joinpath("eval.wav")) + + # PQevalAudio neeeds 48 kHz sampling rate + r48 = np.clip( pyaudio3dtools.audioarray.resample(ref_sig.astype(float), fs, 48000), -32768, 32767 ).astype(np.int16) + t48 = np.clip( pyaudio3dtools.audioarray.resample(eval_sig.astype(float), fs, 48000), -32768, 32767 ).astype(np.int16) + + pyaudio3dtools.audiofile.writefile(tmp_file_ref, r48, 48000) + pyaudio3dtools.audiofile.writefile(tmp_file_eval, t48, 48000) + + cmd = ["PQevalAudio", tmp_file_ref, tmp_file_eval] + + result = subprocess.run(cmd, capture_output=True) + if result.returncode != 0: + raise RuntimeError(f"Error running PQevalaudio: {result.stderr}") + + return result.stdout.decode("utf8") + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("ref_file", type=str) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 814a6441df7a74d55f8344b10962758ebafc395f..61f40072cd3913d007c4dbb677f8f8c931f36498 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -45,7 +45,6 @@ from tests.cmp_pcm import cmp_pcm from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties from tests.testconfig import PARAM_FILE - VALID_DEC_OUTPUT_CONF = [ "MONO", "STEREO", @@ -153,6 +152,7 @@ def test_param_file_tests( get_mld_lim, abs_tol, get_ssnr, + get_odg, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -351,6 +351,7 @@ def test_param_file_tests( abs_tol=abs_tol, allow_differing_lengths=allow_differing_lengths, get_ssnr=get_ssnr, + get_odg=get_odg, ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index e188d4a11454ff08769f4477a2dcec76ff63af59..d191f87e10da9ef0d835d1045366410f70c545a2 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -94,6 +94,7 @@ def test_sba_plc_system( get_mld_lim, abs_tol, get_ssnr, + get_odg ): SID = 0 if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: @@ -135,6 +136,7 @@ def test_sba_plc_system( get_mld_lim=get_mld_lim, abs_tol=abs_tol, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -161,6 +163,7 @@ def sba_dec_plc( get_mld_lim=0, abs_tol=0, get_ssnr=False, + get_odg=False, ): # ------------ run cmd ------------ @@ -219,6 +222,7 @@ def sba_dec_plc( mld_lim=get_mld_lim, abs_tol=abs_tol, get_ssnr=get_ssnr, + get_odg=get_odg, ) props = parse_properties(reason, cmp_result!=0, props_to_record) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index ce99c55a272c9fa69459d34bea7903e126586965..3dd7e6dc6714c9d876735cff5273a9d52f726ed7 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -108,6 +108,7 @@ def test_pca_enc( decoder_only, abs_tol, get_ssnr, + get_odg, ): pca = True tag = tag + fs + "c" @@ -165,6 +166,7 @@ def test_pca_enc( pca=pca, abs_tol=abs_tol, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -199,6 +201,7 @@ def test_sba_enc_system( decoder_only, abs_tol, get_ssnr, + get_odg, ): if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -276,6 +279,7 @@ def test_sba_enc_system( get_mld_lim=get_mld_lim, abs_tol=abs_tol, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -301,6 +305,7 @@ def test_spar_hoa2_enc_system( decoder_only, abs_tol, get_ssnr, + get_odg, ): fs = "48" dtx = "0" @@ -355,6 +360,7 @@ def test_spar_hoa2_enc_system( get_mld_lim=get_mld_lim, abs_tol=abs_tol, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -380,6 +386,7 @@ def test_spar_hoa3_enc_system( decoder_only, abs_tol, get_ssnr, + get_odg, ): fs = "48" dtx = "0" @@ -434,6 +441,7 @@ def test_spar_hoa3_enc_system( get_mld_lim=get_mld_lim, abs_tol=abs_tol, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -463,6 +471,7 @@ def test_sba_enc_BWforce_system( decoder_only, abs_tol, get_ssnr, + get_odg, ): if dtx == "1" and ivas_br not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -523,6 +532,7 @@ def test_sba_enc_BWforce_system( get_mld_lim=get_mld_lim, abs_tol=abs_tol, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -686,6 +696,7 @@ def sba_dec( pca=False, abs_tol=0, get_ssnr=False, + get_odg=False, ): # -------- run cmd ------------ # sampling rate to BW mapping @@ -751,6 +762,7 @@ def sba_dec( mld_lim=get_mld_lim, abs_tol=abs_tol, get_ssnr=get_ssnr, + get_odg=get_odg, ) props = parse_properties(reason, cmp_result!=0, props_to_record) diff --git a/tests/conftest.py b/tests/conftest.py index 24446c9678fa25d0475979f71ccd65aa21ea2e49..c5d827569f63a4361ead420df5e83d07357f11c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,7 +42,7 @@ import textwrap from pathlib import Path from subprocess import TimeoutExpired, run from typing import Optional, Union -from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN +from .constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN, ODG_PATTERN logger = logging.getLogger(__name__) USE_LOGGER_FOR_DBG = False # current tests do not make use of the logger feature @@ -180,6 +180,12 @@ def pytest_addoption(parser): help="Compute Segmental SNR (SSNR) between ref and dut output instead of just comparing for bitexactness", ) + parser.addoption( + "--odg", + action="store_true", + help="Get Objective Difference Grade for both conditions during comparison and report difference", + ) + parser.addoption( "--create_ref", action="store_true", @@ -259,6 +265,14 @@ def get_ssnr(request): return request.config.option.ssnr +@pytest.fixture(scope="session", autouse=True) +def get_odg(request): + """ + Return indication to compute ssnr during ref/dut comparison. + """ + return request.config.option.odg + + @pytest.fixture(scope="session") def abs_tol(request) -> int: """ @@ -566,11 +580,20 @@ class DecoderFrontend: raise ValueError(f'Wrong system "{system}"!') if not os.path.isfile(netsim_path): - raise FileNotFoundError(f"network simulator binary {netsim_path} not found!\n") + raise FileNotFoundError( + f"network simulator binary {netsim_path} not found!\n" + ) netsim_bitstream_path = input_bitstream_path.with_suffix(".netsimout") tracefile_path = input_bitstream_path.with_suffix(".netsimtrace") # TODO: need to check if the "1" works with every profile - netsim_command = [netsim_path, netsim_profile, input_bitstream_path, netsim_bitstream_path, tracefile_path, "1"] + netsim_command = [ + netsim_path, + netsim_profile, + input_bitstream_path, + netsim_bitstream_path, + tracefile_path, + "1", + ] print(netsim_command) try: run(netsim_command, check=True, cwd=run_dir) @@ -810,12 +833,14 @@ def pytest_configure(config): @pytest.fixture(scope="session") -def props_to_record(request, get_mld, get_ssnr) -> str: +def props_to_record(request, get_mld, get_ssnr, get_odg) -> str: props = ["MAXIMUM ABS DIFF"] if get_mld: props.append("MLD") if get_ssnr: props.append("SSNR") + if get_odg: + props.append("ODG") return props @@ -845,6 +870,13 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: min_ssnr_channel = ssnrs.index(min_ssnr) props["MIN_SSNR"] = min_ssnr props["MIN_SSNR_CHANNEL"] = min_ssnr_channel + elif prop == "ODG": + odgs = re.findall(ODG_PATTERN, text_to_parse) + print(odgs) + min_odg = min(odgs) + min_odg_channel = odgs.index(min_odg) + props["MIN_ODG"] = min_odg + props["MIN_ODG_CHANNEL"] = min_odg_channel return props diff --git a/tests/constants.py b/tests/constants.py index 0f985d0b567057b5ad7cd074fd4c7ebb1502ddf5..e6ea1c1589fbfb9e16d71776737599ac0bfa25e5 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -7,4 +7,6 @@ TESTV_DIR = SCRIPTS_DIR.joinpath("testv") # regex patterns for parsing the output from cmp_pcm -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" +ODG_PATTERN_PQEVALAUDIO = r"Objective Difference Grade: (-*\d*\.\d*)" +ODG_PATTERN = r"ODG: (-*\d*\.\d*)" SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)"