diff --git a/scripts/create_histogram_summary.py b/scripts/create_histogram_summary.py index 1ab951ac72af1f926a3a0bc7d5ba08bda781a9ef..6e940c883af20185c5559cbd85428867b237eb5c 100644 --- a/scripts/create_histogram_summary.py +++ b/scripts/create_histogram_summary.py @@ -3,14 +3,16 @@ import argparse import math import numpy as np + # These next three lines are added as a precaution in case the gitlab runner # needs DISPLAY to render the plots, even if they are written to file. import matplotlib + matplotlib.use("Agg") import matplotlib.pyplot as plt import csv import os -from parse_xml_report import FORMATS, CATEGORIES +from parse_xml_report import IVAS_FORMATS, EVS_FORMATS, IVAS_CATEGORIES, EVS_CATEGORIES """ Parses a CSV report and creates a summary report. @@ -44,11 +46,22 @@ if __name__ == "__main__": help="Measure, any of: MLD, DIFF, SSNR, default: MLD", default=["MLD"], ) + parser.add_argument( + "--evs", + action="store_true", + help="Parse using EVS 26.444 formats", + ) args = parser.parse_args() csv_report = args.csv_report csv_summary = args.csv_summary csv_image = args.csv_image measure = args.measure[0] + if args.evs: + FORMATS = EVS_FORMATS + CATEGORIES = EVS_CATEGORIES + else: + FORMATS = IVAS_FORMATS + CATEGORIES = IVAS_CATEGORIES limits_per_measure = { "MLD": ("MLD", [0, 5, 10, math.inf]), diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 366e02aba5ae14fc2ca203c5bda1db1699a751f2..3be7a9ca0b62e62d2469a07c9142a8fdd3879a84 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -12,7 +12,7 @@ Parse a junit report and create a summary report. PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"] -FORMATS = { +IVAS_FORMATS = { "Stereo": r"stereo", "ISM": r"ISM", "Multichannel": r"Multi-channel", @@ -23,7 +23,16 @@ FORMATS = { "Renderer": r"renderer", } -CATEGORIES = { +EVS_FORMATS = { + "AMRWBIO_dec": r"Readme_AMRWB_IO_dec", + "AMRWBIO_enc": r"Readme_AMRWB_IO_enc", + "EVS_dec": r"Readme_EVS_dec", + "EVS_enc": r"Readme_EVS_enc", + "EVS_JBM_dec": r"Readme_JBM_dec", +} + + +IVAS_CATEGORIES = { "Normal operation": r".*", "DTX": r"DTX", "PLC": r"%", @@ -31,6 +40,14 @@ CATEGORIES = { "JBM": r"JBM", } +EVS_CATEGORIES = { + "Normal operation": r".*", + "DTX": r"DTX", + "PLC": r"b10|f06", + "Bitrate switching": r"sw", + "JBM": r"JBM", +} + # Main routine if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -42,10 +59,20 @@ if __name__ == "__main__": help="XML junit report input file, e.g. report-junit.xml", ) parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. report.csv") + parser.add_argument( + "--evs", + action="store_true", + help="Parse using EVS 26.444 formats", + ) args = parser.parse_args() xml_report = args.xml_report csv_file = args.csv_file - + if args.evs: + FORMATS = EVS_FORMATS + CATEGORIES = EVS_CATEGORIES + else: + FORMATS = IVAS_FORMATS + CATEGORIES = IVAS_CATEGORIES tree = ElementTree.parse(xml_report) testsuite = tree.find(".//testsuite") diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index e299047492f8a3d5922d2c1d750253556f07616c..5b335840e6634ed3286c4fe0ecc99eaa2bad3302 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -128,18 +128,26 @@ def cmp_pcm( def pqevalaudio_wrapper( - ref_sig: np.ndarray, - eval_sig: np.ndarray, - fs: int, - ) -> str: + ref_sig: np.ndarray, + eval_sig: np.ndarray, + fs: int, +) -> str: with tempfile.TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) tmp_file_ref = str(tmp_dir.joinpath("ref.wav")) tmp_file_eval = str(tmp_dir.joinpath("eval.wav")) # PQevalAudio neeeds 48 kHz sampling rate - r48 = np.clip( pyaudio3dtools.audioarray.resample(ref_sig.astype(float), fs, 48000), -32768, 32767 ).astype(np.int16) - t48 = np.clip( pyaudio3dtools.audioarray.resample(eval_sig.astype(float), fs, 48000), -32768, 32767 ).astype(np.int16) + r48 = np.clip( + pyaudio3dtools.audioarray.resample(ref_sig.astype(float), fs, 48000), + -32768, + 32767, + ).astype(np.int16) + t48 = np.clip( + pyaudio3dtools.audioarray.resample(eval_sig.astype(float), fs, 48000), + -32768, + 32767, + ).astype(np.int16) pyaudio3dtools.audiofile.writefile(tmp_file_ref, r48, 48000) pyaudio3dtools.audiofile.writefile(tmp_file_eval, t48, 48000) @@ -167,6 +175,7 @@ if __name__ == "__main__": parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs") parser.add_argument("--get_mld", action="store_true") parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim") + parser.add_argument("--get_odg", action="store_true") args = parser.parse_args() result, msg = cmp_pcm(**vars(args)) diff --git a/tests/test_26444.py b/tests/test_26444.py index d23c01e8a83d389eb1e6b07d2273236391334039..fa6214c66a8c5164ba636346839c8041aa7ae197 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -34,9 +34,13 @@ Execute tests specified via a parameter file. import filecmp import os -import subprocess - +import re import pytest +import shutil + +from tests.cmp_pcm import cmp_pcm +from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties + test_dict = {} TEST_DIR = "evs_be_test" @@ -47,6 +51,7 @@ scripts = [ "Readme_EVS_enc.txt", "Readme_JBM_dec.txt", ] + for s in scripts: with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp: tag = "" @@ -70,62 +75,116 @@ for s in scripts: dec_opts = "" diff_opts = "" + @pytest.mark.parametrize("test_tag", list(test_dict.keys())) -def test_evs_26444(runner_frontend, test_tag): +def test_evs_26444( + record_property, + props_to_record, + dut_encoder_frontend: EncoderFrontend, + dut_decoder_frontend: DecoderFrontend, + ref_decoder_frontend: DecoderFrontend, + test_tag, + get_mld, + get_mld_lim, + abs_tol, + get_ssnr, + get_odg, +): enc_opts, dec_opts, diff_opts = test_dict[test_tag] - - result = None - if enc_opts: - enc_opts = enc_opts.replace("./", TEST_DIR + "/") - enc_opts = enc_opts.replace("-rf rf_config.cfg", "-rf " + TEST_DIR + "/rf_config.cfg") # Special handling of this arguments since the path is missing - cmd = ["./IVAS_cod", "-q"] + enc_opts.split()[1:] - print(" ".join(["Encoder command: "] + cmd)) - runner_frontend.run(cmd) - if dec_opts: - dec_opts = dec_opts.replace("./", TEST_DIR + "/") - cmd = ["./IVAS_dec", "-q"] + dec_opts.split()[1:] - print(" ".join(["Decoder command: "] + cmd)) - runner_frontend.run(cmd) - - result = runner_frontend.result - - if result != None and result.returncode: - pytest.fail("Non-zero returncode for command: " + " ".join(cmd)) diff_opts = diff_opts.replace("./", TEST_DIR + "/") - if ";" in diff_opts: - cmd1, cmd2 = diff_opts.split(";") - cmd1 = cmd1.split() - cmd2 = cmd2.split() - result1 = filecmp.cmp(cmd1[0], cmd1[1]) - result2 = filecmp.cmp(cmd2[2], cmd2[3]) - else: - cmd1 = diff_opts.split() - result1 = filecmp.cmp(cmd1[0], cmd1[1]) - result2 = True - if not (result1 and result2): - pytest.fail("Output differs") + if enc_opts: + args = enc_opts.split()[1:] + + bitrate = args[-4] + sampling_rate = args[-3] + in_file = args[-2] + out_file = args[-1] + add_option_list = args[:-4] + + dut_encoder_frontend.run( + bitrate, + sampling_rate, + in_file, + out_file, + add_option_list=add_option_list, + run_dir=TEST_DIR, + ) -class Runner: - def __init__(self) -> None: - self.returncode = None - self.result = None - - def run(self, cmd: str) -> None: - result = subprocess.run(cmd, capture_output=True, check=False) - self.result = result - self.returncode = result.returncode - - def _check_run(self): - if self.returncode is not None: - if self.returncode: - pytest.fail( "Command terminated with a non-0 return code" ) - -@pytest.fixture(scope="function") -def runner_frontend() -> Runner: - runner = Runner() - yield runner - - # Fixture teardown - runner._check_run() + if dec_opts: + args = dec_opts.split()[1:] + output_config = "" # Empty for EVS operation + sampling_rate = args[-3] + in_file = args[-2] + out_file = args[-1] + add_option_list = args[:-3] + + dut_decoder_frontend.run( + output_config, + sampling_rate, + in_file, + out_file, + add_option_list=add_option_list, + run_dir=TEST_DIR, + ) + + # Run comparison on encoder and decoder test cases + equal = True + for diff_opt in diff_opts.split(";"): + pattern = r"(\$DIFF_BIN\s?-w\s?)?" + diff_opt = re.sub(pattern, "", diff_opt).strip() + [ref, test] = diff_opt.split()[:2] + # Run audio file comparison if MLD/SSNR/PEAQ is requested and the test is either a bitstream or audio (not JBM tracefile) + # and the ref_decoder_path is specified (otherwise ref_decoder_frontend is None) + if ( + (get_mld or get_ssnr or get_odg) + and ("COD" in ref or "OUT" in ref) + and ref_decoder_frontend + ): + if enc_opts: + for file in [ref, test]: + output_config = "" # Empty for EVS operation + in_file = file + out_file = file + ".wav" + add_option_list = [] + ref_decoder_frontend.run( + output_config, + sampling_rate, + in_file, + out_file, + add_option_list=add_option_list, + ) + fs = int(sampling_rate) * 1000 + reffile = ref + ".wav" + testfile = test + ".wav" + else: + fs = int(re.search(r"(\d+)kHz", ref).group(1)) * 1000 + # pyaudio3dtools.audiofile.readfile only handles .wav, .pcm and .raw suffixes. + reffile = ref + ".pcm" + testfile = test + ".pcm" + shutil.copy(ref, reffile) + shutil.copy(test, testfile) + output_differs, reason = cmp_pcm( + reffile, + testfile, + output_config, + fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + allow_differing_lengths=False, + get_ssnr=get_ssnr, + get_odg=get_odg, + ) + + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) + equal &= not output_differs + + else: + equal &= filecmp.cmp(ref, test) + + if not equal: + pytest.fail("Output differs")