Merge branch 'ci/enable-mld-for-test-26444' into 'main' (782844df) · Commits · IVAS Codec Public Collaboration / IVAS Codec

scripts/create_histogram_summary.py

+14 −1

Original line number	Diff line number	Diff line
		@@ -3,14 +3,16 @@
		import argparse
		import math
		import numpy as np

		# These next three lines are added as a precaution in case the gitlab runner
		# needs DISPLAY to render the plots, even if they are written to file.
		import matplotlib

		matplotlib.use("Agg")
		import matplotlib.pyplot as plt
		import csv
		import os
		from parse_xml_report import FORMATS, CATEGORIES
		from parse_xml_report import IVAS_FORMATS, EVS_FORMATS, IVAS_CATEGORIES, EVS_CATEGORIES

		"""
		Parses a CSV report and creates a summary report.
		@@ -44,11 +46,22 @@ if __name__ == "__main__":
		help="Measure, any of: MLD, DIFF, SSNR, default: MLD",
		default=["MLD"],
		)
		parser.add_argument(
		"--evs",
		action="store_true",
		help="Parse using EVS 26.444 formats",
		)
		args = parser.parse_args()
		csv_report = args.csv_report
		csv_summary = args.csv_summary
		csv_image = args.csv_image
		measure = args.measure[0]
		if args.evs:
		FORMATS = EVS_FORMATS
		CATEGORIES = EVS_CATEGORIES
		else:
		FORMATS = IVAS_FORMATS
		CATEGORIES = IVAS_CATEGORIES

		limits_per_measure = {
		"MLD": ("MLD", [0, 5, 10, math.inf]),

scripts/parse_xml_report.py

+30 −3

Original line number	Diff line number	Diff line
		@@ -12,7 +12,7 @@ Parse a junit report and create a summary report.

		PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"]

		FORMATS = {
		IVAS_FORMATS = {
		"Stereo": r"stereo",
		"ISM": r"ISM",
		"Multichannel": r"Multi-channel",
		@@ -23,7 +23,16 @@ FORMATS = {
		"Renderer": r"renderer",
		}

		CATEGORIES = {
		EVS_FORMATS = {
		"AMRWBIO_dec": r"Readme_AMRWB_IO_dec",
		"AMRWBIO_enc": r"Readme_AMRWB_IO_enc",
		"EVS_dec": r"Readme_EVS_dec",
		"EVS_enc": r"Readme_EVS_enc",
		"EVS_JBM_dec": r"Readme_JBM_dec",
		}


		IVAS_CATEGORIES = {
		"Normal operation": r".*",
		"DTX": r"DTX",
		"PLC": r"%",
		@@ -31,6 +40,14 @@ CATEGORIES = {
		"JBM": r"JBM",
		}

		EVS_CATEGORIES = {
		"Normal operation": r".*",
		"DTX": r"DTX",
		"PLC": r"b10\|f06",
		"Bitrate switching": r"sw",
		"JBM": r"JBM",
		}

		# Main routine
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		@@ -42,10 +59,20 @@ if __name__ == "__main__":
		help="XML junit report input file, e.g. report-junit.xml",
		)
		parser.add_argument("csv_file", type=str, help="Output CSV file, e.g. report.csv")
		parser.add_argument(
		"--evs",
		action="store_true",
		help="Parse using EVS 26.444 formats",
		)
		args = parser.parse_args()
		xml_report = args.xml_report
		csv_file = args.csv_file

		if args.evs:
		FORMATS = EVS_FORMATS
		CATEGORIES = EVS_CATEGORIES
		else:
		FORMATS = IVAS_FORMATS
		CATEGORIES = IVAS_CATEGORIES
		tree = ElementTree.parse(xml_report)

		testsuite = tree.find(".//testsuite")

tests/cmp_pcm.py

+15 −6

Original line number	Diff line number	Diff line
		@@ -138,8 +138,16 @@ def pqevalaudio_wrapper(
		tmp_file_eval = str(tmp_dir.joinpath("eval.wav"))

		# PQevalAudio neeeds 48 kHz sampling rate
		r48 = np.clip( pyaudio3dtools.audioarray.resample(ref_sig.astype(float), fs, 48000), -32768, 32767 ).astype(np.int16)
		t48 = np.clip( pyaudio3dtools.audioarray.resample(eval_sig.astype(float), fs, 48000), -32768, 32767 ).astype(np.int16)
		r48 = np.clip(
		pyaudio3dtools.audioarray.resample(ref_sig.astype(float), fs, 48000),
		-32768,
		32767,
		).astype(np.int16)
		t48 = np.clip(
		pyaudio3dtools.audioarray.resample(eval_sig.astype(float), fs, 48000),
		-32768,
		32767,
		).astype(np.int16)

		pyaudio3dtools.audiofile.writefile(tmp_file_ref, r48, 48000)
		pyaudio3dtools.audiofile.writefile(tmp_file_eval, t48, 48000)
		@@ -167,6 +175,7 @@ if __name__ == "__main__":
		parser.add_argument("-s", "--sampling_rate", type=int, default=48000, dest="fs")
		parser.add_argument("--get_mld", action="store_true")
		parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim")
		parser.add_argument("--get_odg", action="store_true")
		args = parser.parse_args()

		result, msg = cmp_pcm(**vars(args))

tests/test_26444.py

+114 −55

Original line number	Diff line number	Diff line
		@@ -34,9 +34,13 @@ Execute tests specified via a parameter file.

		import filecmp
		import os
		import subprocess

		import re
		import pytest
		import shutil

		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend, parse_properties


		test_dict = {}
		TEST_DIR = "evs_be_test"
		@@ -47,6 +51,7 @@ scripts = [
		"Readme_EVS_enc.txt",
		"Readme_JBM_dec.txt",
		]

		for s in scripts:
		with open(os.path.join(TEST_DIR, s), "r", encoding="UTF-8") as fp:
		tag = ""
		@@ -70,62 +75,116 @@ for s in scripts:
		dec_opts = ""
		diff_opts = ""


		@pytest.mark.parametrize("test_tag", list(test_dict.keys()))
		def test_evs_26444(runner_frontend, test_tag):
		def test_evs_26444(
		record_property,
		props_to_record,
		dut_encoder_frontend: EncoderFrontend,
		dut_decoder_frontend: DecoderFrontend,
		ref_decoder_frontend: DecoderFrontend,
		test_tag,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg,
		):
		enc_opts, dec_opts, diff_opts = test_dict[test_tag]

		result = None
		if enc_opts:
		enc_opts = enc_opts.replace("./", TEST_DIR + "/")
		enc_opts = enc_opts.replace("-rf rf_config.cfg", "-rf " + TEST_DIR + "/rf_config.cfg") # Special handling of this arguments since the path is missing
		cmd = ["./IVAS_cod", "-q"] + enc_opts.split()[1:]
		print(" ".join(["Encoder command: "] + cmd))
		runner_frontend.run(cmd)
		if dec_opts:
		dec_opts = dec_opts.replace("./", TEST_DIR + "/")
		cmd = ["./IVAS_dec", "-q"] + dec_opts.split()[1:]
		print(" ".join(["Decoder command: "] + cmd))
		runner_frontend.run(cmd)

		result = runner_frontend.result

		if result != None and result.returncode:
		pytest.fail("Non-zero returncode for command: " + " ".join(cmd))

		diff_opts = diff_opts.replace("./", TEST_DIR + "/")
		if ";" in diff_opts:
		cmd1, cmd2 = diff_opts.split(";")
		cmd1 = cmd1.split()
		cmd2 = cmd2.split()
		result1 = filecmp.cmp(cmd1[0], cmd1[1])
		result2 = filecmp.cmp(cmd2[2], cmd2[3])
		else:
		cmd1 = diff_opts.split()
		result1 = filecmp.cmp(cmd1[0], cmd1[1])
		result2 = True
		if not (result1 and result2):
		pytest.fail("Output differs")


		class Runner:
		def __init__(self) -> None:
		self.returncode = None
		self.result = None

		def run(self, cmd: str) -> None:
		result = subprocess.run(cmd, capture_output=True, check=False)
		self.result = result
		self.returncode = result.returncode
		if enc_opts:
		args = enc_opts.split()[1:]

		bitrate = args[-4]
		sampling_rate = args[-3]
		in_file = args[-2]
		out_file = args[-1]
		add_option_list = args[:-4]

		dut_encoder_frontend.run(
		bitrate,
		sampling_rate,
		in_file,
		out_file,
		add_option_list=add_option_list,
		run_dir=TEST_DIR,
		)

		def _check_run(self):
		if self.returncode is not None:
		if self.returncode:
		pytest.fail( "Command terminated with a non-0 return code" )
		if dec_opts:
		args = dec_opts.split()[1:]
		output_config = "" # Empty for EVS operation
		sampling_rate = args[-3]
		in_file = args[-2]
		out_file = args[-1]
		add_option_list = args[:-3]

		dut_decoder_frontend.run(
		output_config,
		sampling_rate,
		in_file,
		out_file,
		add_option_list=add_option_list,
		run_dir=TEST_DIR,
		)

		# Run comparison on encoder and decoder test cases
		equal = True
		for diff_opt in diff_opts.split(";"):
		pattern = r"(\$DIFF_BIN\s?-w\s?)?"
		diff_opt = re.sub(pattern, "", diff_opt).strip()
		[ref, test] = diff_opt.split()[:2]
		# Run audio file comparison if MLD/SSNR/PEAQ is requested and the test is either a bitstream or audio (not JBM tracefile)
		# and the ref_decoder_path is specified (otherwise ref_decoder_frontend is None)
		if (
		(get_mld or get_ssnr or get_odg)
		and ("COD" in ref or "OUT" in ref)
		and ref_decoder_frontend
		):
		if enc_opts:
		for file in [ref, test]:
		output_config = "" # Empty for EVS operation
		in_file = file
		out_file = file + ".wav"
		add_option_list = []
		ref_decoder_frontend.run(
		output_config,
		sampling_rate,
		in_file,
		out_file,
		add_option_list=add_option_list,
		)
		fs = int(sampling_rate) * 1000
		reffile = ref + ".wav"
		testfile = test + ".wav"
		else:
		fs = int(re.search(r"(\d+)kHz", ref).group(1)) * 1000
		# pyaudio3dtools.audiofile.readfile only handles .wav, .pcm and .raw suffixes.
		reffile = ref + ".pcm"
		testfile = test + ".pcm"
		shutil.copy(ref, reffile)
		shutil.copy(test, testfile)
		output_differs, reason = cmp_pcm(
		reffile,
		testfile,
		output_config,
		fs,
		get_mld=get_mld,
		mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		allow_differing_lengths=False,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		)

		props = parse_properties(reason, output_differs, props_to_record)
		for k, v in props.items():
		record_property(k, v)
		equal &= not output_differs

		@pytest.fixture(scope="function")
		def runner_frontend() -> Runner:
		runner = Runner()
		yield runner
		else:
		equal &= filecmp.cmp(ref, test)

		# Fixture teardown
		runner._check_run()
		if not equal:
		pytest.fail("Output differs")

Admin message