add odg diff to pytst suite (d24aaa3d) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/cmp_pcm.py

+47 −0

Original line number	Diff line number	Diff line
		@@ -3,6 +3,11 @@
		import argparse
		import os
		import sys
		import tempfile
		import re
		import subprocess
		from pathlib import Path
		from typing import Optional

		THIS_PATH = os.path.join(os.getcwd(), __file__)
		sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts"))
		@@ -10,6 +15,7 @@ sys.path.append(os.path.join(os.path.dirname(THIS_PATH), "../scripts"))
		import numpy as np
		import pyaudio3dtools
		import pyivastest
		from constants import ODG_PATTERN


		def cmp_pcm(
		@@ -22,6 +28,8 @@ def cmp_pcm(
		mld_lim=0,
		abs_tol=0,
		get_ssnr=False,
		get_odg=False,
		orig_file: Optional[Path]=None
		) -> (int, str):
		"""
		Compare 2 PCM files for bitexactness
		@@ -98,9 +106,48 @@ def cmp_pcm(
		reason += msg + "\n"
		print(msg)

		if get_odg:
		reason += "\n"
		orig_sig, _ = pyaudio3dtools.audiofile.readfile(orig_file)
		for n in range(nchannels):
		pqeval_output_ref = pqevalaudio_wrapper(s1[:, n], orig_sig[:, n])
		pqeval_output_cmp = pqevalaudio_wrapper(s2[:, n], orig_sig[:, n])

		match_ref = re.match(ODG_PATTERN, pqeval_output_ref)
		match_cmp = re.match(ODG_PATTERN, pqeval_output_cmp)

		odg_ref = float(match_ref.groups()[0])
		odg_cmp = float(match_cmp.groups()[0])
		odg_diff = odg_cmp - odg_ref
		msg = f"Channel {n} ODG diff: {odg_diff}"
		reason += msg
		print(msg)

		return output_differs, reason


		def pqevalaudio_wrapper(
		ref_sig: np.ndarray,
		eval_sig: np.ndarray,
		fs: int,
		) -> str:
		with tempfile.TemporaryDirectory() as tmp_dir:
		tmp_dir = Path(tmp_dir)
		tmp_file_ref = str(tmp_dir.joinpath("ref.wav"))
		tmp_file_eval = str(tmp_dir.joinpath("eval.wav"))

		pyaudio3dtools.audiofile.writefile(tmp_file_ref, ref_sig, fs)
		pyaudio3dtools.audiofile.writefile(tmp_file_eval, eval_sig, fs)

		cmd = ["PQevalAudio", tmp_file_ref, tmp_file_eval]

		result = subprocess.run(cmd, capture_output=True)
		if result.returncode != 0:
		raise RuntimeError("Error running PQevalaudio")

		return result.stdout.decode("utf8")


		if __name__ == "__main__":
		parser = argparse.ArgumentParser()
		parser.add_argument("ref_file", type=str)

tests/codec_be_on_mr_nonselection/test_param_file.py

+10 −1

Original line number	Diff line number	Diff line
		@@ -45,7 +45,7 @@ import numpy as np
		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend
		from tests.testconfig import PARAM_FILE
		from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN
		from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN, SSNR_PATTERN, ODG_DIFF_PATTERN

		VALID_DEC_OUTPUT_CONF = [
		"MONO",
		@@ -153,6 +153,7 @@ def test_param_file_tests(
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg,
		):
		enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]

		@@ -351,6 +352,7 @@ def test_param_file_tests(
		abs_tol=abs_tol,
		allow_differing_lengths=allow_differing_lengths,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		)
		md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)

		@@ -365,6 +367,13 @@ def test_param_file_tests(
		record_property("MIN_SSNR", min_ssnr)
		record_property("MIN_SSNR_CHANNEL", min_ssnr_channel)

		if get_odg:
		odg_diffs = re.findall(ODG_DIFF_PATTERN, reason)
		max_odg_diff = max(odg_diffs)
		max_odg_diff_channel = odg_diffs.index(max_odg_diff)
		record_property("MAX_ODG_DIFF", max_odg_diff)
		record_property("MAX_ODG_DIFF_CHANNEL", max_odg_diff_channel)

		max_diff = 0
		if output_differs:
		search_result = re.search(MAX_DIFF_PATTERN, reason)

tests/conftest.py

+14 −0

Original line number	Diff line number	Diff line
		@@ -178,6 +178,12 @@ def pytest_addoption(parser):
		help="Compute Segmental SNR (SSNR) between ref and dut output instead of just comparing for bitexactness",
		)

		parser.addoption(
		"--odg",
		action="store_true",
		help="Get Objective Difference Grade for both conditions during comparison and report difference",
		)

		parser.addoption(
		"--create_ref",
		action="store_true",
		@@ -257,6 +263,14 @@ def get_ssnr(request):
		return request.config.option.ssnr


		@pytest.fixture(scope="session", autouse=True)
		def get_odg(request):
		"""
		Return indication to compute ssnr during ref/dut comparison.
		"""
		return request.config.option.odg


		@pytest.fixture(scope="session")
		def abs_tol(request) -> int:
		"""

tests/constants.py

+2 −0

Original line number	Diff line number	Diff line
		@@ -2,3 +2,5 @@
		MLD_PATTERN = r"MLD: ([\d\.]*)"
		MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)"
		SSNR_PATTERN = r"Channel \d* SSNR: (\d\.\d)"
		ODG_PATTERN = r"Objective Difference Grade: (-\d\.\d*)"
		ODG_DIFF_PATTERN = r"ODG diff: (-\d\.\d*)"