take tracefiles into account for MLD comparisons (697d0f42) · Commits · IVAS Codec Public Collaboration / IVAS Codec

scripts/pyaudio3dtools/audioarray.py

+55 −25

Original line number	Diff line number	Diff line
		@@ -239,6 +239,8 @@ def compare(
		ssnr_thresh_high: float = np.inf,
		apply_thresholds_to_ref_only: bool = False,
		test_start_offset_ms: int = 0,
		ref_jbm_tf: Optional[Path] = None,
		test_jbm_tf: Optional[Path] = None,
		) -> dict:
		"""Compare two audio arrays

		@@ -284,6 +286,10 @@ def compare(
		test = test[test_start_offset_samples:, :]

		framesize = fs // 50
		if ref.shape[0] != test.shape[0]:
		min_len = min(ref.shape[0], test.shape[0])
		diff = abs(test[:min_len, :] - ref[:min_len, :])
		else:
		diff = abs(test - ref)
		max_diff = int(diff.max())
		result = {
		@@ -355,37 +361,61 @@ def compare(
		result["nframes_diff_percentage"] = nframes_diff_percentage

		if get_mld:

		def parse_wav_diff(proc: subprocess.CompletedProcess) -> int:
		mld_max = 0
		for line in proc.stdout.splitlines():
		if line.strip().startswith("MLD:"):
		start = line.find("max:") + 4
		stop = line.find("@")
		mld = float(line[start:stop].strip())
		if mld > mld_max:
		mld_max = mld

		# TODO probably needs a fix to show up in pytest
		if proc.returncode:
		print(f"{proc.stderr}\n{proc.stdout}")
		return mld_max

		mld_max = 0
		toolsdir = Path(__file__).parent.parent.joinpath("tools")

		curr_platform = platform.system()
		if curr_platform not in {"Windows", "Linux", "Darwin"}:
		raise NotImplementedError(f"MLD tool not available for {curr_platform}")
		raise NotImplementedError(
		f"wav-diff tool not available for {curr_platform}"
		)
		search_path = toolsdir.joinpath(curr_platform.replace("Windows", "Win32"))
		mld = search_path.joinpath("mld")
		wdiff = search_path.joinpath("wav-diff")

		if not mld.exists():
		mld = shutil.which("mld")
		if mld is None:
		raise FileNotFoundError(f"MLD tool not found in {search_path} or PATH!")
		warnings.warn(f"MLD binary not found in {search_path}! Falling back to {mld}!")
		if not wdiff.exists():
		wdiff = shutil.which("wav-diff")
		if wdiff is None:
		raise FileNotFoundError(
		f"wav-diff tool not found in {search_path} or PATH!"
		)
		warnings.warn(
		f"MLD binary not found in {search_path}! Falling back to {wdiff}!"
		)

		with tempfile.TemporaryDirectory() as tmpdir:
		for i in range(nchannels):
		tmpfile_ref = Path(tmpdir).joinpath(f"ref_ch{i+1}.wav")
		tmpfile_test = Path(tmpdir).joinpath(f"test_ch{i+1}.wav")
		r48 = np.clip(
		resample(ref[:, i].astype(float), fs, 48000), -32768, 32767
		).astype(
		np.int16
		) # Convert to float for resample, then to int16 for wavfile.write
		t48 = np.clip(
		resample(test[:, i].astype(float), fs, 48000), -32768, 32767
		).astype(np.int16)
		wavfile.write(str(tmpfile_ref), 48000, r48)
		wavfile.write(str(tmpfile_test), 48000, t48)
		out = subprocess.check_output([mld, tmpfile_ref, tmpfile_test])
		mld_max = max(mld_max, float(out.split()[3]))
		tmpfile_ref = Path(tmpdir).joinpath("ref.wav")
		tmpfile_test = Path(tmpdir).joinpath("test.wav")
		wavfile.write(str(tmpfile_ref), fs, ref.astype(np.int16))
		wavfile.write(str(tmpfile_test), fs, test.astype(np.int16))

		cmd = [str(wdiff), str(tmpfile_ref), str(tmpfile_test)]
		if ref_jbm_tf and test_jbm_tf:
		cmd.extend(
		[
		"--ref-jbm-trace",
		str(ref_jbm_tf),
		"--cut-jbm-trace",
		str(test_jbm_tf),
		]
		)
		proc = subprocess.run(cmd, capture_output=True, text=True)
		mld_max = parse_wav_diff(proc)

		result["MLD"] = mld_max

tests/cmp_pcm.py

+19 −12

Original line number	Diff line number	Diff line
		@@ -29,6 +29,8 @@ def cmp_pcm(
		abs_tol=0,
		get_ssnr=False,
		get_odg=False,
		ref_jbm_tf: Optional[Path] = None,
		cut_jbm_tf: Optional[Path] = None,
		) -> (int, str):
		"""
		Compare 2 PCM files for bitexactness
		@@ -57,12 +59,7 @@ def cmp_pcm(
		# In case of wav input, override the nchannels with the one from the wav header
		nchannels = s1.shape[1]

		if allow_differing_lengths:
		# to allow for MLD comparison, shorten longer file
		min_len = min(s1.shape[0], s2.shape[0])
		s1 = s1[:min_len, :]
		s2 = s2[:min_len, :]
		elif s1.shape != s2.shape:
		if s1.shape != s2.shape and not allow_differing_lengths:
		print(
		f"file size in samples: file 1 = {s1.shape[0]},",
		f"file 2 = {s2.shape[0]}",
		@@ -81,6 +78,8 @@ def cmp_pcm(
		get_mld=get_mld,
		get_ssnr=get_ssnr,
		ssnr_thresh_low=-50,
		ref_jbm_tf=ref_jbm_tf,
		test_jbm_tf=cut_jbm_tf,
		)

		output_differs = 0
		@@ -138,8 +137,16 @@ def pqevalaudio_wrapper(
		tmp_file_eval = str(tmp_dir.joinpath("eval.wav"))

		# PQevalAudio neeeds 48 kHz sampling rate
		r48 = np.clip( pyaudio3dtools.audioarray.resample(ref_sig.astype(float), fs, 48000), -32768, 32767 ).astype(np.int16)
		t48 = np.clip( pyaudio3dtools.audioarray.resample(eval_sig.astype(float), fs, 48000), -32768, 32767 ).astype(np.int16)
		r48 = np.clip(
		pyaudio3dtools.audioarray.resample(ref_sig.astype(float), fs, 48000),
		-32768,
		32767,
		).astype(np.int16)
		t48 = np.clip(
		pyaudio3dtools.audioarray.resample(eval_sig.astype(float), fs, 48000),
		-32768,
		32767,
		).astype(np.int16)

		pyaudio3dtools.audiofile.writefile(tmp_file_ref, r48, 48000)
		pyaudio3dtools.audiofile.writefile(tmp_file_eval, t48, 48000)

tests/codec_be_on_mr_nonselection/test_param_file.py

+5 −0

Original line number	Diff line number	Diff line
		@@ -333,6 +333,9 @@ def test_param_file_tests(
		dut_rtp_num_last = np.genfromtxt(dut_tracefile_dec, delimiter=";", usecols=[0])[-1]
		ref_rtp_num_last = np.genfromtxt(ref_tracefile_dec, delimiter=";", usecols=[0])[-1]
		tracefile_last_rtp_numbers_differ = dut_rtp_num_last != ref_rtp_num_last
		else:
		dut_tracefile_dec = None
		ref_tracefile_dec = None

		# same sequence number -> likely no crash, assume length difference is due to difference in TSM
		# to get MLD and abs diff values for now - even though they might be meaningless due to
		@@ -351,6 +354,8 @@ def test_param_file_tests(
		allow_differing_lengths=allow_differing_lengths,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		ref_jbm_tf=ref_tracefile_dec,
		cut_jbm_tf=dut_tracefile_dec,
		)
		md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)