update as per latest main (ba1d8b76) · Commits · IVAS Codec Public Collaboration / IVAS Codec

scripts/ivas_conformance/runConformance.py

+54 −27

Original line number	Diff line number	Diff line
		@@ -29,6 +29,7 @@ submitted to and settled by the final, binding jurisdiction of the courts of Mun
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		import argparse
		import os
		import platform
		@@ -474,9 +475,9 @@ class MLDConformance:
		def genEncoderReferences(self, tag: str, encPytestTag: str):
		# RUN ENCODER'S OUTPUT DECODED WITH REF DECODER
		testDesc = self.TestDesc[tag][encPytestTag]
		assert isinstance(
		testDesc, BitstreamTestDescriptor
		), f"Expected bitstream test descriptor for {tag}"
		assert isinstance(testDesc, BitstreamTestDescriptor), (
		f"Expected bitstream test descriptor for {tag}"
		)

		# Decode the encoded output with Reference IVAS decoder
		if tag == "ENC":
		@@ -510,9 +511,9 @@ class MLDConformance:

		def analyseWavOutputTest(self, tag: str, dutPytestTag: str):
		testDesc = self.TestDesc[tag][dutPytestTag]
		assert isinstance(
		testDesc, TestDescriptor
		), f"Expected pcm test descriptor for {tag}"
		assert isinstance(testDesc, TestDescriptor), (
		f"Expected pcm test descriptor for {tag}"
		)

		if self.args.be_test:
		DUTmdFiles = self.getMDfileList(outFile=testDesc.dutOutput)
		@@ -537,9 +538,9 @@ class MLDConformance:

		def analyseOneEncoderTest(self, tag: str, encPytestTag: str):
		testDesc = self.TestDesc[tag][encPytestTag]
		assert isinstance(
		testDesc, BitstreamTestDescriptor
		), f"Expected bitstream test descriptor for {tag}"
		assert isinstance(testDesc, BitstreamTestDescriptor), (
		f"Expected bitstream test descriptor for {tag}"
		)

		if self.args.be_test:
		self.beTest(
		@@ -566,9 +567,9 @@ class MLDConformance:

		def analyseOneIsarEncoderTest(self, tag: str, pytestTag: str):
		testDesc = self.TestDesc[tag][pytestTag]
		assert isinstance(
		testDesc, BitstreamTestDescriptor
		), f"Expected bitstream test descriptor for {tag}"
		assert isinstance(testDesc, BitstreamTestDescriptor), (
		f"Expected bitstream test descriptor for {tag}"
		)

		if self.args.be_test:
		self.beTest(
		@@ -787,9 +788,9 @@ class MLDConformance:
		with tempfile.TemporaryDirectory() as tmpdir:
		refSamples, fsR = readfile(refFile, outdtype="float")
		dutSamples, fsD = readfile(dutFile, outdtype="float")
		assert (
		refSamples.shape[1] == dutSamples.shape[1]
		), "No of channels mismatch if ref vs cut"
		assert refSamples.shape[1] == dutSamples.shape[1], (
		"No of channels mismatch if ref vs cut"
		)
		maxDiff, rmsdB, beSamplesPercent = self.getSampleStats(
		refSamples, dutSamples
		)
		@@ -865,13 +866,33 @@ class MLDConformance:
		with open(self.BEcsv[tag], "a") as f:
		f.write(f"{pytestTag}, {BE_flag}\n")

		for i in range(0, len(DUTmdFileList)):
		if os.path.exists(DUTmdFileList[i]):
		assert len(DUTmdFileList) == len(REFmdFileList)
		for refMDfile, dutMDfile in zip(REFmdFileList, DUTmdFileList):
		ref_exist = os.path.exists(refMDfile)
		dut_exist = os.path.exists(dutMDfile)

		assert ref_exist == dut_exist
		if not ref_exist:
		continue

		_, ref_suffix = os.path.splitext(refMDfile)
		_, dut_suffix = os.path.splitext(dutMDfile)
		assert ref_suffix == dut_suffix

		BE_flag = 0
		if not filecmp.cmp(REFmdFileList[i], DUTmdFileList[i]):
		BE_flag = 1
		if ref_suffix == ".csv":
		with open(refMDfile, "r") as f_ref:
		with open(dutMDfile, "r") as f_dut:
		ref_content = f_ref.read()
		dut_content = f_dut.read()
		BE_flag = int(not (ref_content == dut_content))
		elif ref_suffix == ".met":
		BE_flag = int(not filecmp.cmp(refMDfile, dutMDfile))
		else:
		assert False, f"MD file has unknown suffix {ref_suffix}"

		with open(self.BEcsv[tag], "a") as f:
		f.write(f"{DUTmdFileList[i]}, {BE_flag}\n")
		f.write(f"{dutMDfile}, {BE_flag}\n")

		def doBEanalysis(self, selectTag="all"):
		keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag]
		@@ -904,7 +925,9 @@ class MLDConformance:
		diff = dutMLD - refMLD

		if diff.max() > threshold:
		print(f"\033[91mMLD Corridor failed for {tag} with max MLD diff of {diff.max()} \033[00m")
		print(
		f"\033[91mMLD Corridor failed for {tag} with max MLD diff of {diff.max()} \033[00m"
		)
		maxMLD = np.maximum(dutMLD, refMLD)
		mldWithTags = np.column_stack(
		(
		@@ -923,7 +946,9 @@ class MLDConformance:
		with open(refMldFile2, "w") as f:
		np.savetxt(f, mldWithTags, fmt="%s", delimiter=",")
		else:
		print(f"\033[92mMLD Corridor passed for {tag} with max MLD diff of {diff.max()} \033[00m")
		print(
		f"\033[92mMLD Corridor passed for {tag} with max MLD diff of {diff.max()} \033[00m"
		)

		def doAnalysis(self, selectTag="all", corridor=False):
		keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag]
		@@ -987,7 +1012,9 @@ class MLDConformance:
		)
		self.computeCorridor(mldRefWithTags, mdlCutWithTags, tag)
		else:
		print(f"\033[91mMissing reference MLD file for {tag} : {refMldFile} \033[00m")
		print(
		f"\033[91mMissing reference MLD file for {tag} : {refMldFile} \033[00m"
		)


		if __name__ == "__main__":

tests/conformance-test/test_26252.py

+38 −5

Original line number	Diff line number	Diff line
		@@ -128,9 +128,42 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re
		subprocess.run([isar_post_renderer_path] + isar_post_rend_opts.split()[1:], check = True)

		diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path)
		result = True

		diff_in = {".wav": False, ".met": False, ".csv": False, ".192": False, ".bit": False}

		for cmd in diff_opts.split(';'):
		result = result and filecmp.cmp(cmd.split()[1], cmd.split()[2])
		if not result:
		assert False, "Output differs"
		file_a = Path(cmd.split()[1])
		file_b = Path(cmd.split()[2])

		suffix = file_a.suffix
		assert suffix == file_b.suffix
		assert suffix in diff_in

		# for .csv ISM metadata files, do text-based comparison to not take line endings into account
		# everything else (.wav output files and MASA metadata files) is compared as binary files
		if suffix == ".csv":
		with open(file_a, "r") as f_a:
		with open(file_b, "r") as f_b:
		a_content = f_a.read()
		b_content = f_b.read()

		files_equal = a_content == b_content
		else:
		files_equal = filecmp.cmp(file_a, file_b)

		if not files_equal:
		diff_in[suffix] = True

		if any(diff_in.values()):
		result_str = "Output differs in: "
		if diff_in[".csv"]:
		result_str += "object metadata "
		if diff_in[".met"]:
		result_str += "MASA metadata "
		if diff_in[".wav"]:
		result_str += "waveform"
		if diff_in[".192"] or diff_in[".bit"]:
		result_str += "bitstream"

		pytest.fail(result_str)