Add option to regenerate encoder references (139ee0c7) · Commits · IVAS Codec Public Collaboration / IVAS Codec

scripts/ivas_conformance/runConformance.py

+46 −15

Original line number	Diff line number	Diff line
		@@ -59,6 +59,7 @@ class MLDConformance:
		self.CutBins = dict()
		self.Commands = dict()
		self.multiprocessing = not args.no_multi_processing
		self.regenEncRefs = args.regenerate_enc_refs
		self.dryrun = args.dryrun
		self.verbose = args.verbose
		self.executedTests = Value("i", 0)
		@@ -85,7 +86,9 @@ class MLDConformance:
		self.sampleStats = dict()
		for tag in MLDConformance.IVAS_Bins.keys():
		self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv")
		self.sampleStats[tag] = os.path.join(self.outputDir, f"sampleStats_{tag}.csv")
		self.sampleStats[tag] = os.path.join(
		self.outputDir, f"sampleStats_{tag}.csv"
		)

		self.setup()

		@@ -173,11 +176,21 @@ class MLDConformance:
		), "Failed to Map Encoder Commands to Decoder Commands"

		def runOneEncoderTest(self, command: str):

		encPytestTag = self.getEncPytestTag(command)

		if self.regenEncRefs:
		refCommand = self.reformatCommand(command=command, ref=True)
		refEncOutput = self.getOutputFile(refCommand)
		refCommand = self.setCommandExec(tag="ENC", command=refCommand, ref=True)
		self.process(command=refCommand)
		self.process(
		command=self.setCommandExec(tag="ENC", command=refCommand, ref=True)
		)
		else:
		refEncOutput = self.getOutputFile(command)
		refEncOutput = refEncOutput.replace(
		"$CUT_PATH/dut/param_file/enc/",
		f"{self.testvecDir}/testv/ref/param_file/enc/",
		)

		# Run reference Encoder
		encCommandIdx = self.Commands["ENC"].index(command)
		@@ -207,7 +220,9 @@ class MLDConformance:
		)
		self.process(command=" ".join(dutDecCmd))
		self.process(command=" ".join(refDecCmd))
		self.mld("ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile)
		self.mld(
		"ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile
		)

		def runOneDecoderTest(self, tag: str, command: str):
		dutPytestTag = self.getPcmPytestTag(command)
		@@ -246,7 +261,9 @@ class MLDConformance:
		dutRendCmd = " ".join([self.CutBins["REND"]] + command.split()[1:])
		dutRendOutputFile = self.getRendOutputFile(dutRendCmd)
		self.process(command=dutRendCmd)
		self.mld("REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile)
		self.mld(
		"REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile
		)

		def getOutputFile(self, command: str):
		return command.split()[-1]
		@@ -400,10 +417,12 @@ class MLDConformance:
		100.0 * np.sum(diff == 0) / refSamples.shape[0] / refSamples.shape[1]
		)
		maxDiff = np.abs(diff).max()
		rmsdB = int(10.0 * np.log10(np.average(diff*2) + np.finfo(np.float64).eps) 10) / 10.0
		rmsdB = (
		int(10.0 * np.log10(np.average(diff*2) + np.finfo(np.float64).eps) 10)
		/ 10.0
		)
		return (maxDiff, rmsdB, beSamplesPercent)


		def mld(self, tag, pytestTag, refFile, dutFile):
		mldThisFile = np.zeros(0)
		with tempfile.TemporaryDirectory() as tmpdir:
		@@ -412,7 +431,9 @@ class MLDConformance:
		assert (
		refSamples.shape[1] == dutSamples.shape[1]
		), "No of channels mismatch if ref vs cut"
		maxDiff, rmsdB, beSamplesPercent = self.getSampleStats(refSamples, dutSamples)
		maxDiff, rmsdB, beSamplesPercent = self.getSampleStats(
		refSamples, dutSamples
		)

		nChans = refSamples.shape[1]

		@@ -454,8 +475,9 @@ class MLDConformance:
		with open(self.mldcsv[tag], "ab") as f:
		np.savetxt(f, mldThisFile, delimiter=",")
		with open(self.sampleStats[tag], "a") as f:
		f.write(f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n")

		f.write(
		f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n"
		)

		def doAnalysis(self, selectTag="all"):
		keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [selectTag]
		@@ -480,6 +502,7 @@ class MLDConformance:
		print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)")
		print("##########################################################\n")


		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		description="Compare .wav files in two folders using mld per frame"
		@@ -503,6 +526,12 @@ if __name__ == "__main__":
		required=True,
		help="Path to the CUT build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries",
		)
		parser.add_argument(
		"--regenerate_enc_refs",
		default=False,
		action="store_true",
		help="Enable verbose printing",
		)
		parser.add_argument(
		"--verbose",
		default=False,
		@@ -548,7 +577,9 @@ if __name__ == "__main__":
		conformance.accumulateCommands()
		# import sys
		# sys.exit(0)
		testTags = MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode]
		testTags = (
		MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode]
		)
		for tag in testTags:
		if tag == "ISAR":
		# Not implemented yet