From 30254a5db536e360329db4b833d048e090cd7eec Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Fri, 17 Oct 2025 22:17:56 +1100 Subject: [PATCH 1/7] Initial implementation of the conformance script for frame based MLD Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/runConformance.py | 558 +++++++++++++++++++++ 1 file changed, 558 insertions(+) create mode 100644 scripts/ivas_conformance/runConformance.py diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py new file mode 100644 index 0000000000..9df23416c8 --- /dev/null +++ b/scripts/ivas_conformance/runConformance.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python3 + +""" +(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" +import argparse +import os +import platform +import re +import numpy as np +import subprocess +import tempfile +import sys +from typing import Optional +from multiprocessing import Process, Value + +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) + +from pyaudio3dtools.audiofile import readfile, writefile +from pyaudio3dtools.audioarray import resample + + +class MLDConformance: + IVAS_Bins = { + "ENC": "IVAS_cod", + "DEC": "IVAS_dec", + "REND": "IVAS_rend", + "ISAR": "ISAR_post_rend", + } + + def __init__(self, args) -> None: + self.RefBins = dict() + self.CutBins = dict() + self.Commands = dict() + self.multiprocessing = not args.no_multi_processing + self.dryrun = args.dryrun + self.verbose = args.verbose + self.executedTests = Value("i", 0) + self.failedTests = Value("i", 0) + self.testvecDir = args.testvecDir + self.ref_build_path = args.ref_build_path + self.cut_build_path = args.cut_build_path + self.filter = args.filter + self.EncoderToDecoderCmdMap = dict() + + self.scriptsDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + self.outputDir = os.path.join(self.scriptsDir, "CUT_OUTPUTS") + self.toolsdir = os.path.join(self.scriptsDir, "tools") + self.testvDir = os.path.join(self.testvecDir, "testv") + self.refDir = self.testvDir + self.mldbin = os.path.join(self.toolsdir, platform.system(), "mld") + + self.logFile = os.path.join(self.outputDir, "runlog.txt") + self.failedCmdsFile = os.path.join(self.outputDir, "failedCmds.txt") + open(self.logFile, "w").close() + open(self.failedCmdsFile, "w").close() + + self.mldcsv = dict() + self.sampleStats = dict() + for tag in MLDConformance.IVAS_Bins.keys(): + self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") + self.sampleStats[tag] = os.path.join(self.outputDir, f"sampleStats_{tag}.csv") + + self.setup() + + def createDirs(self): + os.makedirs(self.outputDir, exist_ok=True) + subdirs = ["enc", "dec", "renderer", "split_rendering"] + for odir in subdirs: + os.makedirs(os.path.join(self.outputDir, "ref", odir), exist_ok=True) + os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) + + def setup(self): + self.createDirs() + for tag in MLDConformance.IVAS_Bins.keys(): + self.RefBins[tag] = os.path.join( + self.ref_build_path, MLDConformance.IVAS_Bins[tag] + ) + self.CutBins[tag] = os.path.join( + self.cut_build_path, MLDConformance.IVAS_Bins[tag] + ) + self.Commands[tag] = list() + + def accumulateCommands(self): + for root, _, files in os.walk(self.testvecDir): + for file_name in files: + basename, ext = os.path.splitext(file_name) + if ( + ("Readme_IVAS_" in basename) + and ext == ".txt" + and not ("ISAR" in basename) + ): + print(f"Accumulating commands from {file_name}") + file = os.path.join(root, file_name) + self.parseCommandsFile(file) + self.mapEncoderToDecoderCommands() + print("No of tests :") + for key in self.Commands.keys(): + print(f" {key} : {len(self.Commands[key])}") + + def parseCommandsFile(self, filePath): + with open(filePath) as fp: + for line in fp.readlines(): + m = re.search(r"^\$(CUT_.+_BIN) ", line) + if m: + tag = m.group(1).split("_")[1] + if tag in self.Commands.keys(): + self.Commands[tag].append(line) + + def getPcmPytestTag(self, command: str) -> str: + decInput = ( + os.path.basename(command.split()[-2]) + .replace(".fer", "") + .replace("_cut", "") + ) + return decInput.split(".")[-2] + + def getEncPytestTag(self, command: str) -> str: + return os.path.basename(command.split()[-1]).split(".")[-2] + + def mapEncoderToDecoderCommands(self): + decoderPyTestTags = dict() + encoderPyTestTags = dict() + for idx, command in enumerate(self.Commands["DEC"]): + decoderPyTestTags[self.getPcmPytestTag(command)] = idx + for idx, command in enumerate(self.Commands["ENC"]): + encoderPyTestTags[self.getEncPytestTag(command)] = idx + + for encTag in encoderPyTestTags.keys(): + if encTag in decoderPyTestTags.keys(): + self.EncoderToDecoderCmdMap[encoderPyTestTags[encTag]] = ( + decoderPyTestTags[encTag] + ) + if self.verbose: + print( + f"{encTag} {encoderPyTestTags[encTag]} -> {decoderPyTestTags[encTag]}" + ) + print(f"{self.Commands['ENC'][encoderPyTestTags[encTag]]}") + print(f"{self.Commands['DEC'][decoderPyTestTags[encTag]]}") + else: + print(f"{encTag} not fount in decoder") + print( + f"Mapped decoder tests for {len(self.EncoderToDecoderCmdMap)} encoder tests out of {len(self.Commands['ENC'])} tests" + ) + assert len(self.EncoderToDecoderCmdMap) == len( + self.Commands["ENC"] + ), "Failed to Map Encoder Commands to Decoder Commands" + + def runOneEncoderTest(self, command: str): + encPytestTag = self.getEncPytestTag(command) + refCommand = self.reformatCommand(command=command, ref=True) + refEncOutput = self.getOutputFile(refCommand) + refCommand = self.setCommandExec(tag="ENC", command=refCommand, ref=True) + self.process(command=refCommand) + + # Run reference Encoder + encCommandIdx = self.Commands["ENC"].index(command) + command = self.reformatCommand(command=command, ref=False) + command = self.setCommandExec(tag="ENC", command=command, ref=False) + dutEncOutput = self.getOutputFile(command) + self.process(command=command) + assert ".192" in dutEncOutput, "Output file not identified" + + # Decode the encoded output with Reference decoder + dutDecOutputFile = dutEncOutput.replace(".192", "_CUT_REFDECODED.wav") + refDecOutputFile = dutEncOutput.replace(".192", "_REF_REFDECODED.wav") + decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] + command = self.reformatCommand( + command=self.Commands["DEC"][decCommandIdx], ref=False + ) + command = command.replace("-VOIP", "") + dutDecCmd = ( + [self.RefBins["DEC"]] + + command.split()[1:-2] + + [dutEncOutput, dutDecOutputFile] + ) + refDecCmd = ( + [self.RefBins["DEC"]] + + command.split()[1:-2] + + [refEncOutput, refDecOutputFile] + ) + self.process(command=" ".join(dutDecCmd)) + self.process(command=" ".join(refDecCmd)) + self.mld("ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + + def runOneDecoderTest(self, tag: str, command: str): + dutPytestTag = self.getPcmPytestTag(command) + refInputFile = command.split()[-2].replace( + "$REF_PATH/ref", f"{self.testvDir}/ref" + ) + refInputFile = refInputFile.replace("_cut.192.fer", ".192") + # refInputFile = refInputFile.replace(".fer.192", ".192").replace(".192.fer", ".192").replace("_cut.192.fer", ".192").replace("_cut.192", ".192") + refDecOutput = self.getOutputFile(command).replace( + "$CUT_PATH/dut", f"{self.testvDir}/ref" + ) + command = self.reformatCommand(command=command, ref=False) + # command = command.replace("-VOIP", "") + dutDecOutputFile = self.getOutputFile(command) + dutDecCmd = ( + [self.CutBins["DEC"]] + + command.split()[1:-2] + + [refInputFile, dutDecOutputFile] + ) + self.process(command=" ".join(dutDecCmd)) + self.mld("DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile) + + def getRendOutputFile(self, command: str): + cmds = command.split() + for idx, cmd in enumerate(cmds): + if cmd == "-o" and (idx + 1) < len(cmds): + return cmds[idx + 1] + assert False, "Outputname not found" + + def runOneRendererTest(self, tag: str, command: str): + refRendOutputFile = self.getRendOutputFile(command).replace( + "$CUT_PATH/renderer", f"{self.testvDir}/renderer" + ) + rendPytestTag = os.path.basename(refRendOutputFile).split(".")[-2] + command = self.reformatCommand(command=command, ref=False) + dutRendCmd = " ".join([self.CutBins["REND"]] + command.split()[1:]) + dutRendOutputFile = self.getRendOutputFile(dutRendCmd) + self.process(command=dutRendCmd) + self.mld("REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile) + + def getOutputFile(self, command: str): + return command.split()[-1] + + def setCommandExec(self, tag: str, command, ref: bool): + exec = self.RefBins[tag] if ref else self.CutBins[tag] + commands = command.split() + return " ".join([exec, *commands[1:]]) + + def reformatCommand(self, command: str, ref: bool) -> str: + command = command.replace("$TESTV_PATH", self.testvecDir) + command = command.replace( + "$REF_PATH/split_rendering", f"{self.testvecDir}/testv/split_rendering" + ) + + ################ HACKS ######################### + command = command.replace("_cut.192.fer", ".192") + command = command.replace("_cut.192", ".192") + command = command.replace(".fer.192", ".192") + command = command.replace(".192.fer", ".192") + ################################################## + + command = command.replace( + "$REF_PATH/ref/param_file/enc/", f"{self.outputDir}/ref/enc/" + ) + command = command.replace( + "$REF_PATH/ref/param_file/dec/", f"{self.outputDir}/ref/dec/" + ) + command = command.replace( + "$REF_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" + ) + + if ref: + command = command.replace( + "$CUT_PATH/dut/param_file/enc/", f"{self.outputDir}/ref/enc/" + ) + command = command.replace( + "$CUT_PATH/dut/param_file/dec/", f"{self.outputDir}/ref/dec/" + ) + command = command.replace( + "$CUT_PATH/renderer/cut/", f"{self.outputDir}/ref/renderer/" + ) + command = command.replace( + "$CUT_PATH/split_rendering/cut/", + f"{self.outputDir}/ref/split_rendering/", + ) + command = command.replace( + "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" + ) + command = command.replace( + "$CUT_PATH/dut/sba_bs/raw/", f"{self.outputDir}/ref/dec/" + ) + else: + command = command.replace( + "$CUT_PATH/dut/param_file/enc/", f"{self.outputDir}/dut/enc/" + ) + command = command.replace( + "$CUT_PATH/dut/param_file/dec/", f"{self.outputDir}/dut/dec/" + ) + command = command.replace( + "$CUT_PATH/renderer/cut/", f"{self.outputDir}/dut/renderer/" + ) + command = command.replace( + "$CUT_PATH/split_rendering/cut/", + f"{self.outputDir}/dut/split_rendering/", + ) + command = command.replace( + "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" + ) + command = command.replace( + "$CUT_PATH/dut/sba_bs/raw/", f"{self.outputDir}/dut/dec/" + ) + + return command + + def runOneCommand(self, tag: str, command: str, ref: bool): + if tag == "ENC": + self.runOneEncoderTest(command) + elif tag == "DEC": + self.runOneDecoderTest(tag, command) + elif tag == "REND": + self.runOneRendererTest(tag, command) + else: + assert False, f"Un-implemented Tag {tag}" + self.executedTests.value += 1 + self.stats() + + def runTag(self, tag: str, ref: bool = False): + self.executedTests.value = 0 + self.failedTests.value = 0 + # reset MLD, Sample Stats + open(self.mldcsv[tag], "w").close() + with open(self.sampleStats[tag], "w") as f: + f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") + + processes = list() # Multiprocess list + commands = list() + if self.filter: + for command in self.Commands[tag]: + if self.filter in command: + commands.append(command) + else: + commands = self.Commands[tag] + + self.totalTests = len(commands) + print( + f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + ) + if self.multiprocessing: + for command in commands: + p = Process( + target=self.runOneCommand, + args=(tag, command, ref), + ) + processes.append(p) + p.start() + for p in processes: + p.join() + else: + for command in commands: + self.runOneCommand(tag, command, ref) + + def process(self, command) -> int: + if self.verbose: + print(command) + with open(self.logFile, "a") as fd: + fd.write(command + "\n") + + with open(self.logFile, "a") as fd: + if not self.dryrun: + c = subprocess.run( + command, stdout=fd, stderr=subprocess.STDOUT, text=True, shell=True + ) + if c.returncode: + with open(self.failedCmdsFile, "a") as f: + f.write(command + "\n") + self.failedTests.value += 1 + # c.check_returncode() + return 0 + + def stats(self): + print( + f"Executed: {self.executedTests.value} / {self.totalTests} Failed: {self.failedTests.value}", + end="\r", + ) + + def getSampleStats(self, refSamples: np.ndarray, dutSamples: np.ndarray): + nSamples = min(refSamples.shape[0], dutSamples.shape[0]) + diff = (refSamples[:nSamples] / 32768.0) - (dutSamples[:nSamples] / 32768.0) + beSamplesPercent = ( + 100.0 * np.sum(diff == 0) / refSamples.shape[0] / refSamples.shape[1] + ) + maxDiff = np.abs(diff).max() + rmsdB = int(10.0 * np.log10(np.average(diff**2) + np.finfo(np.float64).eps) * 10) / 10.0 + return (maxDiff, rmsdB, beSamplesPercent) + + + def mld(self, tag, pytestTag, refFile, dutFile): + mldThisFile = np.zeros(0) + with tempfile.TemporaryDirectory() as tmpdir: + refSamples, fsR = readfile(refFile, outdtype="float") + dutSamples, fsD = readfile(dutFile, outdtype="float") + assert ( + refSamples.shape[1] == dutSamples.shape[1] + ), "No of channels mismatch if ref vs cut" + maxDiff, rmsdB, beSamplesPercent = self.getSampleStats(refSamples, dutSamples) + + nChans = refSamples.shape[1] + + if fsR != 48000: + refSamples = np.clip(resample(refSamples, fsR, 48000), -32768, 32767) + if fsD != 48000: + dutSamples = np.clip(resample(dutSamples, fsD, 48000), -32768, 32767) + + for ch in range(nChans): + mldFile = os.path.join( + tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD.csv" + ) + refFileMono = os.path.join( + tmpdir, os.path.basename(refFile).replace(".wav", f"_ch{ch}.wav") + ) + dutFileMono = os.path.join( + tmpdir, os.path.basename(dutFile).replace(".wav", f"_ch{ch}.wav") + ) + writefile(refFileMono, refSamples[:, ch], 48000) + writefile(dutFileMono, dutSamples[:, ch], 48000) + command = [ + self.mldbin, + "-f", + "20", + "-o", + mldFile, + "-s", + refFileMono, + dutFileMono, + ] + self.process(" ".join(command)) + mldThisChan = np.loadtxt(mldFile, delimiter=" ", dtype=float) + if ch == 0: + mldThisFile = mldThisChan + else: + mldThisFile = np.maximum(mldThisFile, mldThisChan) + + if mldThisFile.size > 0: + with open(self.mldcsv[tag], "ab") as f: + np.savetxt(f, mldThisFile, delimiter=",") + with open(self.sampleStats[tag], "a") as f: + f.write(f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n") + + + def doAnalysis(self, selectTag = "all"): + keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [ selectTag ] + for tag in keys: + if os.path.exists(self.mldcsv[tag]): + mdlValues = np.loadtxt(self.mldcsv[tag], delimiter=" ", dtype=float) + N = mdlValues.shape[0] + if N == 0: + continue + m0 = np.sum(mdlValues == 0) + m1 = np.sum(mdlValues <= 1.0) + m2 = np.sum(mdlValues <= 2.0) + m5 = np.sum(mdlValues <= 5.0) + + PCNT = lambda num: int(1000 * num / N) / 10.0 + print(f"\n##########################################################") + print(f"<{tag}> Total Frames: {N}") + print(f"<{tag}> MAX MLD across all frames : {mdlValues.max()}") + print(f"<{tag}> Frames with MLD == 0 : {m0} frames ({PCNT(m0)}%)") + print(f"<{tag}> Frames with MLD <= 1 : {m1} frames ({PCNT(m1)}%)") + print(f"<{tag}> Frames with MLD <= 2 : {m2} frames ({PCNT(m2)}%)") + print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)") + print("##########################################################\n") + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Compare .wav files in two folders using mld per frame" + ) + + parser.add_argument( + "--testvecDir", + type=str, + required=True, + help="Base folder containing a set of conformance scripts named Readme_IVAS_xxxx.txt", + ) + parser.add_argument( + "--ref_build_path", + type=str, + required=True, + help="Path to the reference build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", + ) + parser.add_argument( + "--cut_build_path", + type=str, + required=True, + help="Path to the CUT build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", + ) + parser.add_argument( + "--verbose", + default=False, + action="store_true", + help="Enable verbose printing", + ) + parser.add_argument( + "--dryrun", + default=False, + action="store_true", + help="Do not run any processing, just dump command lines to runlog", + ) + + parser.add_argument( + "--filter", + type=str, + default=None, + help="Filter test based on text provided", + ) + parser.add_argument( + "--test-mode", + type=str, + default="ALL", + help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ALL"]', + ) + parser.add_argument( + "--no-multi-processing", + default=False, + action="store_true", + help="Disable multi-processing for sequential test run (debugging)", + ) + parser.add_argument( + "--analyse-only", + default=False, + action="store_true", + help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", + ) + + args = parser.parse_args() + + conformance = MLDConformance(args) + + conformance.accumulateCommands() + # import sys + # sys.exit(0) + testTags = MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] + for tag in testTags: + if tag == "ISAR": + # Not implemented yet + continue + if not args.analyse_only: + conformance.runTag(tag, ref=True) + conformance.doAnalysis(selectTag=tag) -- GitLab From 139ee0c774669b706af654556133fa7a5c8b6396 Mon Sep 17 00:00:00 2001 From: Ripinder Singh Date: Mon, 27 Oct 2025 16:19:17 +1100 Subject: [PATCH 2/7] Add option to regenerate encoder references * Format with black Signed-off-by: Ripinder Singh --- scripts/ivas_conformance/runConformance.py | 61 ++++++++++++++++------ 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 9df23416c8..7e03f3239d 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -59,6 +59,7 @@ class MLDConformance: self.CutBins = dict() self.Commands = dict() self.multiprocessing = not args.no_multi_processing + self.regenEncRefs = args.regenerate_enc_refs self.dryrun = args.dryrun self.verbose = args.verbose self.executedTests = Value("i", 0) @@ -85,7 +86,9 @@ class MLDConformance: self.sampleStats = dict() for tag in MLDConformance.IVAS_Bins.keys(): self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") - self.sampleStats[tag] = os.path.join(self.outputDir, f"sampleStats_{tag}.csv") + self.sampleStats[tag] = os.path.join( + self.outputDir, f"sampleStats_{tag}.csv" + ) self.setup() @@ -173,11 +176,21 @@ class MLDConformance: ), "Failed to Map Encoder Commands to Decoder Commands" def runOneEncoderTest(self, command: str): + encPytestTag = self.getEncPytestTag(command) - refCommand = self.reformatCommand(command=command, ref=True) - refEncOutput = self.getOutputFile(refCommand) - refCommand = self.setCommandExec(tag="ENC", command=refCommand, ref=True) - self.process(command=refCommand) + + if self.regenEncRefs: + refCommand = self.reformatCommand(command=command, ref=True) + refEncOutput = self.getOutputFile(refCommand) + self.process( + command=self.setCommandExec(tag="ENC", command=refCommand, ref=True) + ) + else: + refEncOutput = self.getOutputFile(command) + refEncOutput = refEncOutput.replace( + "$CUT_PATH/dut/param_file/enc/", + f"{self.testvecDir}/testv/ref/param_file/enc/", + ) # Run reference Encoder encCommandIdx = self.Commands["ENC"].index(command) @@ -207,7 +220,9 @@ class MLDConformance: ) self.process(command=" ".join(dutDecCmd)) self.process(command=" ".join(refDecCmd)) - self.mld("ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + self.mld( + "ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile + ) def runOneDecoderTest(self, tag: str, command: str): dutPytestTag = self.getPcmPytestTag(command) @@ -246,7 +261,9 @@ class MLDConformance: dutRendCmd = " ".join([self.CutBins["REND"]] + command.split()[1:]) dutRendOutputFile = self.getRendOutputFile(dutRendCmd) self.process(command=dutRendCmd) - self.mld("REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile) + self.mld( + "REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile + ) def getOutputFile(self, command: str): return command.split()[-1] @@ -400,10 +417,12 @@ class MLDConformance: 100.0 * np.sum(diff == 0) / refSamples.shape[0] / refSamples.shape[1] ) maxDiff = np.abs(diff).max() - rmsdB = int(10.0 * np.log10(np.average(diff**2) + np.finfo(np.float64).eps) * 10) / 10.0 + rmsdB = ( + int(10.0 * np.log10(np.average(diff**2) + np.finfo(np.float64).eps) * 10) + / 10.0 + ) return (maxDiff, rmsdB, beSamplesPercent) - def mld(self, tag, pytestTag, refFile, dutFile): mldThisFile = np.zeros(0) with tempfile.TemporaryDirectory() as tmpdir: @@ -412,7 +431,9 @@ class MLDConformance: assert ( refSamples.shape[1] == dutSamples.shape[1] ), "No of channels mismatch if ref vs cut" - maxDiff, rmsdB, beSamplesPercent = self.getSampleStats(refSamples, dutSamples) + maxDiff, rmsdB, beSamplesPercent = self.getSampleStats( + refSamples, dutSamples + ) nChans = refSamples.shape[1] @@ -454,11 +475,12 @@ class MLDConformance: with open(self.mldcsv[tag], "ab") as f: np.savetxt(f, mldThisFile, delimiter=",") with open(self.sampleStats[tag], "a") as f: - f.write(f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n") - + f.write( + f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n" + ) - def doAnalysis(self, selectTag = "all"): - keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [ selectTag ] + def doAnalysis(self, selectTag="all"): + keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: if os.path.exists(self.mldcsv[tag]): mdlValues = np.loadtxt(self.mldcsv[tag], delimiter=" ", dtype=float) @@ -480,6 +502,7 @@ class MLDConformance: print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)") print("##########################################################\n") + if __name__ == "__main__": parser = argparse.ArgumentParser( description="Compare .wav files in two folders using mld per frame" @@ -503,6 +526,12 @@ if __name__ == "__main__": required=True, help="Path to the CUT build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) + parser.add_argument( + "--regenerate_enc_refs", + default=False, + action="store_true", + help="Enable verbose printing", + ) parser.add_argument( "--verbose", default=False, @@ -548,7 +577,9 @@ if __name__ == "__main__": conformance.accumulateCommands() # import sys # sys.exit(0) - testTags = MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] + testTags = ( + MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] + ) for tag in testTags: if tag == "ISAR": # Not implemented yet -- GitLab From dc04bedd7edded390fe16318c4def3c51fa90301 Mon Sep 17 00:00:00 2001 From: rtyag Date: Tue, 28 Oct 2025 17:23:16 +1100 Subject: [PATCH 3/7] minor fix to delete cut output and recreate with every run, remove -f option with mld --- scripts/ivas_conformance/runConformance.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 7e03f3239d..18b8f02ff8 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -39,6 +39,7 @@ import tempfile import sys from typing import Optional from multiprocessing import Process, Value +import shutil sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) @@ -72,6 +73,9 @@ class MLDConformance: self.scriptsDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) self.outputDir = os.path.join(self.scriptsDir, "CUT_OUTPUTS") + if os.path.exists(self.outputDir): + shutil.rmtree(self.outputDir, ignore_errors=False) + os.mkdir(self.outputDir) self.toolsdir = os.path.join(self.scriptsDir, "tools") self.testvDir = os.path.join(self.testvecDir, "testv") self.refDir = self.testvDir @@ -456,8 +460,6 @@ class MLDConformance: writefile(dutFileMono, dutSamples[:, ch], 48000) command = [ self.mldbin, - "-f", - "20", "-o", mldFile, "-s", -- GitLab From 545adc3013a5938eb3a6ef3200cc1c9939a68576 Mon Sep 17 00:00:00 2001 From: rtyag Date: Tue, 28 Oct 2025 21:35:28 +1100 Subject: [PATCH 4/7] fix no multi processing arg --- scripts/ivas_conformance/runConformance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 18b8f02ff8..6c29cd0094 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -560,7 +560,7 @@ if __name__ == "__main__": help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ALL"]', ) parser.add_argument( - "--no-multi-processing", + "--no_multi_processing", default=False, action="store_true", help="Disable multi-processing for sequential test run (debugging)", -- GitLab From 5d9d0ec21f2e7e6b5c5ae924a644dccd4112044d Mon Sep 17 00:00:00 2001 From: rtyag Date: Fri, 31 Oct 2025 20:47:41 +1100 Subject: [PATCH 5/7] update the script to work with latest BE conformance changes --- scripts/ivas_conformance/runConformance.py | 31 +++++++++++++--------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 6c29cd0094..7841b4c7b9 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -192,9 +192,13 @@ class MLDConformance: else: refEncOutput = self.getOutputFile(command) refEncOutput = refEncOutput.replace( - "$CUT_PATH/dut/param_file/enc/", + "$CUT_PATH/ref/param_file/enc/", f"{self.testvecDir}/testv/ref/param_file/enc/", ) + refEncOutput = refEncOutput.replace( + "$CUT_PATH/ref/sba_bs/pkt/", + f"{self.testvecDir}/testv/ref/sba_bs/pkt/", + ) # Run reference Encoder encCommandIdx = self.Commands["ENC"].index(command) @@ -233,10 +237,10 @@ class MLDConformance: refInputFile = command.split()[-2].replace( "$REF_PATH/ref", f"{self.testvDir}/ref" ) - refInputFile = refInputFile.replace("_cut.192.fer", ".192") + #refInputFile = refInputFile.replace("_cut.192.fer", ".192") # refInputFile = refInputFile.replace(".fer.192", ".192").replace(".192.fer", ".192").replace("_cut.192.fer", ".192").replace("_cut.192", ".192") refDecOutput = self.getOutputFile(command).replace( - "$CUT_PATH/dut", f"{self.testvDir}/ref" + "$CUT_PATH/ref", f"{self.testvDir}/ref" ) command = self.reformatCommand(command=command, ref=False) # command = command.replace("-VOIP", "") @@ -247,7 +251,10 @@ class MLDConformance: + [refInputFile, dutDecOutputFile] ) self.process(command=" ".join(dutDecCmd)) - self.mld("DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile) + + ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### + if refInputFile.find("_cut.192.fer") == -1: + self.mld("DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile) def getRendOutputFile(self, command: str): cmds = command.split() @@ -302,10 +309,10 @@ class MLDConformance: if ref: command = command.replace( - "$CUT_PATH/dut/param_file/enc/", f"{self.outputDir}/ref/enc/" + "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/ref/enc/" ) command = command.replace( - "$CUT_PATH/dut/param_file/dec/", f"{self.outputDir}/ref/dec/" + "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/ref/dec/" ) command = command.replace( "$CUT_PATH/renderer/cut/", f"{self.outputDir}/ref/renderer/" @@ -315,17 +322,17 @@ class MLDConformance: f"{self.outputDir}/ref/split_rendering/", ) command = command.replace( - "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" + "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/ref/enc/" ) command = command.replace( - "$CUT_PATH/dut/sba_bs/raw/", f"{self.outputDir}/ref/dec/" + "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/ref/dec/" ) else: command = command.replace( - "$CUT_PATH/dut/param_file/enc/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/dut/enc/" ) command = command.replace( - "$CUT_PATH/dut/param_file/dec/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dut/dec/" ) command = command.replace( "$CUT_PATH/renderer/cut/", f"{self.outputDir}/dut/renderer/" @@ -335,10 +342,10 @@ class MLDConformance: f"{self.outputDir}/dut/split_rendering/", ) command = command.replace( - "$CUT_PATH/dut/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" ) command = command.replace( - "$CUT_PATH/dut/sba_bs/raw/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/dut/dec/" ) return command -- GitLab From b4e4cb983a9ebc2d5906095faeda564d5487b079 Mon Sep 17 00:00:00 2001 From: rtyag Date: Fri, 31 Oct 2025 21:00:10 +1100 Subject: [PATCH 6/7] adding a shell script to generate BE conformance package --- ivas_be_conf_test_gen.sh | 43 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 ivas_be_conf_test_gen.sh diff --git a/ivas_be_conf_test_gen.sh b/ivas_be_conf_test_gen.sh new file mode 100644 index 0000000000..f5d4c524f6 --- /dev/null +++ b/ivas_be_conf_test_gen.sh @@ -0,0 +1,43 @@ +make -f Makefile clean +make -f Makefile -j CLANG=0 +cp IVAS_cod IVAS_cod_ref +cp IVAS_dec IVAS_dec_ref +cp IVAS_rend IVAS_rend_ref +cp ISAR_post_rend ISAR_post_rend_ref +python3 scripts/prepare_combined_format_inputs.py +python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html +python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt +rm -rf testvec +mkdir testvec +mkdir testvec/binauralRenderer_interface +mkdir testvec/testv +mkdir testvec/testv/renderer +mkdir testvec/testv/split_rendering +mkdir testvec/bin +cp scripts/testv/* testvec/testv +cp -r scripts/ls_layouts testvec +cp -r scripts/object_edit testvec +cp -r scripts/switchPaths testvec +cp -r scripts/trajectories testvec +cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface +cp -r tests/ref testvec/testv/ref +cp -r tests/dut/* testvec/testv/ref +cp -r tests/renderer/cut testvec/testv/renderer/ref +cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref +cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs +cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + +python3 scripts/cleanup_26252.py + +cp -r tests/conformance-test testvec/ +cp Readme_IVAS_dec.txt testvec +cp Readme_IVAS_enc.txt testvec +cp Readme_IVAS_rend.txt testvec +cp Readme_IVAS_JBM_dec.txt testvec +cp Readme_IVAS_ISAR_dec.txt testvec +cp Readme_IVAS_ISAR_post_rend.txt testvec + +cp IVAS_cod testvec/bin +cp IVAS_dec testvec/bin +cp IVAS_rend testvec/bin +cp ISAR_post_rend testvec/bin -- GitLab From 6416a60c0057007b70a78769efae1d0f2a857fc9 Mon Sep 17 00:00:00 2001 From: rtyag Date: Fri, 31 Oct 2025 23:26:53 +1100 Subject: [PATCH 7/7] add CLANG O0 option to makefile, fix renderer tests partially --- Makefile | 3 +++ ivas_be_conf_test_gen.sh | 27 +++++++++++----------- scripts/ivas_conformance/runConformance.py | 8 +++---- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index c72aaacd8d..48c102c6ab 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,9 @@ LDLIBS += -lm # Clang sanitizer compiler options CCCLANG = clang +ifeq "$(CLANG)" "0" +CC = $(CCCLANG) +endif ifeq "$(CLANG)" "1" CC = $(CCCLANG) CFLAGS += -fsanitize=memory diff --git a/ivas_be_conf_test_gen.sh b/ivas_be_conf_test_gen.sh index f5d4c524f6..2536de1f89 100644 --- a/ivas_be_conf_test_gen.sh +++ b/ivas_be_conf_test_gen.sh @@ -5,27 +5,26 @@ cp IVAS_dec IVAS_dec_ref cp IVAS_rend IVAS_rend_ref cp ISAR_post_rend ISAR_post_rend_ref python3 scripts/prepare_combined_format_inputs.py -python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html +python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt rm -rf testvec mkdir testvec mkdir testvec/binauralRenderer_interface mkdir testvec/testv -mkdir testvec/testv/renderer +mkdir testvec/testv/renderer_short mkdir testvec/testv/split_rendering mkdir testvec/bin -cp scripts/testv/* testvec/testv -cp -r scripts/ls_layouts testvec -cp -r scripts/object_edit testvec -cp -r scripts/switchPaths testvec -cp -r scripts/trajectories testvec -cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface -cp -r tests/ref testvec/testv/ref -cp -r tests/dut/* testvec/testv/ref -cp -r tests/renderer/cut testvec/testv/renderer/ref -cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref -cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs -cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns +cp -r scripts/testv/* testvec/testv +cp -r scripts/ls_layouts testvec +cp -r scripts/object_edit testvec +cp -r scripts/switchPaths testvec +cp -r scripts/trajectories testvec +cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface +cp -r tests/ref testvec/testv/ref +cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref +cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref +cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs +cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns python3 scripts/cleanup_26252.py diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 7841b4c7b9..00252b52a8 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -98,7 +98,7 @@ class MLDConformance: def createDirs(self): os.makedirs(self.outputDir, exist_ok=True) - subdirs = ["enc", "dec", "renderer", "split_rendering"] + subdirs = ["enc", "dec", "renderer_short", "split_rendering"] for odir in subdirs: os.makedirs(os.path.join(self.outputDir, "ref", odir), exist_ok=True) os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) @@ -265,7 +265,7 @@ class MLDConformance: def runOneRendererTest(self, tag: str, command: str): refRendOutputFile = self.getRendOutputFile(command).replace( - "$CUT_PATH/renderer", f"{self.testvDir}/renderer" + "$CUT_PATH/renderer_short", f"{self.testvDir}/renderer_short" ) rendPytestTag = os.path.basename(refRendOutputFile).split(".")[-2] command = self.reformatCommand(command=command, ref=False) @@ -315,7 +315,7 @@ class MLDConformance: "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/ref/dec/" ) command = command.replace( - "$CUT_PATH/renderer/cut/", f"{self.outputDir}/ref/renderer/" + "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/ref/renderer_short/" ) command = command.replace( "$CUT_PATH/split_rendering/cut/", @@ -335,7 +335,7 @@ class MLDConformance: "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dut/dec/" ) command = command.replace( - "$CUT_PATH/renderer/cut/", f"{self.outputDir}/dut/renderer/" + "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/dut/renderer_short/" ) command = command.replace( "$CUT_PATH/split_rendering/cut/", -- GitLab