From c9802041a7601fc20c28a88fc5ce0ce8f702b9de Mon Sep 17 00:00:00 2001 From: Rishabh Tyagi Date: Mon, 10 Nov 2025 19:43:15 +1100 Subject: [PATCH] add BE conformance to runconformance script --- scripts/ivas_conformance/README.md | 46 +++++-- scripts/ivas_conformance/runConformance.py | 142 ++++++++++++++++----- 2 files changed, 146 insertions(+), 42 deletions(-) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 6355ecfcb6..0f617246b1 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -231,7 +231,13 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ENC ``` -- Analyse DUT IVAS Encoder Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse --be-test + ``` + +- Analyse NON-BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse @@ -243,10 +249,16 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=DEC ``` -- Analyse DUT IVAS Decoder Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT IVAS Decoder Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse --be-test + ``` + +- Analyse NON-BE conformance DUT IVAS Decoder Outputs Only (on Reference Platform) ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=DEC --analyse + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse ``` - Run DUT IVAS Renderer Tests Only (on Target Platform) @@ -255,10 +267,16 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=REND ``` -- Analyse DUT Renderer Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT Renderer Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse --be-test + ``` + +- Analyse NON-BE conformance DUT Renderer Outputs Only ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=REND --analyse + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse ``` - Run DUT ISAR Encoder Tests Only (on Target Platform) @@ -267,7 +285,13 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR_ENC ``` -- Analyse DUT ISAR Encoder Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse --be-test + ``` + +- Analyse NON-BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse @@ -279,8 +303,14 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR ``` -- Analyse DUT ISAR Decoder Outputs Only (on Reference Platform) +- Analyse BE conformance for DUT ISAR Decoder Outputs Only + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse --be-test + ``` + +- Analyse NON-BE conformance DUT ISAR Decoder Outputs Only ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR --analyse + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse ``` diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 92596403df..bb56bfacc6 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -46,6 +46,7 @@ import scipy.io.wavfile as wav import warnings import math import scipy.signal as sig +import filecmp sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) @@ -246,11 +247,13 @@ class MLDConformance: self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") self.CutBins = dict() self.mldcsv = dict() + self.BEcsv = dict() self.sampleStats = dict() for tag in IVAS_Bins.keys(): self.CutBins[tag] = os.path.join(self.cut_build_path, IVAS_Bins[tag]) self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") + self.BEcsv[tag] = os.path.join(self.outputDir, f"BE_{tag}.csv") self.sampleStats[tag] = os.path.join( self.outputDir, f"sampleStats_{tag}.csv" ) @@ -502,49 +505,63 @@ class MLDConformance: assert isinstance( testDesc, TestDesciptor ), f"Expected pcm test descriptor for {tag}" - ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### - if testDesc.rawCmdline.find("_cut.192.fer") == -1: - self.mld( - tag, - dutPytestTag, - refFile=testDesc.refOutput, - dutFile=testDesc.dutOutput, - ) + + if self.args.be_test: + DUTmdFiles = self.getMDfileList(outFile=testDesc.dutOutput) + REFmdFiles = self.getMDfileList(outFile=testDesc.refOutput) + self.beTest(tag, dutPytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput, DUTmdFileList=DUTmdFiles, REFmdFileList=REFmdFiles ) + else: + ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### + if testDesc.rawCmdline.find("_cut.192.fer") == -1: + self.mld( + tag, + dutPytestTag, + refFile=testDesc.refOutput, + dutFile=testDesc.dutOutput, + ) def analyseOneEncoderTest(self, tag: str, encPytestTag: str): testDesc = self.TestDesc[tag][encPytestTag] assert isinstance( testDesc, BitstrmTestDescriptor ), f"Expected bitstream test descriptor for {tag}" - refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") - dutDecOutputFile = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav") - # Decode the encoded output with Reference IVAS decoder - dutDecCmd = testDesc.refDecCmdline.split()[:-2] + [ - testDesc.dutOutput, - dutDecOutputFile, - ] - dutDecCmd = ["" if x == "-VOIP" else x for x in dutDecCmd] - dutDecCmd = " ".join(dutDecCmd) - self.process(command=dutDecCmd) - self.mld(tag, encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + if self.args.be_test: + self.beTest(tag, encPytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput ) + else: + refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") + + dutDecOutputFile = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav") + # Decode the encoded output with Reference IVAS decoder + dutDecCmd = testDesc.refDecCmdline.split()[:-2] + [ + testDesc.dutOutput, + dutDecOutputFile, + ] + dutDecCmd = ["" if x == "-VOIP" else x for x in dutDecCmd] + dutDecCmd = " ".join(dutDecCmd) + self.process(command=dutDecCmd) + self.mld(tag, encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) def analyseOneIsarEncoderTest(self, tag: str, pytestTag: str): testDesc = self.TestDesc[tag][pytestTag] assert isinstance( testDesc, BitstrmTestDescriptor ), f"Expected bitstream test descriptor for {tag}" - refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") - dutDecOutputFile = testDesc.dutOutput.replace(".splt.bit", ".wav") - # Decode the encoded output with Reference ISAR decoder - dutDecCmd = testDesc.refDecCmdline.split() - for idx, cmd in enumerate(dutDecCmd): - if cmd == "-o" and (idx + 1) < len(dutDecCmd): - dutDecCmd[idx + 1] = dutDecOutputFile - if cmd == "-i" and (idx + 1) < len(dutDecCmd): - dutDecCmd[idx + 1] = testDesc.dutOutput - self.process(command=" ".join(dutDecCmd)) - self.mld(tag, pytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + + if self.args.be_test: + self.beTest(tag, pytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput ) + else: + refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") + dutDecOutputFile = testDesc.dutOutput.replace(".splt.bit", ".wav") + # Decode the encoded output with Reference ISAR decoder + dutDecCmd = testDesc.refDecCmdline.split() + for idx, cmd in enumerate(dutDecCmd): + if cmd == "-o" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = dutDecOutputFile + if cmd == "-i" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = testDesc.dutOutput + self.process(command=" ".join(dutDecCmd)) + self.mld(tag, pytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) def getRendOutputFile(self, command: str): cmds = command.split() @@ -556,6 +573,14 @@ class MLDConformance: def getOutputFile(self, command: str): return command.split()[-1] + def getMDfileList(self, outFile: str): + MDfiles = [] + for i in range(0,3): + MDfiles.append(outFile + '.' + str(i) + '.csv') + MDfiles.append(outFile + '.met') + return MDfiles + + def setCommandExec(self, tag: str, command, ref: bool = False): exec = self.RefBins[tag] if ref else self.CutBins[tag] commands = command.split() @@ -663,9 +688,13 @@ class MLDConformance: def analyseTag(self, tag: str): # reset MLD, Sample Stats - open(self.mldcsv[tag], "w").close() - with open(self.sampleStats[tag], "w") as f: - f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") + if self.args.be_test: + with open(self.BEcsv[tag], "w") as f: + f.write(f"PYTESTTAG, BE=0 NON-BE=1\n") + else: + open(self.mldcsv[tag], "w").close() + with open(self.sampleStats[tag], "w") as f: + f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") selectedTests = list() if self.filter: for pyTestsTag in self.TestDesc[tag].keys(): @@ -685,7 +714,10 @@ class MLDConformance: else: for pyTestsTag in selectedTests: self.analyseOneCommand(tag, pyTestsTag) - self.doAnalysis(selectTag=tag) + if self.args.be_test: + self.doBEanalysis(selectTag=tag) + else: + self.doAnalysis(selectTag=tag) def process(self, command) -> int: if self.args.verbose: @@ -795,6 +827,42 @@ class MLDConformance: f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n" ) + def beTest(self, tag, pytestTag, refFile, dutFile, DUTmdFileList=[], REFmdFileList=[]): + BE_flag = 0 + if not filecmp.cmp(refFile, dutFile): + BE_flag = 1 + with open(self.BEcsv[tag], "a") as f: + f.write( + f"{pytestTag}, {BE_flag}\n" + ) + + for i in range(0,len(DUTmdFileList)): + if os.path.exists(DUTmdFileList[i]): + BE_flag = 0 + if not filecmp.cmp(REFmdFileList[i], DUTmdFileList[i]): + BE_flag = 1 + with open(self.BEcsv[tag], "a") as f: + f.write( + f"{DUTmdFileList[i]}, {BE_flag}\n" + ) + + + def doBEanalysis(self, selectTag="all"): + keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] + for tag in keys: + if os.path.exists(self.BEcsv[tag]): + BEresult = np.loadtxt( + self.BEcsv[tag], + delimiter=",", + dtype=int, + skiprows=1, + usecols=1, + ) + if np.sum(BEresult) > 0: + print(f"<{tag}> FAILED BE TEST, check {self.BEcsv[tag]}") + else: + print(f"<{tag}> PASSED BE TEST") + def doAnalysis(self, selectTag="all"): keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: @@ -901,6 +969,12 @@ if __name__ == "__main__": default="ALL", help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ISAR_ENC", "ALL"]', ) + parser.add_argument( + "--be-test", + default=False, + action="store_true", + help='runs only BE tests', + ) parser.add_argument( "--no-multi-processing", default=False, -- GitLab