diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index e5fb0824ff9854e03a17ed52893398e94dc13e68..78c006fc2d8854bca9e4e89b40084eee606361bb 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -1,22 +1,286 @@ -# IVAS conformance scripts +# IVAS Conformance Scripts -This folder contains scripts for running IVAS conformance tests. This is a placeholder file for instructions. +This folder contains scripts for running IVAS conformance tests. -no-BE conformance USAGE +## Setup for Reference Platform -Following CMDs needs to be executed from ivas-codec root folder: + Reference platform is Ubuntu 24.04 -################generate testvec package and encoder refs (temporary step to obtain testvec package, this step will be removed in final delivery) ############ -sh ivas_be_conf_test_gen.sh -PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=ENC --regenerate-enc-refs +- Verify the Ubuntu Linux release is 24.04 + ```shell + lsb_release -d | grep Ubuntu + ``` + # It might be similar to Ubuntu 24.04.3 LTS -Encoder conformance: - - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=ENC -Decoder conformance: - - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=DEC -Renderer conformance: - - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/ref_bin --cut_build_path=testvec/cut_bin --test-mode=REND -Split renderer confomance: - - +- Install Clang 18 compiler + + ```shell + sudo apt install clang-18 + clang-18 --version + ``` + + Example version observed on Ubuntu 24.04.3 LTS + + ```text + Ubuntu clang version 18.1.3 (1ubuntu1) + Target: x86_64-pc-linux-gnu + Thread model: posix + InstalledDir: /usr/bin + ``` + + It might be required to set Clang-18 as the default clang on the machine + + ```shell + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 + ``` + +- Install Python3.13 on Ubuntu 24.04 LTS + + ```shell + sudo apt install python3.13 python3.13-venv + ``` + +- Create virtual environment for Python 3.13 and install requirements + + ```shell + python3.13 -m venv pyConformance + source pyConformance/bin/activate + cd ivas-codec + python -m pip install -r tests/requirements.txt + ``` + +## Reference Conformance Package Generation + +
+ Expand for detailed procedure + +To generate reference conformance package for distribution + +### Generate Reference Outputs and Readme.txt files + + ```shell + sh ivas_be_conf_test_gen.sh + ``` + +
+ Example output of reference test generation +

+::::::::::::::::::::::::
+------------------------------------------
+Generated html report: file:///home/dolby/git/ivas-codec/report_cmd.htm
+------------------------------------------
+=================================================
+2571 passed, 538 skipped, 230 xfailed in 377.10s (0:06:17)
+=================================================
+Identified 5430 files from scripts
+Removed 1515 files
+Kept 5422 files
+
+
+ +### Generate Reference Decoded Outputs for the Reference Encoded files + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --regenerate-enc-refs + ``` + +### Generate a conformance package zip + + ```shell + zip -r conformance.zip testvec scripts/ivas_conformance scripts/tools + ``` + +
+ +## Run CUT tests on Target platform + + To run CUT binaries on the targeted platform, it is necessary to replicate the initial setup for python and dependency packages. The CUT build of the IVAS binaries should be made available in a selected folder and needed for the next step + + To run IVAS DUT commands on the TARGET platform (may be different from ubuntu/linux) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR + ``` + +
+Example Output of CUT execution +

+Accumulating commands from Readme_IVAS_dec.txt
+Accumulating commands from Readme_IVAS_rend.txt
+Accumulating commands from Readme_IVAS_enc.txt
+Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
+Accumulating commands from Readme_IVAS_ISAR_dec.txt
+Accumulating commands from Readme_IVAS_JBM_dec.txt
+No of tests :
+    ENC : 381
+    DEC : 637
+    REND : 666
+    ISAR_ENC : 1032
+    ISAR : 1032
+Executing tests for ENC   (381 tests)
+Executing tests for DEC   (637 tests)
+Executing tests for REND   (666 tests)
+Executing tests for ISAR_ENC   (1032 tests)
+Executing tests for ISAR   (1032 tests)
+
+
+ + This should generate outputs in scripts/CUT_OUTPUTS folder which looks like below:- + + ```shell + CUT_OUTPUTS + +- runlog.txt : Dump of all the commands run and the outputs (mostly jumbled up due to multiprocessing) + +- failedCmds.txt : Log of all the shell commands that failed execution + +- dec/ : Folder containing all decoder tests CUT outputs + +- enc/ : Folder containing all encoder tests CUT outputs + +- renderer_short/ : Folder containing all renderer tests CUT outputs + +- split_rendering/ : Folder containing all split rendering enc/dec tests + ``` + +## Perform the MLD based analysis on the CUT outputs on reference platform (Ubuntu 24.04) + + If CUT test execution is done on a different platform, the scripts/CUT_OUTPUTS must be copied and provided in the reference platform's scripts/CUT_OUTPUTS. Then the following command is used to perform MLD based analysis on the same, encoded outputs will be implicitly decoded using reference decoder executables and MLD analysis performed on the reference decoded outputs. + + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse + ``` + +
+Example Output of CUT Analysis +

+Accumulating commands from Readme_IVAS_dec.txt
+Accumulating commands from Readme_IVAS_rend.txt
+Accumulating commands from Readme_IVAS_enc.txt
+Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
+Accumulating commands from Readme_IVAS_ISAR_dec.txt
+Accumulating commands from Readme_IVAS_JBM_dec.txt
+No of tests :
+    ENC : 381
+    DEC : 637
+    REND : 666
+    ISAR_ENC : 1032
+    ISAR : 1032
+Analysing tests for ENC   (381 tests)
+##########################################################
+<ENC> Total Frames: 2635800
+<ENC> MAX MLD across all frames : 0.0
+<ENC> Frames with MLD == 0 : 2635800 frames (100.0%)
+<ENC> Frames with MLD <= 1 : 2635800 frames (100.0%)
+<ENC> Frames with MLD <= 2 : 2635800 frames (100.0%)
+<ENC> Frames with MLD <= 5 : 2635800 frames (100.0%)
+<ENC> BE frames percentage = 100.0
+<ENC> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+Analysing tests for DEC   (637 tests)
+##########################################################
+<DEC> Total Frames: 4342140
+<DEC> MAX MLD across all frames : 0.0
+<DEC> Frames with MLD == 0 : 4342140 frames (100.0%)
+<DEC> Frames with MLD <= 1 : 4342140 frames (100.0%)
+<DEC> Frames with MLD <= 2 : 4342140 frames (100.0%)
+<DEC> Frames with MLD <= 5 : 4342140 frames (100.0%)
+<DEC> BE frames percentage = 100.0
+<DEC> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+Analysing tests for REND   (666 tests)
+##########################################################
+<REND> Total Frames: 4799952
+<REND> MAX MLD across all frames : 0.0
+<REND> Frames with MLD == 0 : 4799952 frames (100.0%)
+<REND> Frames with MLD <= 1 : 4799952 frames (100.0%)
+<REND> Frames with MLD <= 2 : 4799952 frames (100.0%)
+<REND> Frames with MLD <= 5 : 4799952 frames (100.0%)
+<REND> BE frames percentage = 100.0
+<REND> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+Analysing tests for ISAR_ENC   (1032 tests)
+##########################################################
+<ISAR_ENC> Total Frames: 2125956
+<ISAR_ENC> MAX MLD across all frames : 0.0
+<ISAR_ENC> Frames with MLD == 0 : 2125956 frames (100.0%)
+<ISAR_ENC> Frames with MLD <= 1 : 2125956 frames (100.0%)
+<ISAR_ENC> Frames with MLD <= 2 : 2125956 frames (100.0%)
+<ISAR_ENC> Frames with MLD <= 5 : 2125956 frames (100.0%)
+<ISAR_ENC> BE frames percentage = 100.0
+<ISAR_ENC> max absolute diff = 0.0, sample range (-32768,32767)
+##########################################################
+Analysing tests for ISAR   (1032 tests)
+##########################################################
+<ISAR> Total Frames: 2125956
+<ISAR> MAX MLD across all frames : 0.0
+<ISAR> Frames with MLD == 0 : 2125956 frames (100.0%)
+<ISAR> Frames with MLD <= 1 : 2125956 frames (100.0%)
+<ISAR> Frames with MLD <= 2 : 2125956 frames (100.0%)
+<ISAR> Frames with MLD <= 5 : 2125956 frames (100.0%)
+<ISAR> BE frames percentage = 100.0
+<ISAR> max absolute diff = 0.0, sample range (-32768, 32767)
+##########################################################
+
+
+ +## Executing specific tests only + +All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Renderer, ISAR Encoder and ISAR Decoder only. The commandline allows for ```-test-mode=``` for this functionality, examples : - + +- Run DUT IVAS Encoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ENC + ``` + +- Analyse DUT IVAS Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse + ``` + +- Run DUT IVAS Decoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=DEC + ``` + +- Analyse DUT IVAS Decoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=DEC --analyse + ``` + +- Run DUT IVAS Renderer Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=REND + ``` + +- Analyse DUT Renderer Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=REND --analyse + ``` + +- Run DUT ISAR Encoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR_ENC + ``` + +- Analyse DUT ISAR Encoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse + ``` + +- Run DUT ISAR Decoder Tests Only (on Target Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR + ``` + +- Analyse DUT ISAR Decoder Outputs Only (on Reference Platform) + + ```shell + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR --analyse + ``` diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 36f92b9d91e321f7f1c71b4386c6dedf187e2261..92596403dfe1325900be4d2f6a813966ffaa02f7 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -39,6 +39,8 @@ import tempfile import sys from typing import Tuple from multiprocessing import Pool +from dataclasses import dataclass +from typing import Union import shutil import scipy.io.wavfile as wav import warnings @@ -138,6 +140,7 @@ def writefile(filename: str, x: np.ndarray, fs: int = 48000) -> None: else: raise ValueError("Wrong input format. Use wav or pcm") + def resample(x: np.ndarray, in_freq: int, out_freq: int) -> np.ndarray: """Resample a multi-channel audio array @@ -183,32 +186,51 @@ def resample(x: np.ndarray, in_freq: int, out_freq: int) -> np.ndarray: return y -class MLDConformance: - IVAS_Bins = { - "ENC": "IVAS_cod", - "DEC": "IVAS_dec", - "REND": "IVAS_rend", - "ISAR_ENC": "IVAS_dec", - "ISAR": "ISAR_post_rend", - } +IVAS_Bins = { + "ENC": "IVAS_cod", + "DEC": "IVAS_dec", + "REND": "IVAS_rend", + "ISAR_ENC": "IVAS_dec", + "ISAR": "ISAR_post_rend", +} + + +@dataclass +class TestDesciptor: + dutOutput: str = "" + refOutput: str = "" + rawCmdline: str = "" + dutCmdline: str = "" + refCmdline: str = "" + + +@dataclass +class BitstrmTestDescriptor(TestDesciptor): + rawDecCmdline: str = "" + dutDecCmdline: str = "" + refDecCmdline: str = "" + +class MLDConformance: def setupCommon(self): self.Commands = dict() + self.TestDesc = dict[ + str, dict[str, Union[TestDesciptor, BitstrmTestDescriptor]] + ]() self.EncoderToDecoderCmdMap = dict() self.IsarEncoderToDecoderCmdMap = dict() - for tag in MLDConformance.IVAS_Bins.keys(): + for tag in IVAS_Bins.keys(): self.Commands[tag] = list() # CREATE OUTPUT DIRECTORY STRUCTURE : CLEAN PREV OUTPUT self.outputDir = os.path.join(self.scriptsDir, "CUT_OUTPUTS") - if os.path.exists(self.outputDir): - if not self.args.dec_for_dut_enc: - shutil.rmtree(self.outputDir, ignore_errors=False) + if self.args.clean_output_dir and os.path.exists(self.outputDir): + shutil.rmtree(self.outputDir, ignore_errors=False) os.makedirs(self.outputDir, exist_ok=True) subdirs = ["enc", "dec", "renderer_short", "split_rendering"] for odir in subdirs: - os.makedirs(os.path.join(self.testvDir, "ref", odir), exist_ok=True) - os.makedirs(os.path.join(self.outputDir, "dut", odir), exist_ok=True) + os.makedirs(os.path.join(self.testvDir, odir), exist_ok=True) + os.makedirs(os.path.join(self.outputDir, odir), exist_ok=True) self.logFile = os.path.join(self.outputDir, "runlog.txt") self.failedCmdsFile = os.path.join(self.outputDir, "failedCmds.txt") @@ -219,17 +241,15 @@ class MLDConformance: self.cut_build_path = args.cut_build_path self.filter = args.filter exe_platform = platform.system() - if exe_platform == 'Windows': - exe_platform = 'Win32' + if exe_platform == "Windows": + exe_platform = "Win32" self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") self.CutBins = dict() self.mldcsv = dict() self.sampleStats = dict() - for tag in MLDConformance.IVAS_Bins.keys(): - self.CutBins[tag] = os.path.join( - self.cut_build_path, MLDConformance.IVAS_Bins[tag] - ) + for tag in IVAS_Bins.keys(): + self.CutBins[tag] = os.path.join(self.cut_build_path, IVAS_Bins[tag]) self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv") self.sampleStats[tag] = os.path.join( self.outputDir, f"sampleStats_{tag}.csv" @@ -238,16 +258,13 @@ class MLDConformance: def setupRef(self): self.RefBins = dict() self.ref_build_path = self.args.ref_build_path - for tag in MLDConformance.IVAS_Bins.keys(): - self.RefBins[tag] = os.path.join( - self.ref_build_path, MLDConformance.IVAS_Bins[tag] - ) + for tag in IVAS_Bins.keys(): + self.RefBins[tag] = os.path.join(self.ref_build_path, IVAS_Bins[tag]) def setup(self): self.setupCommon() self.setupRef() - if not self.args.regenerate_enc_refs: - self.setupDUT() + self.setupDUT() def __init__(self, args) -> None: self.args = args @@ -263,23 +280,23 @@ class MLDConformance: basename, ext = os.path.splitext(file_name) if ("Readme_IVAS_" in basename) and ext == ".txt": print(f"Accumulating commands from {file_name}") + isISAREnc = "IVAS_ISAR_dec" in file_name file = os.path.join(root, file_name) - self.parseCommandsFile(file) - self.mapEncoderToDecoderCommands() - self.mapIsarEncToDecCommands() - print("No of tests :") - for key in self.Commands.keys(): - print(f" {key} : {len(self.Commands[key])}") - - def parseCommandsFile(self, filePath): - isISAREnc = "IVAS_ISAR_dec" in filePath - with open(filePath) as fp: - for line in fp.readlines(): - m = re.search(r"^\$(CUT_.+_BIN) ", line) - if m: - tag = m.group(1).split("_")[1] if not isISAREnc else "ISAR_ENC" - if tag in self.Commands.keys(): - self.Commands[tag].append(line) + with open(file) as fp: + for line in fp.readlines(): + m = re.search(r"^\$(CUT_.+_BIN) ", line) + if m: + tag = ( + m.group(1).split("_")[1] + if not isISAREnc + else "ISAR_ENC" + ) + self.Commands[tag].append(line) + self.TestDesc = self.createTestDescriptors() + + def getRendPyTestTag(self, command: str) -> str: + refRendOutputFile = self.getRendOutputFile(command) + return os.path.basename(refRendOutputFile).split(".")[-2] def getPcmPytestTag(self, command: str) -> str: decInput = ( @@ -303,178 +320,232 @@ class MLDConformance: def getIsarEncPytestTag(self, command: str) -> str: return os.path.basename(command.split()[-1]).split(".")[-3] - def mapIsarEncToDecCommands(self): - decoderPyTestTags = dict() - encoderPyTestTags = dict() - for idx, command in enumerate(self.Commands["ISAR"]): - decoderPyTestTags[self.getIsarDecPytestTag(command)] = idx - for idx, command in enumerate(self.Commands["ISAR_ENC"]): - encoderPyTestTags[self.getIsarEncPytestTag(command)] = idx - - for encTag in encoderPyTestTags.keys(): - if encTag in decoderPyTestTags.keys(): - self.IsarEncoderToDecoderCmdMap[encoderPyTestTags[encTag]] = ( - decoderPyTestTags[encTag] + def createTestDescriptors( + self, + ) -> dict[str, dict[str, Union[TestDesciptor, BitstrmTestDescriptor]]]: + testDesciptor = dict[ + str, dict[str, Union[TestDesciptor, BitstrmTestDescriptor]] + ]() + for tag in IVAS_Bins.keys(): + testDesciptor[tag] = dict[ + str, Union[TestDesciptor, BitstrmTestDescriptor] + ]() + + IvasDecCmdMap = dict() + IsarDecCmdMap = dict() + for command in self.Commands["REND"]: + pyTestTag = self.getRendPyTestTag(command) + refCmdline = self.setCommandExec( + tag="REND", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="REND", command=self.reformatCommand(command) + ) + refOutput = self.getRendOutputFile(refCmdline) + dutOutput = self.getRendOutputFile(dutCmdline) + testDesciptor["REND"][pyTestTag] = TestDesciptor( + rawCmdline=command, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + ) + for command in self.Commands["DEC"]: + pyTestTag = self.getPcmPytestTag(command) + refCmdline = self.setCommandExec( + tag="DEC", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="DEC", command=self.reformatCommand(command) + ) + refOutput = self.getOutputFile(refCmdline) + dutOutput = self.getOutputFile(dutCmdline) + testDesciptor["DEC"][pyTestTag] = TestDesciptor( + rawCmdline=command, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + ) + IvasDecCmdMap[pyTestTag] = command + for command in self.Commands["ISAR"]: + pyTestTag = self.getIsarDecPytestTag(command) + refCmdline = self.setCommandExec( + tag="ISAR", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="ISAR", command=self.reformatCommand(command) + ) + refOutput = self.getRendOutputFile(refCmdline) + dutOutput = self.getRendOutputFile(dutCmdline) + testDesciptor["ISAR"][pyTestTag] = TestDesciptor( + rawCmdline=command, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + ) + IsarDecCmdMap[pyTestTag] = command + for command in self.Commands["ENC"]: + pyTestTag = self.getEncPytestTag(command) + refCmdline = self.setCommandExec( + tag="ENC", command=self.reformatCommand(command, ref=True), ref=True + ) + dutCmdline = self.setCommandExec( + tag="ENC", command=self.reformatCommand(command) + ) + refOutput = self.getOutputFile(refCmdline) + dutOutput = self.getOutputFile(dutCmdline) + assert ".192" in dutOutput, "Output file not identified in dut" + assert ".192" in refOutput, "Output file not identified in ref" + if pyTestTag in IvasDecCmdMap.keys(): + rawDecCmdline = IvasDecCmdMap[pyTestTag] + refDecCmdline = self.setCommandExec( + tag="DEC", + command=self.reformatCommand(rawDecCmdline, ref=True), + ref=True, + ) + dutDecCmdline = self.setCommandExec( + tag="DEC", command=self.reformatCommand(rawDecCmdline) + ) + testDesciptor["ENC"][pyTestTag] = BitstrmTestDescriptor( + rawCmdline=command, + rawDecCmdline=rawDecCmdline, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + dutDecCmdline=dutDecCmdline, + refDecCmdline=refDecCmdline, ) - if self.args.verbose: - print( - f"{encTag} {encoderPyTestTags[encTag]} -> {decoderPyTestTags[encTag]}" - ) - print(f"{self.Commands['ISAR_ENC'][encoderPyTestTags[encTag]]}") - print(f"{self.Commands['ISAR'][decoderPyTestTags[encTag]]}") else: - print(f"{encTag} not fount in decoder") - print( - f"Mapped decoder tests for {len(self.IsarEncoderToDecoderCmdMap)} encoder tests out of {len(self.Commands['ISAR_ENC'])} tests" - ) - assert len(self.IsarEncoderToDecoderCmdMap) == len( - self.Commands["ISAR_ENC"] - ), "Failed to Map Encoder Commands to Decoder Commands" - - def mapEncoderToDecoderCommands(self): - decoderPyTestTags = dict() - encoderPyTestTags = dict() - for idx, command in enumerate(self.Commands["DEC"]): - decoderPyTestTags[self.getPcmPytestTag(command)] = idx - for idx, command in enumerate(self.Commands["ENC"]): - encoderPyTestTags[self.getEncPytestTag(command)] = idx - - for encTag in encoderPyTestTags.keys(): - if encTag in decoderPyTestTags.keys(): - self.EncoderToDecoderCmdMap[encoderPyTestTags[encTag]] = ( - decoderPyTestTags[encTag] + print(f"{pyTestTag} not found in decoder") + for command in self.Commands["ISAR_ENC"]: + pyTestTag = self.getIsarEncPytestTag(command) + refCmdline = self.setCommandExec( + tag="ISAR_ENC", + command=self.reformatCommand(command, ref=True), + ref=True, + ) + dutCmdline = self.setCommandExec( + tag="ISAR_ENC", command=self.reformatCommand(command) + ) + refOutput = self.getOutputFile(refCmdline) + dutOutput = self.getOutputFile(dutCmdline) + if pyTestTag in IsarDecCmdMap.keys(): + rawDecCmdline = IsarDecCmdMap[pyTestTag] + refDecCmdline = self.setCommandExec( + tag="ISAR", + command=self.reformatCommand(rawDecCmdline, ref=True), + ref=True, + ) + dutDecCmdline = self.setCommandExec( + tag="ISAR", command=self.reformatCommand(rawDecCmdline) + ) + testDesciptor["ISAR_ENC"][pyTestTag] = BitstrmTestDescriptor( + rawCmdline=command, + rawDecCmdline=rawDecCmdline, + refOutput=refOutput, + dutOutput=dutOutput, + dutCmdline=dutCmdline, + refCmdline=refCmdline, + dutDecCmdline=dutDecCmdline, + refDecCmdline=refDecCmdline, ) - if self.args.verbose: - print( - f"{encTag} {encoderPyTestTags[encTag]} -> {decoderPyTestTags[encTag]}" - ) - print(f"{self.Commands['ENC'][encoderPyTestTags[encTag]]}") - print(f"{self.Commands['DEC'][decoderPyTestTags[encTag]]}") else: - print(f"{encTag} not fount in decoder") - print( - f"Mapped decoder tests for {len(self.EncoderToDecoderCmdMap)} encoder tests out of {len(self.Commands['ENC'])} tests" - ) - assert len(self.EncoderToDecoderCmdMap) == len( - self.Commands["ENC"] - ), "Failed to Map Encoder Commands to Decoder Commands" - - def genEncoderReferences( - self, command: str, encCommandIdx: int, encTag: str = "ENC" - ): - # RUN ENCODER COMMAND LINE WITH REFERENCE ENCODER - refCommand = self.reformatCommand(command, ref=True) - refEncOutput = self.getOutputFile(refCommand) - # if not os.path.exists(refEncOutput): - self.process( - command=self.setCommandExec(tag=encTag, command=refCommand, ref=True) - ) + print(f"{pyTestTag} not found in ISAR decoder") + print("No of tests :") + for tag in testDesciptor.keys(): + print(f" {tag} : {len(testDesciptor[tag])}") - # FIND CORRESPONDING DECODER COMMAND - if encTag == "ISAR_ENC": - decTag = "ISAR" - decCommandIdx = self.IsarEncoderToDecoderCmdMap[encCommandIdx] - refDecOutputFile = refEncOutput.replace(".splt.bit", ".splt.REFDECODED.wav") - else: - decTag = "DEC" - decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] - refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") + return testDesciptor - command = self.reformatCommand( - command=self.Commands[decTag][decCommandIdx], ref=True - ) - - if "-VOIP_hf_only" not in command: - command = command.replace("-VOIP", "") - if encTag == "ISAR_ENC": - refDecCmd = [self.RefBins[decTag]] + command.split()[1:] - else: - refDecCmd = ( - [self.RefBins[decTag]] - + command.split()[1:-2] - + [refEncOutput, refDecOutputFile] - ) + def genEncoderReferences(self, tag: str, encPytestTag: str): + # RUN ENCODER'S OUTPUT DECODED WITH REF DECODER + testDesc = self.TestDesc[tag][encPytestTag] + assert isinstance( + testDesc, BitstrmTestDescriptor + ), f"Expected bitstream test descriptor for {tag}" - self.process(command=" ".join(refDecCmd)) + # Decode the encoded output with Reference IVAS decoder + if tag == "ENC": + refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") + refDecCmd = testDesc.refDecCmdline.split()[:-2] + [ + testDesc.refOutput, + refDecOutputFile, + ] + else: + refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") + refDecCmd = testDesc.refDecCmdline.split() + for idx, cmd in enumerate(refDecCmd): + if cmd == "-o" and (idx + 1) < len(refDecCmd): + refDecCmd[idx + 1] = refDecOutputFile + break + refDecCmd = ["" if x == "-VOIP" else x for x in refDecCmd] + refDecCmd = " ".join(refDecCmd) + self.process(command=refDecCmd) self.stats() def runReferenceGeneration(self, encTag="ENC"): - commands = conformance.Commands[encTag] - self.totalTests = len(commands) + selectedTests = list(self.TestDesc[encTag].keys()) + self.totalTests = len(selectedTests) if not self.args.no_multi_processing: with Pool() as pool: - args = [(command, commandIdx, encTag) for commandIdx, command in enumerate(commands)] + args = [(encTag, pyTestsTag) for pyTestsTag in selectedTests] pool.starmap(self.genEncoderReferences, args) else: - for commandIdx, command in enumerate(commands): - conformance.genEncoderReferences(command, commandIdx, encTag) - - def runOneEncoderTest(self, command: str): - encPytestTag = self.getEncPytestTag(command) - refEncOutput = self.getOutputFile(command) - refEncOutput = refEncOutput.replace( - "$CUT_PATH/ref/param_file/enc/", - f"{self.testvecDir}/testv/ref/param_file/enc/", - ) - refEncOutput = refEncOutput.replace( - "$CUT_PATH/ref/sba_bs/pkt/", - f"{self.testvecDir}/testv/ref/sba_bs/pkt/", - ) - refDecOutputFile = refEncOutput.replace(".192", "_REFDECODED.wav") - - # Run CUT Encoder - encCommandIdx = self.Commands["ENC"].index(command) - command = self.reformatCommand(command) - command = self.setCommandExec(tag="ENC", command=command) - dutEncOutput = self.getOutputFile(command) - - if not self.args.dec_for_dut_enc: - self.process(command=command) - - assert ".192" in dutEncOutput, "Output file not identified" - - # Decode the encoded output with Reference decoder - dutDecOutputFile = dutEncOutput.replace(".192", "_CUT_REFDECODED.wav") - decCommandIdx = self.EncoderToDecoderCmdMap[encCommandIdx] - command = self.reformatCommand(command=self.Commands["DEC"][decCommandIdx]) - if "-VOIP_hf_only" not in command: - command = command.replace("-VOIP", "") - dutDecCmd = ( - [self.RefBins["DEC"]] - + command.split()[1:-2] - + [dutEncOutput, dutDecOutputFile] - ) - self.process(command=" ".join(dutDecCmd)) - self.mld( - "ENC", encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile - ) - - def runOneDecoderTest(self, tag: str, command: str): - dutPytestTag = self.getPcmPytestTag(command) - refInputFile = command.split()[-2].replace( - "$REF_PATH/ref", f"{self.testvDir}/ref" - ) - # refInputFile = refInputFile.replace("_cut.192.fer", ".192") - # refInputFile = refInputFile.replace(".fer.192", ".192").replace(".192.fer", ".192").replace("_cut.192.fer", ".192").replace("_cut.192", ".192") - refDecOutput = self.getOutputFile(command).replace( - "$CUT_PATH/ref", f"{self.testvDir}/ref" - ) - command = self.reformatCommand(command) - # command = command.replace("-VOIP", "") - dutDecOutputFile = self.getOutputFile(command) - dutDecCmd = ( - [self.CutBins["DEC"]] - + command.split()[1:-2] - + [refInputFile, dutDecOutputFile] - ) - self.process(command=" ".join(dutDecCmd)) - + for pyTestsTag in selectedTests: + self.genEncoderReferences(encTag, pyTestsTag) + + def analyseWavOutputTest(self, tag: str, dutPytestTag: str): + testDesc = self.TestDesc[tag][dutPytestTag] + assert isinstance( + testDesc, TestDesciptor + ), f"Expected pcm test descriptor for {tag}" ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare ##### - if refInputFile.find("_cut.192.fer") == -1: + if testDesc.rawCmdline.find("_cut.192.fer") == -1: self.mld( - "DEC", dutPytestTag, refFile=refDecOutput, dutFile=dutDecOutputFile + tag, + dutPytestTag, + refFile=testDesc.refOutput, + dutFile=testDesc.dutOutput, ) + def analyseOneEncoderTest(self, tag: str, encPytestTag: str): + testDesc = self.TestDesc[tag][encPytestTag] + assert isinstance( + testDesc, BitstrmTestDescriptor + ), f"Expected bitstream test descriptor for {tag}" + refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav") + + dutDecOutputFile = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav") + # Decode the encoded output with Reference IVAS decoder + dutDecCmd = testDesc.refDecCmdline.split()[:-2] + [ + testDesc.dutOutput, + dutDecOutputFile, + ] + dutDecCmd = ["" if x == "-VOIP" else x for x in dutDecCmd] + dutDecCmd = " ".join(dutDecCmd) + self.process(command=dutDecCmd) + self.mld(tag, encPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + + def analyseOneIsarEncoderTest(self, tag: str, pytestTag: str): + testDesc = self.TestDesc[tag][pytestTag] + assert isinstance( + testDesc, BitstrmTestDescriptor + ), f"Expected bitstream test descriptor for {tag}" + refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav") + dutDecOutputFile = testDesc.dutOutput.replace(".splt.bit", ".wav") + # Decode the encoded output with Reference ISAR decoder + dutDecCmd = testDesc.refDecCmdline.split() + for idx, cmd in enumerate(dutDecCmd): + if cmd == "-o" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = dutDecOutputFile + if cmd == "-i" and (idx + 1) < len(dutDecCmd): + dutDecCmd[idx + 1] = testDesc.dutOutput + self.process(command=" ".join(dutDecCmd)) + self.mld(tag, pytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile) + def getRendOutputFile(self, command: str): cmds = command.split() for idx, cmd in enumerate(cmds): @@ -482,63 +553,6 @@ class MLDConformance: return cmds[idx + 1] assert False, "Outputname not found" - def runOneRendererTest(self, tag: str, command: str): - refRendOutputFile = self.getRendOutputFile(command).replace( - "$CUT_PATH/renderer_short", f"{self.testvDir}/renderer_short" - ) - rendPytestTag = os.path.basename(refRendOutputFile).split(".")[-2] - command = self.reformatCommand(command) - dutRendCmd = " ".join([self.CutBins["REND"]] + command.split()[1:]) - dutRendOutputFile = self.getRendOutputFile(dutRendCmd) - self.process(command=dutRendCmd) - self.mld( - "REND", rendPytestTag, refFile=refRendOutputFile, dutFile=dutRendOutputFile - ) - - def runOneIsarEncoderTest(self, command: str): - encCommandIdx = self.Commands["ISAR_ENC"].index(command) - decCommandIdx = self.IsarEncoderToDecoderCmdMap[encCommandIdx] - - isarEncPytestTag = self.getIsarEncPytestTag(command) - refEncCommand = self.reformatCommand(command, ref=True) - refEncOutput = self.getOutputFile(refEncCommand) - refDecOutputFile = refEncOutput.replace(".splt.bit", ".wav") - - # Run CUT Encoder - dutEncCommand = self.reformatCommand(command) - dutEncOutput = self.getOutputFile(dutEncCommand) - self.process(command=self.setCommandExec(tag="ISAR_ENC", command=dutEncCommand)) - - # Decode the encoded output with Reference decoder - dutDecCommand = self.reformatCommand( - command=self.Commands["ISAR"][decCommandIdx] - ) - dutDecOutputFile = self.getRendOutputFile(dutDecCommand) - self.process( - command=self.setCommandExec(tag="ISAR", command=dutDecCommand, ref=True) - ) - self.mld( - "ISAR_ENC", - isarEncPytestTag, - refFile=refDecOutputFile, - dutFile=dutDecOutputFile, - ) - - def runOneIsarDecoderTest(self, command: str): - isarEncPytestTag = self.getIsarDecPytestTag(command) - refDecCommand = self.reformatCommand(command, ref=True) - refDecOutputFile = self.getRendOutputFile(refDecCommand) - - # Decode the encoded output with Reference decoder - dutDecCommand = self.reformatCommand(command) - dutDecOutputFile = self.getRendOutputFile(dutDecCommand) - self.process( - command=self.setCommandExec(tag="ISAR", command=dutDecCommand) - ) - self.mld( - "ISAR", isarEncPytestTag, refFile=refDecOutputFile, dutFile=dutDecOutputFile - ) - def getOutputFile(self, command: str): return command.split()[-1] @@ -549,13 +563,6 @@ class MLDConformance: def reformatCommand(self, command: str, ref: bool = False) -> str: command = command.replace("$TESTV_PATH", self.testvecDir) - ################ HACKS ######################### - #command = command.replace("_cut.192.fer", ".192") - #command = command.replace("_cut.192", ".192") - #command = command.replace(".fer.192", ".192") - #command = command.replace(".192.fer", ".192") - ################################################## - if ref: command = command.replace( "$CUT_PATH/dut/sba_bs/pkt/", f"{self.testvDir}/ref/sba_bs/pkt/" @@ -567,38 +574,38 @@ class MLDConformance: "$CUT_PATH/ref/param_file/", f"{self.testvDir}/ref/param_file/" ) command = command.replace( - "$CUT_PATH/renderer_short/ref/", f"{self.testvDir}/ref/renderer_short/" + "$CUT_PATH/renderer_short/ref/", f"{self.testvDir}/renderer_short/ref/" ) command = command.replace( "$CUT_PATH/split_rendering/ref", - f"{self.testvDir}/ref/split_rendering", + f"{self.testvDir}/split_rendering/ref", ) command = command.replace( "$CUT_PATH/ref/sba_bs/", f"{self.testvDir}/ref/sba_bs/" ) else: command = command.replace( - "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/param_file/enc/", f"{self.outputDir}/enc/" ) command = command.replace( - "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/param_file/dec/", f"{self.outputDir}/dec/" ) command = command.replace( - "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/dut/renderer_short/" + "$CUT_PATH/renderer_short/ref/", f"{self.outputDir}/renderer_short/ref" ) command = command.replace( "$CUT_PATH/split_rendering/cut/", - f"{self.outputDir}/dut/split_rendering/", + f"{self.outputDir}/split_rendering/cut/", ) command = command.replace( "$CUT_PATH/split_rendering/ref", - f"{self.outputDir}/dut/split_rendering", + f"{self.outputDir}/split_rendering/", ) command = command.replace( - "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/dut/enc/" + "$CUT_PATH/ref/sba_bs/pkt/", f"{self.outputDir}/enc/" ) command = command.replace( - "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/dut/dec/" + "$CUT_PATH/ref/sba_bs/raw/", f"{self.outputDir}/dec/" ) command = command.replace( @@ -612,45 +619,73 @@ class MLDConformance: ) return command - def runOneCommand(self, tag: str, command: str): + def runOneCommand(self, tag: str, pyTestsTag: str): + # Run CUT Cmdline + testDesc = self.TestDesc[tag][pyTestsTag] + self.process(command=testDesc.dutCmdline) + self.stats() + + def analyseOneCommand(self, tag: str, pyTestsTag: str): if tag == "ENC": - self.runOneEncoderTest(command) + self.analyseOneEncoderTest(tag, pyTestsTag) elif tag == "DEC": - self.runOneDecoderTest(tag, command) + self.analyseWavOutputTest(tag, pyTestsTag) elif tag == "REND": - self.runOneRendererTest(tag, command) + self.analyseWavOutputTest(tag, pyTestsTag) elif tag == "ISAR_ENC": - self.runOneIsarEncoderTest(command) + self.analyseOneIsarEncoderTest(tag, pyTestsTag) elif tag == "ISAR": - self.runOneIsarDecoderTest(command) + self.analyseWavOutputTest(tag, pyTestsTag) else: assert False, f"Un-implemented Tag {tag}" self.stats() def runTag(self, tag: str): + selectedTests = list() + if self.filter: + for pyTestsTag in self.TestDesc[tag].keys(): + if self.filter in self.TestDesc[tag][pyTestsTag].rawCmdline: + selectedTests.append(pyTestsTag) + else: + selectedTests = list(self.TestDesc[tag].keys()) + + self.totalTests = len(selectedTests) + print( + f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + ) + if not self.args.no_multi_processing: + with Pool() as pool: + args = [(tag, pyTestsTag) for pyTestsTag in selectedTests] + pool.starmap(self.runOneCommand, args) + else: + for pyTestsTag in selectedTests: + self.runOneCommand(tag, pyTestsTag) + + def analyseTag(self, tag: str): # reset MLD, Sample Stats open(self.mldcsv[tag], "w").close() with open(self.sampleStats[tag], "w") as f: f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") - commands = list() + selectedTests = list() if self.filter: - for command in self.Commands[tag]: - if self.filter in command: - commands.append(command) + for pyTestsTag in self.TestDesc[tag].keys(): + if self.filter in self.TestDesc[tag][pyTestsTag].rawCmdline: + selectedTests.append(pyTestsTag) else: - commands = self.Commands[tag] + selectedTests = list(self.TestDesc[tag].keys()) - self.totalTests = len(commands) + self.totalTests = len(selectedTests) print( - f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + f"Analysing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" ) if not self.args.no_multi_processing: with Pool() as pool: - args = [(tag, command) for command in commands] - pool.starmap(self.runOneCommand, args) + args = [(tag, pyTestsTag) for pyTestsTag in selectedTests] + pool.starmap(self.analyseOneCommand, args) else: - for command in commands: - self.runOneCommand(tag, command) + for pyTestsTag in selectedTests: + self.analyseOneCommand(tag, pyTestsTag) + self.doAnalysis(selectTag=tag) def process(self, command) -> int: if self.args.verbose: @@ -687,6 +722,13 @@ class MLDConformance: def mld(self, tag, pytestTag, refFile, dutFile): mldThisFile = np.zeros(0) + with open(self.failedCmdsFile, "a") as f: + if not os.path.exists(refFile): + f.write(f"File does not exists: {refFile}\n") + return + if not os.path.exists(dutFile): + f.write(f"File does not exists : {dutFile}\n") + return with tempfile.TemporaryDirectory() as tmpdir: refSamples, fsR = readfile(refFile, outdtype="float") dutSamples, fsD = readfile(dutFile, outdtype="float") @@ -707,12 +749,14 @@ class MLDConformance: for ch in range(nChans): mldFile = os.path.join( tmpdir, f"{tempfile.gettempprefix()}_ch{ch}_MLD2.txt" - ) + ) refFileMono = os.path.join( - tmpdir, os.path.basename(refFile).replace(".wav", f"_REF_ch{ch}.wav") + tmpdir, + os.path.basename(refFile).replace(".wav", f"_REF_ch{ch}.wav"), ) dutFileMono = os.path.join( - tmpdir, os.path.basename(dutFile).replace(".wav", f"_DUT_ch{ch}.wav") + tmpdir, + os.path.basename(dutFile).replace(".wav", f"_DUT_ch{ch}.wav"), ) writefile(refFileMono, refSamples[:, ch], 48000) writefile(dutFileMono, dutSamples[:, ch], 48000) @@ -724,15 +768,20 @@ class MLDConformance: dutFileMono, ] + # Output on stdout with wavdiffbin with open(mldFile, "w") as fd: c = subprocess.run( - " ".join(command), stdout=fd, stderr=subprocess.STDOUT, text=True, shell=True + " ".join(command), + stdout=fd, + stderr=subprocess.STDOUT, + text=True, + shell=True, ) - fd.close() - mldThisChan = np.loadtxt(mldFile, delimiter=";", dtype=float, skiprows=1) + mldThisChan = np.loadtxt( + mldFile, delimiter=";", dtype=float, skiprows=1 + ) mldThisChan = mldThisChan[:, 2] - if ch == 0: mldThisFile = mldThisChan else: @@ -747,14 +796,26 @@ class MLDConformance: ) def doAnalysis(self, selectTag="all"): - keys = MLDConformance.IVAS_Bins.keys() if selectTag == "all" else [selectTag] + keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: if os.path.exists(self.mldcsv[tag]): mdlValues = np.loadtxt(self.mldcsv[tag], delimiter=" ", dtype=float) - bePercent = np.loadtxt(self.sampleStats[tag], delimiter=",", dtype=float, skiprows=1, usecols=3) - maxDiff = np.loadtxt(self.sampleStats[tag], delimiter=",", dtype=float, skiprows=1, usecols=1) + bePercent = np.loadtxt( + self.sampleStats[tag], + delimiter=",", + dtype=float, + skiprows=1, + usecols=3, + ) + maxDiff = np.loadtxt( + self.sampleStats[tag], + delimiter=",", + dtype=float, + skiprows=1, + usecols=1, + ) bePercentAvg = np.average(bePercent) - maxDiffmax = np.max(maxDiff)*32768.0 + maxDiffmax = np.max(maxDiff) * 32768.0 N = mdlValues.shape[0] if N == 0: continue @@ -774,7 +835,9 @@ class MLDConformance: print(f"<{tag}> Frames with MLD <= 2 : {m2} frames ({PCNT(m2)}%)") print(f"<{tag}> Frames with MLD <= 5 : {m5} frames ({PCNT(m5)}%)") print(f"<{tag}> BE samples percentage = {bePercentAvg}") - print(f"<{tag}> max absolute diff = {maxDiffmax}, sample range (-32768, 32767)") + print( + f"<{tag}> max absolute diff = {maxDiffmax}, sample range (-32768, 32767)" + ) print("##########################################################\n") @@ -792,11 +855,13 @@ if __name__ == "__main__": parser.add_argument( "--ref_build_path", type=str, + default="", help="Path to the reference build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) parser.add_argument( "--cut_build_path", type=str, + default="", help="Path to the CUT build folder containing IVAS Encoder, Decoder, Renderer and Post Render binaries", ) parser.add_argument( @@ -810,7 +875,7 @@ if __name__ == "__main__": default=False, action="store_true", help="process dut ENC files with REF decoder", - ) + ) parser.add_argument( "--verbose", default=False, @@ -834,7 +899,7 @@ if __name__ == "__main__": "--test-mode", type=str, default="ALL", - help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ALL"]', + help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ISAR_ENC", "ALL"]', ) parser.add_argument( "--no-multi-processing", @@ -843,16 +908,27 @@ if __name__ == "__main__": help="Disable multi-processing for sequential test run (debugging)", ) parser.add_argument( - "--analyse-only", + "--analyse", + default=False, + action="store_true", + help="Perform MLD analysis on CUT outputs generated", + ) + parser.add_argument( + "--report-only", + default=False, + action="store_true", + help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", + ) + parser.add_argument( + "-c", + "--clean-output-dir", default=False, action="store_true", help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", ) - args = parser.parse_args() conformance = MLDConformance(args) - conformance.accumulateCommands() if args.regenerate_enc_refs: @@ -860,10 +936,11 @@ if __name__ == "__main__": conformance.runReferenceGeneration(encTag="ENC") sys.exit(0) - testTags = ( - MLDConformance.IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] - ) + testTags = IVAS_Bins.keys() if args.test_mode == "ALL" else [args.test_mode] for tag in testTags: - if not args.analyse_only: + if args.report_only: + conformance.doAnalysis(selectTag=tag) + elif not args.analyse: conformance.runTag(tag) - conformance.doAnalysis(selectTag=tag) + else: + conformance.analyseTag(tag)