Commit 9fe09161 authored by TYAGIRIS's avatar TYAGIRIS
Browse files

Merge branch 'ivas_conf_be_MD' into '1378_conformance_script_isar' 

BE conformance tests to runconformance.py

See merge request !2382
parents 09c33620 c9802041

scripts/ivas_conformance/README.md

+38 −8
Original line number Diff line number Diff line
@@ -231,7 +231,13 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere 
  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ENC

  ```



- Analyse DUT IVAS Encoder Outputs Only (on Reference Platform)

- Analyse BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform)



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse --be-test

  ```



- Analyse NON-BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform)



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse
@@ -243,10 +249,16 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere 
  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=DEC

  ```



- Analyse DUT IVAS Decoder Outputs Only (on Reference Platform)

- Analyse BE conformance for DUT IVAS Decoder Outputs Only



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse --be-test

  ```



- Analyse NON-BE conformance DUT IVAS Decoder Outputs Only (on Reference Platform)



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=DEC --analyse

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec  --test-mode=DEC --analyse

  ```



- Run DUT IVAS Renderer Tests Only (on Target Platform)
@@ -255,10 +267,16 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere 
  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=REND

  ```



- Analyse DUT Renderer Outputs Only (on Reference Platform)

- Analyse BE conformance for DUT Renderer Outputs Only 



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse --be-test

  ```



- Analyse NON-BE conformance DUT Renderer Outputs Only 



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=REND --analyse

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse

  ```



- Run DUT ISAR Encoder Tests Only (on Target Platform)
@@ -267,7 +285,13 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere 
  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR_ENC

  ```



- Analyse DUT ISAR Encoder Outputs Only (on Reference Platform)

- Analyse BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform)



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse --be-test

  ```



- Analyse NON-BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform)



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse
@@ -279,8 +303,14 @@ All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Rendere 
  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR

  ```



- Analyse DUT ISAR Decoder Outputs Only (on Reference Platform)

- Analyse BE conformance for DUT ISAR Decoder Outputs Only



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec  --test-mode=ISAR --analyse --be-test

  ```



- Analyse NON-BE conformance DUT ISAR Decoder Outputs Only 



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR --analyse

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec  --test-mode=ISAR --analyse

  ```

scripts/ivas_conformance/runConformance.py

+108 −34
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ import scipy.io.wavfile as wav 
import warnings

import math

import scipy.signal as sig

import filecmp



sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
@@ -246,11 +247,13 @@ class MLDConformance: 
        self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff")

        self.CutBins = dict()

        self.mldcsv = dict()

        self.BEcsv = dict()

        self.sampleStats = dict()



        for tag in IVAS_Bins.keys():

            self.CutBins[tag] = os.path.join(self.cut_build_path, IVAS_Bins[tag])

            self.mldcsv[tag] = os.path.join(self.outputDir, f"mld_{tag}.csv")

            self.BEcsv[tag] = os.path.join(self.outputDir, f"BE_{tag}.csv")

            self.sampleStats[tag] = os.path.join(

                self.outputDir, f"sampleStats_{tag}.csv"

            )
@@ -502,6 +505,12 @@ class MLDConformance: 
        assert isinstance(

            testDesc, TestDesciptor

        ), f"Expected pcm test descriptor for {tag}"



        if self.args.be_test:

            DUTmdFiles = self.getMDfileList(outFile=testDesc.dutOutput)

            REFmdFiles = self.getMDfileList(outFile=testDesc.refOutput)

            self.beTest(tag, dutPytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput, DUTmdFileList=DUTmdFiles, REFmdFileList=REFmdFiles )

        else:

            ##### skip MLD verification for files with only 1 frame as MLD does not run with such files. Possible solution: append 0s and then compare #####

            if testDesc.rawCmdline.find("_cut.192.fer") == -1:

                self.mld(
@@ -516,6 +525,10 @@ class MLDConformance: 
        assert isinstance(

            testDesc, BitstrmTestDescriptor

        ), f"Expected bitstream test descriptor for {tag}"



        if self.args.be_test:

            self.beTest(tag, encPytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput )

        else:

            refDecOutputFile = testDesc.refOutput.replace(".192", "_REFDECODED.wav")



            dutDecOutputFile = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav")
@@ -534,6 +547,10 @@ class MLDConformance: 
        assert isinstance(

            testDesc, BitstrmTestDescriptor

        ), f"Expected bitstream test descriptor for {tag}"



        if self.args.be_test:

            self.beTest(tag, pytestTag, refFile=testDesc.refOutput, dutFile=testDesc.dutOutput )

        else:

            refDecOutputFile = testDesc.refOutput.replace(".splt.bit", ".wav")

            dutDecOutputFile = testDesc.dutOutput.replace(".splt.bit", ".wav")

            # Decode the encoded output with Reference ISAR decoder
@@ -556,6 +573,14 @@ class MLDConformance: 
    def getOutputFile(self, command: str):

        return command.split()[-1]



    def getMDfileList(self, outFile: str):

        MDfiles = []

        for i in range(0,3):

            MDfiles.append(outFile + '.' + str(i) + '.csv')

        MDfiles.append(outFile + '.met')

        return MDfiles





    def setCommandExec(self, tag: str, command, ref: bool = False):

        exec = self.RefBins[tag] if ref else self.CutBins[tag]

        commands = command.split()
@@ -663,6 +688,10 @@ class MLDConformance: 


    def analyseTag(self, tag: str):

        # reset MLD, Sample Stats

        if self.args.be_test:

            with open(self.BEcsv[tag], "w") as f:

                f.write(f"PYTESTTAG, BE=0  NON-BE=1\n")

        else:        

            open(self.mldcsv[tag], "w").close()

            with open(self.sampleStats[tag], "w") as f:

                f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n")
@@ -685,6 +714,9 @@ class MLDConformance: 
        else:

            for pyTestsTag in selectedTests:

                self.analyseOneCommand(tag, pyTestsTag)

        if self.args.be_test:

            self.doBEanalysis(selectTag=tag)

        else:

            self.doAnalysis(selectTag=tag)



    def process(self, command) -> int:
@@ -795,6 +827,42 @@ class MLDConformance: 
                    f"{pytestTag}, {maxDiff}, {rmsdB}, {beSamplesPercent}, {mldThisFile.max()}\n"

                )



    def beTest(self, tag, pytestTag, refFile, dutFile, DUTmdFileList=[], REFmdFileList=[]):

            BE_flag = 0

            if not filecmp.cmp(refFile, dutFile):

                BE_flag = 1

            with open(self.BEcsv[tag], "a") as f:

                f.write(

                    f"{pytestTag}, {BE_flag}\n"

                )

            

            for i in range(0,len(DUTmdFileList)):

                if os.path.exists(DUTmdFileList[i]):

                    BE_flag = 0

                    if not filecmp.cmp(REFmdFileList[i], DUTmdFileList[i]):

                        BE_flag = 1

                    with open(self.BEcsv[tag], "a") as f:

                        f.write(

                            f"{DUTmdFileList[i]}, {BE_flag}\n"

                        )





    def doBEanalysis(self, selectTag="all"):

        keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag]

        for tag in keys:

            if os.path.exists(self.BEcsv[tag]):

                BEresult = np.loadtxt(

                    self.BEcsv[tag],

                    delimiter=",",

                    dtype=int,

                    skiprows=1,

                    usecols=1,

                )

                if np.sum(BEresult) > 0:

                    print(f"<{tag}> FAILED BE TEST, check {self.BEcsv[tag]}")

                else:

                    print(f"<{tag}> PASSED BE TEST")



    def doAnalysis(self, selectTag="all"):

        keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag]

        for tag in keys:
@@ -901,6 +969,12 @@ if __name__ == "__main__": 
        default="ALL",

        help='Choose tests to run ["ENC", "DEC", "REND", "ISAR", "ISAR_ENC", "ALL"]',

    )

    parser.add_argument(

        "--be-test",

        default=False,

        action="store_true",

        help='runs only BE tests',

    )    

    parser.add_argument(

        "--no-multi-processing",

        default=False,