diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 2caca4838b2ef1d555cad065a0683655b72491b0..d56974ef0a113d6a44b660f5d66f6403dc0e0db5 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -42,20 +42,25 @@ It might be required to set Clang-18 as the default clang on the machine sudo apt install python3.13 python3.13-venv ``` + +## Run CUT tests on Target platform + +Running the conformance tests requires around 30 gb of disk space and around 6 gb of RAM space. + +To run CUT binaries on the targeted platform, it is necessary to have the correct setup for python and dependency packages. + - Create virtual environment for Python 3.13 and install requirements ```shell python3.13 -m venv pyConformance source pyConformance/bin/activate cd ivas-codec - python -m pip install -r tests/requirements.txt + python -m pip install -r scripts/ivas_conformance/requirements.txt ``` -## Run CUT tests on Target platform - -To run CUT binaries on the targeted platform, it is necessary to replicate the initial setup for python and dependency packages. The CUT build of the IVAS binaries should be made available in a selected folder and needed for the next step +The CUT build of the IVAS binaries should be made available in a selected folder and needed for the next step -To run IVAS DUT commands on the TARGET platform (may be different from ubuntu/linux) +To run IVAS CUT commands on the TARGET platform (may be different from ubuntu/linux) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR @@ -244,91 +249,91 @@ MLD Corridor passed for ISAR with max MLD diff of 0.0 All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Renderer, ISAR Encoder and ISAR Decoder only. The commandline allows for ```-test-mode=``` for this functionality, examples: -- Run DUT IVAS Encoder Tests Only (on Target Platform) +- Run CUT IVAS Encoder Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ENC ``` -- Analyse BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) +- Analyse BE conformance for CUT IVAS Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse --be-test ``` -- Analyse NON-BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) +- Analyse NON-BE conformance for CUT IVAS Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse ``` -- Run DUT IVAS Decoder Tests Only (on Target Platform) +- Run CUT IVAS Decoder Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=DEC ``` -- Analyse BE conformance for DUT IVAS Decoder Outputs Only +- Analyse BE conformance for CUT IVAS Decoder Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse --be-test ``` -- Analyse NON-BE conformance DUT IVAS Decoder Outputs Only (on Reference Platform) +- Analyse NON-BE conformance CUT IVAS Decoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse ``` -- Run DUT IVAS Renderer Tests Only (on Target Platform) +- Run CUT IVAS Renderer Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=REND ``` -- Analyse BE conformance for DUT Renderer Outputs Only +- Analyse BE conformance for CUT Renderer Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse --be-test ``` -- Analyse NON-BE conformance DUT Renderer Outputs Only +- Analyse NON-BE conformance CUT Renderer Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse ``` -- Run DUT ISAR Encoder Tests Only (on Target Platform) +- Run CUT ISAR Encoder Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR_ENC ``` -- Analyse BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) +- Analyse BE conformance for CUT ISAR Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse --be-test ``` -- Analyse NON-BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) +- Analyse NON-BE conformance for CUT ISAR Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse ``` -- Run DUT ISAR Decoder Tests Only (on Target Platform) +- Run CUT ISAR Decoder Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR ``` -- Analyse BE conformance for DUT ISAR Decoder Outputs Only +- Analyse BE conformance for CUT ISAR Decoder Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse --be-test ``` -- Analyse NON-BE conformance DUT ISAR Decoder Outputs Only +- Analyse NON-BE conformance CUT ISAR Decoder Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index e82cbb8300a15cd7992608e5ee351ef32532533d..ad182f869e6d5355d6d9db3532a2b4c8dc34713f 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -252,8 +252,12 @@ class MLDConformance: self.filter = args.filter exe_platform = platform.system() if exe_platform == "Windows": - exe_platform = "Win32" + exe_platform = "Win64" self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") + if not os.path.exists(self.wavdiffbin): + if exe_platform == "Windows": + exe_platform = "Win32" + self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") self.CutBins = dict() self.mldcsv = dict() self.BEcsv = dict() @@ -910,7 +914,7 @@ class MLDConformance: else: print(f"<{tag}> PASSED BE TEST") - def computeCorridor(self, mldRefWithTags, mldCutWithTags, tag, threshold=0.0): + def computeCorridor(self, mldRefWithTags, mldCutWithTags, tag, threshold=0.1): mldDict = dict[str, list]() failed = False