From 872c1d4bc9727e235e135854643c8d267de9dd6c Mon Sep 17 00:00:00 2001 From: Rishabh Tyagi Date: Fri, 21 Nov 2025 17:09:50 -0600 Subject: [PATCH 1/2] backport some conformance changes from SA4 package to main --- scripts/ivas_conformance/README.md | 13 +++++++++---- scripts/ivas_conformance/runConformance.py | 8 ++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 2caca4838b..9c1118a5d2 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -42,18 +42,23 @@ It might be required to set Clang-18 as the default clang on the machine sudo apt install python3.13 python3.13-venv ``` + +## Run CUT tests on Target platform + +Running the conformance tests requires around 30 gb of disk space and around 6 gb of RAM space. + +To run CUT binaries on the targeted platform, it is necessary to have the correct setup for python and dependency packages. + - Create virtual environment for Python 3.13 and install requirements ```shell python3.13 -m venv pyConformance source pyConformance/bin/activate cd ivas-codec - python -m pip install -r tests/requirements.txt + python -m pip install -r scripts/ivas_conformance/requirements.txt ``` -## Run CUT tests on Target platform - -To run CUT binaries on the targeted platform, it is necessary to replicate the initial setup for python and dependency packages. The CUT build of the IVAS binaries should be made available in a selected folder and needed for the next step +The CUT build of the IVAS binaries should be made available in a selected folder and needed for the next step To run IVAS DUT commands on the TARGET platform (may be different from ubuntu/linux) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index e82cbb8300..ad182f869e 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -252,8 +252,12 @@ class MLDConformance: self.filter = args.filter exe_platform = platform.system() if exe_platform == "Windows": - exe_platform = "Win32" + exe_platform = "Win64" self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") + if not os.path.exists(self.wavdiffbin): + if exe_platform == "Windows": + exe_platform = "Win32" + self.wavdiffbin = os.path.join(self.toolsdir, exe_platform, "wav-diff") self.CutBins = dict() self.mldcsv = dict() self.BEcsv = dict() @@ -910,7 +914,7 @@ class MLDConformance: else: print(f"<{tag}> PASSED BE TEST") - def computeCorridor(self, mldRefWithTags, mldCutWithTags, tag, threshold=0.0): + def computeCorridor(self, mldRefWithTags, mldCutWithTags, tag, threshold=0.1): mldDict = dict[str, list]() failed = False -- GitLab From 4066b002d44df664b30865f44c796a443eded0c5 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Mon, 24 Nov 2025 09:32:09 +0100 Subject: [PATCH 2/2] change DUT to CUT --- scripts/ivas_conformance/README.md | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 9c1118a5d2..d56974ef0a 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -60,7 +60,7 @@ To run CUT binaries on the targeted platform, it is necessary to have the correc The CUT build of the IVAS binaries should be made available in a selected folder and needed for the next step -To run IVAS DUT commands on the TARGET platform (may be different from ubuntu/linux) +To run IVAS CUT commands on the TARGET platform (may be different from ubuntu/linux) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR @@ -249,91 +249,91 @@ MLD Corridor passed for ISAR with max MLD diff of 0.0 All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Renderer, ISAR Encoder and ISAR Decoder only. The commandline allows for ```-test-mode=``` for this functionality, examples: -- Run DUT IVAS Encoder Tests Only (on Target Platform) +- Run CUT IVAS Encoder Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ENC ``` -- Analyse BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) +- Analyse BE conformance for CUT IVAS Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse --be-test ``` -- Analyse NON-BE conformance for DUT IVAS Encoder Outputs Only (on Reference Platform) +- Analyse NON-BE conformance for CUT IVAS Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse ``` -- Run DUT IVAS Decoder Tests Only (on Target Platform) +- Run CUT IVAS Decoder Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=DEC ``` -- Analyse BE conformance for DUT IVAS Decoder Outputs Only +- Analyse BE conformance for CUT IVAS Decoder Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse --be-test ``` -- Analyse NON-BE conformance DUT IVAS Decoder Outputs Only (on Reference Platform) +- Analyse NON-BE conformance CUT IVAS Decoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse ``` -- Run DUT IVAS Renderer Tests Only (on Target Platform) +- Run CUT IVAS Renderer Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=REND ``` -- Analyse BE conformance for DUT Renderer Outputs Only +- Analyse BE conformance for CUT Renderer Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse --be-test ``` -- Analyse NON-BE conformance DUT Renderer Outputs Only +- Analyse NON-BE conformance CUT Renderer Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse ``` -- Run DUT ISAR Encoder Tests Only (on Target Platform) +- Run CUT ISAR Encoder Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR_ENC ``` -- Analyse BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) +- Analyse BE conformance for CUT ISAR Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse --be-test ``` -- Analyse NON-BE conformance for DUT ISAR Encoder Outputs Only (on Reference Platform) +- Analyse NON-BE conformance for CUT ISAR Encoder Outputs Only (on Reference Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse ``` -- Run DUT ISAR Decoder Tests Only (on Target Platform) +- Run CUT ISAR Decoder Tests Only (on Target Platform) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR ``` -- Analyse BE conformance for DUT ISAR Decoder Outputs Only +- Analyse BE conformance for CUT ISAR Decoder Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse --be-test ``` -- Analyse NON-BE conformance DUT ISAR Decoder Outputs Only +- Analyse NON-BE conformance CUT ISAR Decoder Outputs Only ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse -- GitLab