From 70eac75997774a247f426f39e5d3a32032b611e3 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Thu, 30 Apr 2026 15:47:49 +0200 Subject: [PATCH 1/7] porting fixes from MR !2572 --- scripts/ivas_conformance/README.md | 284 +++++++------- scripts/ivas_conformance/runConformance.py | 430 ++++++++++++++++----- 2 files changed, 479 insertions(+), 235 deletions(-) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 19763d14b..46769bbb7 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -1,4 +1,4 @@ -# IVAS Conformance Scripts +# IVAS Conformance Scripts, ver 3.1 This folder contains scripts for running IVAS conformance tests. @@ -67,25 +67,37 @@ PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDi
Example Output of CUT execution -

+
+```console
 Accumulating commands from Readme_IVAS_dec.txt
 Accumulating commands from Readme_IVAS_rend.txt
 Accumulating commands from Readme_IVAS_enc.txt
 Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
 Accumulating commands from Readme_IVAS_ISAR_dec.txt
 Accumulating commands from Readme_IVAS_JBM_dec.txt
+
 No of tests :
     ENC : 381
     DEC : 637
     REND : 666
     ISAR_ENC : 1032
     ISAR : 1032
-Executing tests for ENC   (381 tests)
-Executing tests for DEC   (637 tests)
-Executing tests for REND   (666 tests)
-Executing tests for ISAR_ENC   (1032 tests)
-Executing tests for ISAR   (1032 tests)
-
+ +Executing tests for ENC (381 tests): +--------------------------- +[ENC] OK + +... + +Summary of results: +--------------------- +[ENC] OK +[DEC] OK +[REND] OK +[ISAR_ENC] OK +[ISAR] OK +``` +
This should generate outputs in scripts/CUT_OUTPUTS folder which looks like below:- @@ -106,7 +118,7 @@ If CUT test execution is done on a different platform, the scripts/CUT_OUTPUTS m ### Perform the BE comparison on the CUT outputs on reference platform -The BE comparison is performed to the CUT outputs using the command below. Encoded outputs will be decoded using the reference decoder executables as part of the process. The BE comparison is then performed between the CUT and reference decoded outputs. This includes comparison of ".wav"-files, and ".csv" and ".met" metadata files. If any non-BE results are observed, this is reported on the command-line and link to an analysis ".csv" file is given. The analysis file shows which exact files were non-BE. An example passing output is shown below. If all test sets print `PASSED BE TEST`, then CUT outputs are BE-conformant. +The BE comparison is performed to the CUT outputs using the command below. Encoded outputs are decoded using the reference decoder executables as part of the process. The BE comparison is then performed between the CUT and reference decoded outputs. This includes comparison of `.wav` files and `.csv`/`.met` metadata files. If non-BE results are observed, this is reported on the command line and in the generated analysis CSV output. ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --be-test @@ -114,30 +126,39 @@ PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDi
Example Output of BE comparison -

+
+```console
 Accumulating commands from Readme_IVAS_dec.txt
 Accumulating commands from Readme_IVAS_enc.txt
 Accumulating commands from Readme_IVAS_rend.txt
 Accumulating commands from Readme_IVAS_JBM_dec.txt
 Accumulating commands from Readme_IVAS_ISAR_dec.txt
 Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
+
 No of tests :
     ENC : 374
     DEC : 638
     REND : 911
     ISAR_ENC : 1032
     ISAR : 1252
-Analysing tests for ENC   (374 tests)
-<ENC> PASSED BE TEST
-Analysing tests for DEC   (638 tests)
-<DEC> PASSED BE TEST
-Analysing tests for REND   (911 tests)
-<REND> PASSED BE TEST
-Analysing tests for ISAR_ENC   (1032 tests)
-<ISAR_ENC> PASSED BE TEST
-Analysing tests for ISAR   (1252 tests)
-<ISAR> PASSED BE TEST
-
+ +Analysing tests for ENC (374 tests): +--------------------------- + +[ENC] OK (ERRORS=0, BE=374, NON-BE=0, MLD CORRIDOR FAILURES=0) +[DEC] OK (ERRORS=0, BE=638, NON-BE=0, MLD CORRIDOR FAILURES=0) + +... + +Summary of results: +--------------------- +[ENC] OK +[DEC] OK +[REND] OK +[ISAR_ENC] OK +[ISAR] OK +``` +
### Perform the MLD based non-BE analysis on the CUT outputs on reference platform (Ubuntu 24.04) @@ -151,187 +172,158 @@ PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDi
Example Output of non-BE analysis -

+
+```console
 Accumulating commands from Readme_IVAS_dec.txt
 Accumulating commands from Readme_IVAS_enc.txt
 Accumulating commands from Readme_IVAS_rend.txt
 Accumulating commands from Readme_IVAS_JBM_dec.txt
 Accumulating commands from Readme_IVAS_ISAR_dec.txt
 Accumulating commands from Readme_IVAS_ISAR_post_rend.txt
+
 No of tests :
     ENC : 374
     DEC : 638
     REND : 911
     ISAR_ENC : 1032
     ISAR : 1252
-Analysing tests for ENC   (374 tests)
-
-##########################################################
-<ENC> Total Frames: 3074220
-<ENC> MAX MLD across all frames : 0.0
-<ENC> Frames with MLD == 0 : 3074220 frames (100.0%)
-<ENC> Frames with MLD <= 0.5 : 3074220 frames (100.0%)
-<ENC> Frames with MLD <= 1 : 3074220 frames (100.0%)
-<ENC> Frames with MLD <= 2 : 3074220 frames (100.0%)
-<ENC> Frames with MLD <= 5 : 3074220 frames (100.0%)
-<ENC> BE samples percentage = 100.0
-<ENC> max absolute diff = 0.0, sample range (-32768, 32767)
-##########################################################
-
-MLD Corridor passed for ENC with max MLD diff of 0.0
-Analysing tests for DEC   (638 tests)
 
-##########################################################
-<DEC> Total Frames: 5079252
-<DEC> MAX MLD across all frames : 0.0
-<DEC> Frames with MLD == 0 : 5079252 frames (100.0%)
-<DEC> Frames with MLD <= 0.5 : 5079252 frames (100.0%)
-<DEC> Frames with MLD <= 1 : 5079252 frames (100.0%)
-<DEC> Frames with MLD <= 2 : 5079252 frames (100.0%)
-<DEC> Frames with MLD <= 5 : 5079252 frames (100.0%)
-<DEC> BE samples percentage = 100.0
-<DEC> max absolute diff = 0.0, sample range (-32768, 32767)
-##########################################################
-
-MLD Corridor passed for DEC with max MLD diff of 0.0
-Analysing tests for REND   (911 tests)
+Analysing tests for ENC  (374 tests):
+---------------------------
 
 ##########################################################
-<REND> Total Frames: 5576907
-<REND> MAX MLD across all frames : 0.0
-<REND> Frames with MLD == 0 : 5576907 frames (100.0%)
-<REND> Frames with MLD <= 0.5 : 5576907 frames (100.0%)
-<REND> Frames with MLD <= 1 : 5576907 frames (100.0%)
-<REND> Frames with MLD <= 2 : 5576907 frames (100.0%)
-<REND> Frames with MLD <= 5 : 5576907 frames (100.0%)
-<REND> BE samples percentage = 100.0
-<REND> max absolute diff = 0.0, sample range (-32768, 32767)
+ Total Frames: 3074220
+ MAX MLD across all frames : 0.0
+ Frames with MLD == 0 : 3074220 frames (100.0%)
+ Frames with MLD <= 0.5 : 3074220 frames (100.0%)
+ Frames with MLD <= 1 : 3074220 frames (100.0%)
+ Frames with MLD <= 2 : 3074220 frames (100.0%)
+ Frames with MLD <= 5 : 3074220 frames (100.0%)
+ BE samples percentage = 100.0
+ max absolute diff = 0.0, sample range (-32768, 32767)
 ##########################################################
 
-MLD Corridor passed for REND with max MLD diff of 0.0
-Analysing tests for ISAR_ENC   (1032 tests)
+[ENC] OK (ERRORS=0, BE=374, NON-BE=0, MLD CORRIDOR FAILURES=0)
 
-##########################################################
-<ISAR_ENC> Total Frames: 2125956
-<ISAR_ENC> MAX MLD across all frames : 0.0
-<ISAR_ENC> Frames with MLD == 0 : 2125956 frames (100.0%)
-<ISAR_ENC> Frames with MLD <= 0.5 : 2125956 frames (100.0%)
-<ISAR_ENC> Frames with MLD <= 1 : 2125956 frames (100.0%)
-<ISAR_ENC> Frames with MLD <= 2 : 2125956 frames (100.0%)
-<ISAR_ENC> Frames with MLD <= 5 : 2125956 frames (100.0%)
-<ISAR_ENC> BE samples percentage = 100.0
-<ISAR_ENC> max absolute diff = 0.0, sample range (-32768, 32767)
-##########################################################
-
-MLD Corridor passed for ISAR_ENC with max MLD diff of 0.0
-Analysing tests for ISAR   (1252 tests)
+...
 
-##########################################################
-<ISAR> Total Frames: 2590956
-<ISAR> MAX MLD across all frames : 0.0
-<ISAR> Frames with MLD == 0 : 2590956 frames (100.0%)
-<ISAR> Frames with MLD <= 0.5 : 2590956 frames (100.0%)
-<ISAR> Frames with MLD <= 1 : 2590956 frames (100.0%)
-<ISAR> Frames with MLD <= 2 : 2590956 frames (100.0%)
-<ISAR> Frames with MLD <= 5 : 2590956 frames (100.0%)
-<ISAR> BE samples percentage = 100.0
-<ISAR> max absolute diff = 0.0, sample range (-32768, 32767)
-##########################################################
+Summary of results:
+---------------------
+[ENC] OK
+[DEC] OK
+[REND] OK
+[ISAR_ENC] OK
+[ISAR] OK
+```
 
-MLD Corridor passed for ISAR with max MLD diff of 0.0
-
## Executing specific tests only -All CUT tests can be run specifically for IVAS Encoder,IVAS Decoder,IVAS Renderer, ISAR Encoder and ISAR Decoder only. The commandline allows for ```-test-mode=``` for this functionality, examples: - -- Run CUT IVAS Encoder Tests Only (on Target Platform) - - ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ENC - ``` - -- Analyse BE conformance for CUT IVAS Encoder Outputs Only (on Reference Platform) - - ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse --be-test - ``` - -- Analyse NON-BE conformance for CUT IVAS Encoder Outputs Only (on Reference Platform) - - ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ENC --analyse - ``` - -- Run CUT IVAS Decoder Tests Only (on Target Platform) - - ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=DEC - ``` - -- Analyse BE conformance for CUT IVAS Decoder Outputs Only - - ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse --be-test - ``` - -- Analyse NON-BE conformance CUT IVAS Decoder Outputs Only (on Reference Platform) - - ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=DEC --analyse - ``` - -- Run CUT IVAS Renderer Tests Only (on Target Platform) +Use `--filter TOKEN` to control conformance level, test groups, output formats, and substring matching. + +- Token types: + - `LEVEL1`, `LEVEL2`, `LEVEL3`: conformance levels. `LEVEL3` is the default. + - `ENC`, `DEC`, `REND`, `ISAR`, `ISAR_ENC`: test groups. + - `MONO`, `STEREO`, `EXT`, `HOA`, `SBA`, `MC`: output-format tokens. + - `HOA` expands to `HOA2`, `HOA3`. + - `SBA` expands to `FOA`, `HOA2`, `HOA3`. + - `MC` expands to `5_1`, `7_1`, `5_1_4`, `5_1_2`, `7_1_4`. + - Any other token is treated as a case-insensitive substring match. +- Token modifiers: + - `TOKEN`: restrictive token. Multiple restrictive tokens combine with logical AND. + - `+TOKEN`: additive token. Adds matching tests even if they would otherwise be excluded. + - `-TOKEN`: subtractive token. Removes matching tests from the final selection. + - `TOKEN*`: wildcard token. Matches all known tokens starting with the given prefix. + - `+TOKEN*`, `-TOKEN*`: wildcard token with add/remove behavior. + +### LEVEL1, LEVEL2 and LEVEL3 behavior + +For all levels, the default test-group baseline is `ENC` + `DEC`. +`REND`, `ISAR`, and `ISAR_ENC` are optional and are only included if explicitly selected as plain test-group tokens or added via `+REND`, `+ISAR` and `+ISAR_ENC`. + +When `--filter LEVEL1` is specified, the following default tests are run: + +- Encoder (`ENC`) tests: only tests with bitrate up to 80 kbps (inclusive). +- Decoder (`DEC`) tests: + - `EXT` output format: only bitrate up to 80 kbps (inclusive). + - `MONO` output format: all bitrates. + - `STEREO` output format: all bitrates. + +- The default `LEVEL1` tests may be restricted by adding more tokens (acting as logical AND). + - Example: `--filter LEVEL1 DEC MONO` keeps only `MONO` tests from the LEVEL1-eligible DEC set. + - Example: `--filter LEVEL1 JBM` keeps all LEVEL1-eligible ENC tests but only JBM tests from the LEVEL1-eligible DEC tests. +- `+TOKEN` adds tests to the final LEVEL1 selection, even if they would otherwise be restricted. + - Example: `--filter LEVEL1 DEC JBM +BINAURAL*` runs JBM-matching LEVEL1 DEC tests and additionally includes DEC tests with `BINAURAL` output formats, i.e. `BINAURAL`, `BINAURAL_IR`, `BINAURAL_ROOM_IR`, `BINAURAL_REVERB`. +- `-TOKEN` removes matching tests from the final LEVEL1 selection (including tests added via `+TOKEN`). + - Example: `--filter LEVEL1 DEC +JBM -VOIP` adds JBM-matching tests to LEVEL1 DEC tests and excludes DEC `VOIP` tests. +- Renderer and ISAR tests are not run by default in `LEVEL1`. + - Add `+REND`, `+ISAR`, and/or `+ISAR_ENC` in `--filter` to include them. + - Use `+ISAR*` if you want wildcard expansion across all `ISAR*`-prefixed test-group tokens. + +When `--filter LEVEL2` is specified, all selection rules above remain the same, +except the bitrate cap is set to 192 kbps: + +- Encoder (`ENC`) tests: only tests with bitrate up to 192 kbps (inclusive). +- Decoder (`DEC`) tests: + - `EXT` output format: only bitrate up to 192 kbps (inclusive). + - `MONO` output format: all bitrates. + - `STEREO` output format: all bitrates. + +When `--filter LEVEL3` is specified, there are no restrictions on the bitrate or output formats. + +Examples (non-BE): + +- Default behavior (same as LEVEL3 baseline): run only ENC and DEC test groups ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=REND + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse ``` -- Analyse BE conformance for CUT Renderer Outputs Only +- LEVEL3 plus renderer and ISAR test groups ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse --be-test + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL3 +REND +ISAR +ISAR_ENC ``` -- Analyse NON-BE conformance CUT Renderer Outputs Only +- LEVEL1 baseline (ENC+DEC with LEVEL1 restrictions) ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=REND --analyse + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 ``` -- Run CUT ISAR Encoder Tests Only (on Target Platform) +- LEVEL1 plus renderer and ISAR test groups ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR_ENC + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 +REND +ISAR +ISAR_ENC ``` -- Analyse BE conformance for CUT ISAR Encoder Outputs Only (on Reference Platform) +- LEVEL1 with additional case-insensitive command substring filtering ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse --be-test + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 DEC voip ``` -- Analyse NON-BE conformance for CUT ISAR Encoder Outputs Only (on Reference Platform) +- LEVEL1 with additive BINAURAL decoder matching ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --test-mode=ISAR_ENC --analyse + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 +BINAURAL ``` -- Run CUT ISAR Decoder Tests Only (on Target Platform) +- LEVEL1 with restrictive and additive terms together ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --test-mode=ISAR + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 DEC JBM +BINAURAL ``` -- Analyse BE conformance for CUT ISAR Decoder Outputs Only +- LEVEL2 baseline (ENC+DEC with LEVEL2 restrictions) ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse --be-test + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL2 ``` -- Analyse NON-BE conformance CUT ISAR Decoder Outputs Only +- LEVEL2 plus renderer and ISAR test groups ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --test-mode=ISAR --analyse + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL2 +REND +ISAR +ISAR_ENC ``` diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 574d7be13..939d08fe1 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -53,6 +53,64 @@ import scipy.signal as sig sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) + +def _preprocess_filter_args(): + """Preprocess sys.argv to handle --filter with minus-prefixed tokens. + + Argparse stops consuming args when it encounters e.g. '-TERM', treating it as a flag. + This function escapes filter tokens by wrapping minus-prefixed tokens with a marker, + allowing argparse to treat them as regular arguments. + + Stores the escape mapping in _FILTER_ESCAPES for later unescaping. + + Returns: modified sys.argv with filter tokens escaped + """ + global _FILTER_ESCAPES + _FILTER_ESCAPES = {} + marker_prefix = "@FILT_" + result_argv = [] + i = 0 + + while i < len(sys.argv): + arg = sys.argv[i] + + # When we see --filter, process all following tokens until next option (--) or end + if arg == "--filter": + result_argv.append(arg) + i += 1 + + # Collect all filter tokens, escaping those that look like flags + while i < len(sys.argv): + tok = sys.argv[i] + + # Stop if we hit the next option (--something or -X where X is not escaped) + if tok.startswith("--") or (tok.startswith("-") and len(tok) <= 2 and tok != "-"): + # Exception: stop only if it's a known argparse option + known_short_options = {"-h", "--help"} + if tok in known_short_options or tok.startswith("--"): + break + break + + # Escape tokens starting with - to protect them from argparse + if tok.startswith("-"): + escaped = f"{marker_prefix}{len(_FILTER_ESCAPES)}" + _FILTER_ESCAPES[escaped] = tok + result_argv.append(escaped) + else: + result_argv.append(tok) + + i += 1 + else: + result_argv.append(arg) + i += 1 + + + return result_argv + + +# Module-level dict to track filter token escapes +_FILTER_ESCAPES = {} + def readfile( filename: str, nchannels: int = 1, fs: int = 48000, outdtype="float" ) -> Tuple[np.ndarray, int]: @@ -197,7 +255,32 @@ IVAS_Bins = { "ISAR": "ISAR_post_rend", } -DECODER_OUTPUT_FORMATS = {"MONO", "STEREO", "EXT"} +# Automatically append .exe on Windows +if platform.system() == "Windows": + IVAS_Bins = {k: f"{v}.exe" for k, v in IVAS_Bins.items()} + +DECODER_OUTPUT_FORMATS = { + "MONO", + "STEREO", + "BINAURAL", + "BINAURAL_ROOM_IR", + "BINAURAL_ROOM_REVERB", + "5_1", + "7_1", + "5_1_4", + "5_1_2", + "7_1_4", + "FOA", + "HOA2", + "HOA3", + "EXT", +} + +DECODER_OUTPUT_FORMAT_ALIASES = { + "HOA": {"HOA2", "HOA3"}, + "SBA": {"FOA", "HOA2", "HOA3"}, + "MC": {"5_1", "7_1", "5_1_4", "5_1_2", "7_1_4"}, +} def validate_build_binaries(parser, build_path: str, build_label: str) -> None: @@ -209,7 +292,6 @@ def validate_build_binaries(parser, build_path: str, build_label: str) -> None: ) missing = [] - is_windows = platform.system() == "Windows" for tag, binary in IVAS_Bins.items(): candidate = os.path.join(abs_build_path, binary) candidate_exe = f"{candidate}.exe" @@ -238,7 +320,7 @@ ReferenceMasaMDFiles = { "REND": "masa_ref_REND.csv", } -PROGRAM_VERSION = "4.0" +PROGRAM_VERSION = "3.1" @dataclass @@ -375,7 +457,7 @@ class MLDConformance: def setupDUT(self): self.cut_build_path = self.args.cut_build_path - self.filter = self.args.filter + self.filter = getattr(self.args, "filter_display", self.args.filter) exe_platform = platform.system() if exe_platform == "Windows": exe_platform = "Win64" @@ -614,10 +696,6 @@ class MLDConformance: ) else: print(f"{pyTestTag} not found in ISAR decoder") - print("No of tests :") - for tag in testDesciptor.keys(): - print(f" {tag} : {len(testDesciptor[tag])}") - return testDesciptor def genEncoderReferences(self, tag: str, encPytestTag: str): @@ -1068,13 +1146,19 @@ class MLDConformance: values.append(float(match.replace("_", "."))) return values - def _isBitrateAtMost80(self, rawCmdline: str) -> bool: - """Check if all bitrates in command line are <= 80 kbps. - - For bitrate switching tests (e.g., 'from_32_kbps_to_96_kbps'), this checks - that the upper (target) bitrate does not exceed 80 kbps.""" + def _isBitrateAtMost(self, rawCmdline: str, max_kbps: float) -> bool: + """Check if all bitrates in command line are <= max_kbps. + + For bitrate switching tests (e.g., 'from_32_kbps_to_96_kbps'), this checks + that the upper (target) bitrate does not exceed max_kbps.""" values = self._extractKbpsValues(rawCmdline) - return bool(values) and max(values) <= 80.0 + return bool(values) and max(values) <= float(max_kbps) + + def _isBitrateAtMost80(self, rawCmdline: str) -> bool: + return self._isBitrateAtMost(rawCmdline, 80.0) + + def _isBitrateAtMost192(self, rawCmdline: str) -> bool: + return self._isBitrateAtMost(rawCmdline, 192.0) def _outputFormatsInCommand(self, rawCmdline: str) -> set[str]: text = rawCmdline.upper() @@ -1108,30 +1192,6 @@ class MLDConformance: text = rawCmdline.lower() return any(term.lower() in text for term in terms) - def _expectedMasaMDMetFiles(self, tag: str) -> list[str]: - """Return expected DUT-side .met files for a tag's selected tests.""" - if tag not in {"ENC", "DEC", "REND"}: - return [] - - met_files = [] - for pyTestsTag in self.getSelectedTestsForTag(tag): - testDesc = self.TestDesc[tag][pyTestsTag] - - if tag == "ENC": - if not isinstance(testDesc, BitstreamTestDescriptor): - continue - dut_dec_output = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav") - met_files.append(dut_dec_output + ".met") - else: - if not isinstance(testDesc, TestDescriptor): - continue - met_files.append(testDesc.dutOutput + ".met") - - return met_files - - def _hasAnyProducedMasaMDMetFile(self, tag: str) -> bool: - return any(os.path.exists(path) for path in self._expectedMasaMDMetFiles(tag)) - def _matchesLevel1(self, tag: str, rawCmdline: str) -> bool: if tag == "ENC": return self._isBitrateAtMost80(rawCmdline) @@ -1154,6 +1214,28 @@ class MLDConformance: # For REND/ISAR/ISAR_ENC under LEVEL1, tag-level inclusion is decided at testTags parsing. return True + def _matchesLevel2(self, tag: str, rawCmdline: str) -> bool: + if tag == "ENC": + return self._isBitrateAtMost192(rawCmdline) + + if tag == "DEC": + formats = self._outputFormatsInCommand(rawCmdline) + requested_formats = set(getattr(self.args, "filter_decoder_formats", [])) + + ext_ok = "EXT" in formats and self._isBitrateAtMost192(rawCmdline) + mono_ok = "MONO" in formats + stereo_ok = "STEREO" in formats + default_level2_dec_ok = ext_ok or mono_ok or stereo_ok + + if requested_formats: + # Plain decoder format tokens are restrictive under LEVEL2. + return default_level2_dec_ok and bool(formats.intersection(requested_formats)) + + return default_level2_dec_ok + + # For REND/ISAR/ISAR_ENC under LEVEL2, tag-level inclusion is decided at testTags parsing. + return True + def _testPassesFilter(self, tag: str, rawCmdline: str) -> bool: level = getattr(self.args, "filter_level", "LEVEL3") restrictive_terms = getattr(self.args, "filter_restrictive_terms", []) @@ -1168,9 +1250,11 @@ class MLDConformance: passes_level = True if level == "LEVEL1": passes_level = self._matchesLevel1(tag, rawCmdline) + elif level == "LEVEL2": + passes_level = self._matchesLevel2(tag, rawCmdline) passes_requested_formats = True - if tag in {"ENC", "DEC"} and requested_formats: + if requested_formats: cmd_formats = self._outputFormatsInCommand(rawCmdline) passes_requested_formats = bool(cmd_formats.intersection(requested_formats)) @@ -1196,19 +1280,14 @@ class MLDConformance: def runTag(self, tag: str) -> bool: failed_before = self.getFailedCommandCount() - selectedTests = list() - if self.filter: - for pyTestsTag in self.TestDesc[tag].keys(): - if self.filter in self.TestDesc[tag][pyTestsTag].rawCmdline: - selectedTests.append(pyTestsTag) - else: - selectedTests = list(self.TestDesc[tag].keys()) + selectedTests = self.getSelectedTestsForTag(tag) self.totalTests = len(selectedTests) print( - f"Executing tests for {tag} {'Filter=' + self.filter if self.filter else ''} ({self.totalTests} tests)", + f"Executing tests for {tag} ({self.totalTests} tests):", flush=True, ) + print("---------------------------") if not self.args.no_multi_processing: with Pool() as pool: args = [ @@ -1289,19 +1368,14 @@ class MLDConformance: with open(self.sampleStats[tag], "w") as f: f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n") - selectedTests = [] - if self.filter: - for pyTestsTag in self.TestDesc[tag].keys(): - if self.filter in self.TestDesc[tag][pyTestsTag].rawCmdline: - selectedTests.append(pyTestsTag) - else: - selectedTests = list(self.TestDesc[tag].keys()) + selectedTests = self.getSelectedTestsForTag(tag) self.totalTests = len(selectedTests) print( - f"Analysing tests for {tag} {'Filter=' + self.filter if self.filter else ''} ({self.totalTests} tests)", + f"Analysing tests for {tag} ({self.totalTests} tests):", flush=True, ) + print("---------------------------") def handle_test_result( testPrefix, @@ -1410,6 +1484,7 @@ class MLDConformance: if self.args.regenerate_mld_ref: return command_fail_count == 0 and analysis_ok + print() result_details = f"ERRORS={command_fail_count}, BE={be_count}, NON-BE={non_be_count}" if not self.args.be_test: result_details += f", MLD CORRIDOR FAILURES={failure_count}" @@ -1418,13 +1493,15 @@ class MLDConformance: print() if command_fail_count == 0 and failure_count == 0 and analysis_ok: - print(f"[{tag}] OK ({result_details})\n") + print( + f"[{tag}] OK (ERRORS={command_fail_count}, BE={be_count}, NON-BE={non_be_count}, MLD CORRIDOR FAILURES={failure_count})\n" + ) return True - print(f"[{tag}] FAILED ({result_details})") - if self.args.be_test and be_failure_csv is not None: - print(f"[{tag}] BE test failed, check {be_failure_csv}") - elif worst_failure is not None: + print( + f"[{tag}] FAILED (ERRORS={command_fail_count}, BE={be_count}, NON-BE={non_be_count}, MLD CORRIDOR FAILURES={failure_count})" + ) + if worst_failure is not None: print( f"[{tag}] Worst MLD corridor failure: {worst_failure['prefix']} {worst_failure['tag']} " f"(NON-BE, MLD_MAX={worst_failure['mld']})" @@ -1444,7 +1521,7 @@ class MLDConformance: contextPrefix: str = "", emitConsole: bool = True, returnOutput: bool = False, - ) -> int: + ) -> Union[int, Tuple[int, str]]: contextPrefix = contextPrefix or (f"[{tag}]" if tag else "") return self._process( command, @@ -1463,12 +1540,14 @@ class MLDConformance: contextPrefix: str = "", emitConsole: bool = True, returnOutput: bool = False, - ) -> int: + ) -> Union[int, Tuple[int, str]]: prefix = (contextPrefix + " ") if contextPrefix else "" if self.args.verbose and emitConsole: print(f"{prefix}Command: {command}", flush=True) if self.args.dryrun: self.appendRunlog(command=command) + if returnOutput: + return 0, "" return 0 c = subprocess.run( @@ -1560,7 +1639,7 @@ class MLDConformance: if not row: continue - # vlad: MASA MD CSV columns are: frame index, then per-frame diff metrics. + # MASA MD CSV columns are: frame index, then per-frame diff metrics. # We Use a single scalar per frame (max abs over all diff columns) # so corridor handling is the same as MLD's one-value-per-frame pipeline. metric_values = [] @@ -1645,7 +1724,7 @@ class MLDConformance: ) return None, (command_str, err_msg) - # vlad: a Non-zero return code from masaDiffTool can indicate detected differences, + # A Non-zero return code from masaDiffTool can indicate detected differences, # but which is valid for non-BE analysis. Treat missing/invalid CSV as failure. if frame_scores.size == 0: self.appendFailed( @@ -1851,6 +1930,12 @@ class MLDConformance: keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: if os.path.exists(self.BEcsv[tag]): + # For filtered runs it is valid to have zero selected tests for a tag. + # In that case the BE csv contains only the header; skip loadtxt to avoid warnings. + with open(self.BEcsv[tag], "r") as f: + non_empty_lines = [line for line in f if line.strip()] + if len(non_empty_lines) <= 1: + continue BEresult = np.loadtxt( self.BEcsv[tag], delimiter=",", @@ -1858,11 +1943,10 @@ class MLDConformance: skiprows=1, usecols=1, ) - if np.sum(BEresult) > 0: - return False, self.BEcsv[tag] - return True, None - - return True, None + # if np.sum(BEresult) > 0: + # print(f"<{tag}> FAILED BE TEST, check {self.BEcsv[tag]}") + # else: + # print(f"<{tag}> PASSED BE TEST") def computeCorridor( self, @@ -1908,7 +1992,7 @@ class MLDConformance: ref_preview = ", ".join(refTags[:3]) if ref_count else "" dut_preview = ", ".join(dutTags[:3]) if dut_count else "" warn_msg = ( - f"Warning: {tag} {metricLabel} corridor comparison skipped because reference and DUT frame tags do not match " + f"Warning: {tag} corridor comparison skipped because reference and DUT frame tags do not match " f"(ref_count={ref_count}, dut_count={dut_count}, ref_first=[{ref_preview}], dut_first=[{dut_preview}])." ) print(f"\033[93m{warn_msg}\033[00m") @@ -2118,12 +2202,8 @@ class MLDConformance: if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Compare .wav files in two folders using mld per frame" - ) - parser.add_argument( - "--version", - action="version", - version=f"%(prog)s {PROGRAM_VERSION}", + description="Compare .wav files in two folders using mld per frame", + formatter_class=argparse.RawTextHelpFormatter ) parser.add_argument( @@ -2171,16 +2251,35 @@ if __name__ == "__main__": parser.add_argument( "--filter", - type=str, + nargs="+", default=None, - help="Filter test based on text provided", - ) - parser.add_argument( - "--test-mode", - type=str, - default=None, - choices=["ENC", "DEC", "REND", "ISAR", "ISAR_ENC"], - help='Choose one test group to run ["ENC", "DEC", "REND", "ISAR", "ISAR_ENC"]. If omitted, all are run.', + metavar="TOKEN", + help=( + "Select which tests to run. Default baseline: ENC+DEC tests (REND/ISAR optional).\n" + "\n" + "Token types:\n" + " LEVEL1, LEVEL2, LEVEL3 — Conformance level. LEVEL1: ≤80 kbps; LEVEL2: ≤192 kbps; LEVEL3: unlimited (default).\n" + " ENC, DEC, REND, ISAR, ISAR_ENC — Test groups.\n" + " MONO, STEREO, EXT, HOA, SBA, MC — Output formats. Aliases: HOA→{HOA2,HOA3}, SBA→{FOA,HOA2,HOA3}, MC→{5_1,7_1,5_1_4,5_1_2,7_1_4}.\n" + " (any other) — Substring match (case-insensitive). Multiple terms combine with AND.\n" + "\n" + "Token modifiers:\n" + " +TOKEN — Add matching tests to selection (even if they would be excluded).\n" + " -TOKEN — Remove matching tests from selection.\n" + " TOKEN* — Wildcard: match all tokens starting with TOKEN (e.g., ISAR* → {ISAR, ISAR_ENC}; BINAURAL* → {BINAURAL, BINAURAL_IR, BINAURAL_ROOM_IR, ...}).\n" + " +TOKEN*, -TOKEN* — Wildcard with add/remove modifiers (e.g., +ISAR*, -BINAURAL*).\n" + "\n" + "Examples:\n" + " --filter LEVEL1 Run LEVEL1 ENC+DEC (≤80 kbps).\n" + " --filter LEVEL2 Run LEVEL2 ENC+DEC (≤192 kbps).\n" + " --filter LEVEL1 DEC MONO Restrict to DEC MONO tests at LEVEL1.\n" + " --filter LEVEL1 +REND +ISAR Add REND and ISAR to LEVEL1 baseline ENC+DEC.\n" + " --filter DEC HOA Run DEC tests with HOA2/HOA3 outputs.\n" + " --filter DEC -voip Run DEC tests except those matching 'voip'.\n" + " --filter ISAR* Add ISAR and ISAR_ENC (wildcard expansion).\n" + " --filter DEC +BINAURAL* Run LEVEL3 DEC + all BINAURAL output formats.\n" + " --filter +ISAR* -voip Add ISAR/ISAR_ENC groups then remove 'voip' tests.\n" + ), ) parser.add_argument( "--be-test", @@ -2207,11 +2306,10 @@ if __name__ == "__main__": help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", ) parser.add_argument( - "-c", "--clean-output-dir", default=False, action="store_true", - help="Do not run DUT, use existing mld and bitdiff stats files to generate analysis only", + help="Delete and recreate the CUT_OUTPUTS directory before running, discarding any previous DUT outputs", ) parser.add_argument( "--regenerate-mld-ref", @@ -2219,7 +2317,21 @@ if __name__ == "__main__": action="store_true", help="Run analysis and unconditionally regenerate mld_ref2 files for all tags", ) - args = parser.parse_args() + # Preprocess sys.argv to handle --filter with minus-prefixed tokens like -JBM + modified_argv = _preprocess_filter_args() + args = parser.parse_args(modified_argv[1:]) # Skip program name; parse_args expects args without it + + # Unescape filter tokens that were escaped during sys.argv preprocessing + if args.filter: + marker_prefix = "@FILT_" + unescaped_filter = [] + for tok in args.filter: + # Unescape any tokens that were wrapped by preprocessing + if tok in _FILTER_ESCAPES: + unescaped_filter.append(_FILTER_ESCAPES[tok]) + else: + unescaped_filter.append(tok) + args.filter = unescaped_filter if not os.path.isdir(args.testvecDir): parser.error( @@ -2231,19 +2343,157 @@ if __name__ == "__main__": if args.cut_build_path: validate_build_binaries(parser, args.cut_build_path, "CUT") + # Parse --filter into level + optional tag selection + optional format/substring filters. + raw_filter = " ".join(args.filter) if args.filter else "" + filter_tokens = [tok for tok in re.split(r"[\s,]+", raw_filter.strip()) if tok] + + valid_tags = set(IVAS_Bins.keys()) + valid_levels = {"LEVEL1", "LEVEL2", "LEVEL3"} + valid_decoder_formats = set(DECODER_OUTPUT_FORMATS).union( + DECODER_OUTPUT_FORMAT_ALIASES.keys() + ) + + level_tokens = [] + tag_tokens = [] + tag_add_tokens = [] + tag_remove_tokens = [] + decoder_format_tokens = [] + restrictive_terms = [] + additive_terms = [] + subtractive_terms = [] + + for tok in filter_tokens: + sign = "" + base_tok = tok + if len(tok) > 1 and tok[0] in {"+", "-"}: + sign = tok[0] + base_tok = tok[1:] + + upper_tok = base_tok.upper() + + # Prefix wildcard selection for tags and decoder output formats, e.g. ISAR*, BINAURAL*. + if upper_tok.endswith("*") and len(upper_tok) > 1: + prefix = upper_tok[:-1] + matched_tags = sorted(t for t in valid_tags if t.startswith(prefix)) + matched_format_tokens = sorted( + f for f in valid_decoder_formats if f.startswith(prefix) + ) + + if not matched_tags and not matched_format_tokens: + parser.error( + f"Wildcard token '{tok}' did not match any known tag or decoder output format." + ) + + if sign == "+": + tag_add_tokens.extend(matched_tags) + elif sign == "-": + tag_remove_tokens.extend(matched_tags) + else: + tag_tokens.extend(matched_tags) + + expanded_wildcard_formats = [] + for fmt in matched_format_tokens: + expanded_wildcard_formats.extend( + sorted(DECODER_OUTPUT_FORMAT_ALIASES.get(fmt, {fmt})) + ) + + if sign == "+": + additive_terms.extend(expanded_wildcard_formats) + elif sign == "-": + subtractive_terms.extend(expanded_wildcard_formats) + else: + decoder_format_tokens.extend(expanded_wildcard_formats) + + continue + + if upper_tok in valid_levels: + if sign: + parser.error(f"Level token '{tok}' cannot be prefixed with '+' or '-'.") + level_tokens.append(upper_tok) + elif upper_tok in valid_tags: + if sign == "+": + tag_add_tokens.append(upper_tok) + elif sign == "-": + tag_remove_tokens.append(upper_tok) + else: + tag_tokens.append(upper_tok) + elif upper_tok in valid_decoder_formats: + expanded_formats = DECODER_OUTPUT_FORMAT_ALIASES.get(upper_tok, {upper_tok}) + if sign == "+": + additive_terms.append(base_tok) + elif sign == "-": + subtractive_terms.append(base_tok) + else: + decoder_format_tokens.extend(sorted(expanded_formats)) + else: + if sign == "+": + additive_terms.append(base_tok) + elif sign == "-": + subtractive_terms.append(base_tok) + else: + restrictive_terms.append(base_tok) + + if len(set(level_tokens)) > 1: + parser.error("Multiple filter levels specified. Use only one of LEVEL1, LEVEL2, LEVEL3.") + + filter_level = level_tokens[0] if level_tokens else "LEVEL3" + + # Preserve order while removing duplicates. + tag_tokens = list(dict.fromkeys(tag_tokens)) + tag_add_tokens = list(dict.fromkeys(tag_add_tokens)) + tag_remove_tokens = list(dict.fromkeys(tag_remove_tokens)) + decoder_format_tokens = list(dict.fromkeys(decoder_format_tokens)) + + # Default with no --filter: run all tags. + # LEVEL1 default baseline: ENC + DEC (REND/ISAR optional unless explicitly selected/added). + # For other filter cases, plain tag_tokens restrict selected tags and +tag_tokens add tags. + if tag_tokens: + selected_tag_set = set(tag_tokens) + elif filter_tokens and filter_level == "LEVEL1": + selected_tag_set = {"ENC", "DEC"} + else: + selected_tag_set = set(IVAS_Bins.keys()) + for tag in tag_add_tokens: + selected_tag_set.add(tag) + for tag in tag_remove_tokens: + selected_tag_set.discard(tag) + testTags = [tag for tag in IVAS_Bins.keys() if tag in selected_tag_set] + + args.filter_display = raw_filter if raw_filter else None + args.filter_restrictive_terms = restrictive_terms + args.filter_add_terms = additive_terms + args.filter_remove_terms = subtractive_terms + args.filter_decoder_formats = decoder_format_tokens + args.filter_level = filter_level + conformance = MLDConformance(args) conformance.accumulateCommands() + if args.filter_display: + print(f"Applying filter: {args.filter_display}") + print() + if args.regenerate_enc_refs: + print() + print("Reference regeneration targets:") + print(f" ISAR_ENC : {len(conformance.TestDesc['ISAR_ENC'])}") + print(f" ENC : {len(conformance.TestDesc['ENC'])}") + print() conformance.runReferenceGeneration(encTag="ISAR_ENC") conformance.runReferenceGeneration(encTag="ENC") sys.exit(0) + print() + print("No of tests:") + for tag in IVAS_Bins.keys(): + n = len(conformance.getSelectedTestsForTag(tag)) if tag in testTags else 0 + print(f" {tag} : {n}") + print() + # If --regenerate-mld-ref is set, treat as --analyse if args.regenerate_mld_ref: args.analyse = True - testTags = IVAS_Bins.keys() if args.test_mode is None else [args.test_mode] tag_results = {} for tag in testTags: if args.report_only: @@ -2261,3 +2511,5 @@ if __name__ == "__main__": for tag in testTags: tag_status = "OK" if tag_results.get(tag, False) else "FAILED" print(f"[{tag}] {tag_status}") + print() + -- GitLab From e2b8f1f69cb6ba1140a22533075a43b07e57b138 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Thu, 30 Apr 2026 15:49:15 +0200 Subject: [PATCH 2/7] add .exe to Win executables --- scripts/ivas_conformance/README.md | 82 ++++--------------- scripts/ivas_conformance/runConformance.py | 94 +++------------------- 2 files changed, 24 insertions(+), 152 deletions(-) diff --git a/scripts/ivas_conformance/README.md b/scripts/ivas_conformance/README.md index 46769bbb7..1aaced54d 100644 --- a/scripts/ivas_conformance/README.md +++ b/scripts/ivas_conformance/README.md @@ -48,12 +48,11 @@ Running the conformance tests requires around 30 gb of disk space and around 6 g To run CUT binaries on the targeted platform, it is necessary to have the correct setup for python and dependency packages. -- Create virtual environment for Python 3.13 and install requirements +- Create virtual environment for Python 3.13 and install requirements from the root folder of the IVAS codebase ```shell python3.13 -m venv pyConformance source pyConformance/bin/activate - cd ivas-codec python -m pip install -r scripts/ivas_conformance/requirements.txt ``` @@ -220,10 +219,9 @@ Summary of results: ## Executing specific tests only -Use `--filter TOKEN` to control conformance level, test groups, output formats, and substring matching. +Use `--filter TOKEN` to select test groups, output formats, and apply substring matching. - Token types: - - `LEVEL1`, `LEVEL2`, `LEVEL3`: conformance levels. `LEVEL3` is the default. - `ENC`, `DEC`, `REND`, `ISAR`, `ISAR_ENC`: test groups. - `MONO`, `STEREO`, `EXT`, `HOA`, `SBA`, `MC`: output-format tokens. - `HOA` expands to `HOA2`, `HOA3`. @@ -236,94 +234,42 @@ Use `--filter TOKEN` to control conformance level, test groups, output formats, - `-TOKEN`: subtractive token. Removes matching tests from the final selection. - `TOKEN*`: wildcard token. Matches all known tokens starting with the given prefix. - `+TOKEN*`, `-TOKEN*`: wildcard token with add/remove behavior. - -### LEVEL1, LEVEL2 and LEVEL3 behavior - -For all levels, the default test-group baseline is `ENC` + `DEC`. -`REND`, `ISAR`, and `ISAR_ENC` are optional and are only included if explicitly selected as plain test-group tokens or added via `+REND`, `+ISAR` and `+ISAR_ENC`. - -When `--filter LEVEL1` is specified, the following default tests are run: - -- Encoder (`ENC`) tests: only tests with bitrate up to 80 kbps (inclusive). -- Decoder (`DEC`) tests: - - `EXT` output format: only bitrate up to 80 kbps (inclusive). - - `MONO` output format: all bitrates. - - `STEREO` output format: all bitrates. - -- The default `LEVEL1` tests may be restricted by adding more tokens (acting as logical AND). - - Example: `--filter LEVEL1 DEC MONO` keeps only `MONO` tests from the LEVEL1-eligible DEC set. - - Example: `--filter LEVEL1 JBM` keeps all LEVEL1-eligible ENC tests but only JBM tests from the LEVEL1-eligible DEC tests. -- `+TOKEN` adds tests to the final LEVEL1 selection, even if they would otherwise be restricted. - - Example: `--filter LEVEL1 DEC JBM +BINAURAL*` runs JBM-matching LEVEL1 DEC tests and additionally includes DEC tests with `BINAURAL` output formats, i.e. `BINAURAL`, `BINAURAL_IR`, `BINAURAL_ROOM_IR`, `BINAURAL_REVERB`. -- `-TOKEN` removes matching tests from the final LEVEL1 selection (including tests added via `+TOKEN`). - - Example: `--filter LEVEL1 DEC +JBM -VOIP` adds JBM-matching tests to LEVEL1 DEC tests and excludes DEC `VOIP` tests. -- Renderer and ISAR tests are not run by default in `LEVEL1`. - - Add `+REND`, `+ISAR`, and/or `+ISAR_ENC` in `--filter` to include them. - - Use `+ISAR*` if you want wildcard expansion across all `ISAR*`-prefixed test-group tokens. - -When `--filter LEVEL2` is specified, all selection rules above remain the same, -except the bitrate cap is set to 192 kbps: - -- Encoder (`ENC`) tests: only tests with bitrate up to 192 kbps (inclusive). -- Decoder (`DEC`) tests: - - `EXT` output format: only bitrate up to 192 kbps (inclusive). - - `MONO` output format: all bitrates. - - `STEREO` output format: all bitrates. - -When `--filter LEVEL3` is specified, there are no restrictions on the bitrate or output formats. + - **Note**: wildcards must be quoted in shell (e.g. `'ISAR*'`) to prevent the shell from expanding them as filename globs before the script receives them. Examples (non-BE): -- Default behavior (same as LEVEL3 baseline): run only ENC and DEC test groups +- Default behavior: run all test groups (ENC, DEC, REND, ISAR, ISAR_ENC) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse ``` -- LEVEL3 plus renderer and ISAR test groups - - ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL3 +REND +ISAR +ISAR_ENC - ``` - -- LEVEL1 baseline (ENC+DEC with LEVEL1 restrictions) - - ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 - ``` - -- LEVEL1 plus renderer and ISAR test groups - - ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 +REND +ISAR +ISAR_ENC - ``` - -- LEVEL1 with additional case-insensitive command substring filtering +- Run only REND, ISAR, and ISAR_ENC test groups ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 DEC voip + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter REND ISAR ISAR_ENC ``` -- LEVEL1 with additive BINAURAL decoder matching +- Run DEC tests with HOA output formats ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 +BINAURAL + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter DEC HOA ``` -- LEVEL1 with restrictive and additive terms together +- Run DEC tests excluding those matching 'voip' ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 DEC JBM +BINAURAL + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter DEC -voip ``` -- LEVEL2 baseline (ENC+DEC with LEVEL2 restrictions) +- Run DEC tests with any BINAURAL output format ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL2 + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter DEC 'BINAURAL*' ``` -- LEVEL2 plus renderer and ISAR test groups +- Run ISAR and ISAR_ENC groups, excluding 'voip' tests ```shell - PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL2 +REND +ISAR +ISAR_ENC + PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter 'ISAR*' -voip ``` diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 939d08fe1..2b98207c0 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -255,8 +255,10 @@ IVAS_Bins = { "ISAR": "ISAR_post_rend", } +is_windows = platform.system() == "Windows" + # Automatically append .exe on Windows -if platform.system() == "Windows": +if is_windows: IVAS_Bins = {k: f"{v}.exe" for k, v in IVAS_Bins.items()} DECODER_OUTPUT_FORMATS = { @@ -294,13 +296,8 @@ def validate_build_binaries(parser, build_path: str, build_label: str) -> None: missing = [] for tag, binary in IVAS_Bins.items(): candidate = os.path.join(abs_build_path, binary) - candidate_exe = f"{candidate}.exe" - exists = os.path.isfile(candidate) or ( - is_windows and os.path.isfile(candidate_exe) - ) - if not exists: - shown = candidate_exe if is_windows else candidate - missing.append(f"{tag}: {shown}") + if not os.path.isfile(candidate): + missing.append(f"{tag}: {candidate}") if missing: parser.error(f"Missing {build_label} binaries:\n - " + "\n - ".join(missing)) @@ -1192,52 +1189,7 @@ class MLDConformance: text = rawCmdline.lower() return any(term.lower() in text for term in terms) - def _matchesLevel1(self, tag: str, rawCmdline: str) -> bool: - if tag == "ENC": - return self._isBitrateAtMost80(rawCmdline) - - if tag == "DEC": - formats = self._outputFormatsInCommand(rawCmdline) - requested_formats = set(getattr(self.args, "filter_decoder_formats", [])) - - ext_ok = "EXT" in formats and self._isBitrateAtMost80(rawCmdline) - mono_ok = "MONO" in formats - stereo_ok = "STEREO" in formats - default_level1_dec_ok = ext_ok or mono_ok or stereo_ok - - if requested_formats: - # Plain decoder format tokens are restrictive under LEVEL1. - return default_level1_dec_ok and bool(formats.intersection(requested_formats)) - - return default_level1_dec_ok - - # For REND/ISAR/ISAR_ENC under LEVEL1, tag-level inclusion is decided at testTags parsing. - return True - - def _matchesLevel2(self, tag: str, rawCmdline: str) -> bool: - if tag == "ENC": - return self._isBitrateAtMost192(rawCmdline) - - if tag == "DEC": - formats = self._outputFormatsInCommand(rawCmdline) - requested_formats = set(getattr(self.args, "filter_decoder_formats", [])) - - ext_ok = "EXT" in formats and self._isBitrateAtMost192(rawCmdline) - mono_ok = "MONO" in formats - stereo_ok = "STEREO" in formats - default_level2_dec_ok = ext_ok or mono_ok or stereo_ok - - if requested_formats: - # Plain decoder format tokens are restrictive under LEVEL2. - return default_level2_dec_ok and bool(formats.intersection(requested_formats)) - - return default_level2_dec_ok - - # For REND/ISAR/ISAR_ENC under LEVEL2, tag-level inclusion is decided at testTags parsing. - return True - def _testPassesFilter(self, tag: str, rawCmdline: str) -> bool: - level = getattr(self.args, "filter_level", "LEVEL3") restrictive_terms = getattr(self.args, "filter_restrictive_terms", []) additive_terms = getattr(self.args, "filter_add_terms", []) subtractive_terms = getattr(self.args, "filter_remove_terms", []) @@ -1247,19 +1199,13 @@ class MLDConformance: if subtractive_terms and self._matchesAnyTerm(rawCmdline, subtractive_terms): return False - passes_level = True - if level == "LEVEL1": - passes_level = self._matchesLevel1(tag, rawCmdline) - elif level == "LEVEL2": - passes_level = self._matchesLevel2(tag, rawCmdline) - passes_requested_formats = True if requested_formats: cmd_formats = self._outputFormatsInCommand(rawCmdline) passes_requested_formats = bool(cmd_formats.intersection(requested_formats)) passes_restrictive_terms = self._matchesAllTerms(rawCmdline, restrictive_terms) - base_selected = passes_level and passes_restrictive_terms and passes_requested_formats + base_selected = passes_restrictive_terms and passes_requested_formats if base_selected: return True @@ -2258,7 +2204,6 @@ if __name__ == "__main__": "Select which tests to run. Default baseline: ENC+DEC tests (REND/ISAR optional).\n" "\n" "Token types:\n" - " LEVEL1, LEVEL2, LEVEL3 — Conformance level. LEVEL1: ≤80 kbps; LEVEL2: ≤192 kbps; LEVEL3: unlimited (default).\n" " ENC, DEC, REND, ISAR, ISAR_ENC — Test groups.\n" " MONO, STEREO, EXT, HOA, SBA, MC — Output formats. Aliases: HOA→{HOA2,HOA3}, SBA→{FOA,HOA2,HOA3}, MC→{5_1,7_1,5_1_4,5_1_2,7_1_4}.\n" " (any other) — Substring match (case-insensitive). Multiple terms combine with AND.\n" @@ -2270,14 +2215,10 @@ if __name__ == "__main__": " +TOKEN*, -TOKEN* — Wildcard with add/remove modifiers (e.g., +ISAR*, -BINAURAL*).\n" "\n" "Examples:\n" - " --filter LEVEL1 Run LEVEL1 ENC+DEC (≤80 kbps).\n" - " --filter LEVEL2 Run LEVEL2 ENC+DEC (≤192 kbps).\n" - " --filter LEVEL1 DEC MONO Restrict to DEC MONO tests at LEVEL1.\n" - " --filter LEVEL1 +REND +ISAR Add REND and ISAR to LEVEL1 baseline ENC+DEC.\n" " --filter DEC HOA Run DEC tests with HOA2/HOA3 outputs.\n" " --filter DEC -voip Run DEC tests except those matching 'voip'.\n" " --filter ISAR* Add ISAR and ISAR_ENC (wildcard expansion).\n" - " --filter DEC +BINAURAL* Run LEVEL3 DEC + all BINAURAL output formats.\n" + " --filter DEC +BINAURAL* Run DEC + all BINAURAL output formats.\n" " --filter +ISAR* -voip Add ISAR/ISAR_ENC groups then remove 'voip' tests.\n" ), ) @@ -2343,17 +2284,15 @@ if __name__ == "__main__": if args.cut_build_path: validate_build_binaries(parser, args.cut_build_path, "CUT") - # Parse --filter into level + optional tag selection + optional format/substring filters. + # Parse --filter into optional tag selection + optional format/substring filters. raw_filter = " ".join(args.filter) if args.filter else "" filter_tokens = [tok for tok in re.split(r"[\s,]+", raw_filter.strip()) if tok] valid_tags = set(IVAS_Bins.keys()) - valid_levels = {"LEVEL1", "LEVEL2", "LEVEL3"} valid_decoder_formats = set(DECODER_OUTPUT_FORMATS).union( DECODER_OUTPUT_FORMAT_ALIASES.keys() ) - level_tokens = [] tag_tokens = [] tag_add_tokens = [] tag_remove_tokens = [] @@ -2406,11 +2345,7 @@ if __name__ == "__main__": continue - if upper_tok in valid_levels: - if sign: - parser.error(f"Level token '{tok}' cannot be prefixed with '+' or '-'.") - level_tokens.append(upper_tok) - elif upper_tok in valid_tags: + if upper_tok in valid_tags: if sign == "+": tag_add_tokens.append(upper_tok) elif sign == "-": @@ -2433,11 +2368,6 @@ if __name__ == "__main__": else: restrictive_terms.append(base_tok) - if len(set(level_tokens)) > 1: - parser.error("Multiple filter levels specified. Use only one of LEVEL1, LEVEL2, LEVEL3.") - - filter_level = level_tokens[0] if level_tokens else "LEVEL3" - # Preserve order while removing duplicates. tag_tokens = list(dict.fromkeys(tag_tokens)) tag_add_tokens = list(dict.fromkeys(tag_add_tokens)) @@ -2445,12 +2375,9 @@ if __name__ == "__main__": decoder_format_tokens = list(dict.fromkeys(decoder_format_tokens)) # Default with no --filter: run all tags. - # LEVEL1 default baseline: ENC + DEC (REND/ISAR optional unless explicitly selected/added). - # For other filter cases, plain tag_tokens restrict selected tags and +tag_tokens add tags. + # Plain tag_tokens restrict selected tags; +tag_tokens add tags. if tag_tokens: selected_tag_set = set(tag_tokens) - elif filter_tokens and filter_level == "LEVEL1": - selected_tag_set = {"ENC", "DEC"} else: selected_tag_set = set(IVAS_Bins.keys()) for tag in tag_add_tokens: @@ -2464,7 +2391,6 @@ if __name__ == "__main__": args.filter_add_terms = additive_terms args.filter_remove_terms = subtractive_terms args.filter_decoder_formats = decoder_format_tokens - args.filter_level = filter_level conformance = MLDConformance(args) conformance.accumulateCommands() -- GitLab From f618ba398c26a4dd93ad444416b8c402accecf0b Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Fri, 1 May 2026 07:10:24 +0200 Subject: [PATCH 3/7] remove unused fcns --- scripts/ivas_conformance/runConformance.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 2b98207c0..b1c6ac62e 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -1136,27 +1136,6 @@ class MLDConformance: def analyseOneCommandFromTuple(self, args): return self.analyseOneCommand(*args) - def _extractKbpsValues(self, rawCmdline: str) -> list[float]: - """Extract all bitrate values from command line (e.g., from 'at_32_kbps' or 'from_32_kbps_to_96_kbps').""" - values = [] - for match in re.findall(r"(\d+(?:_\d+)?)_kbps", rawCmdline.lower()): - values.append(float(match.replace("_", "."))) - return values - - def _isBitrateAtMost(self, rawCmdline: str, max_kbps: float) -> bool: - """Check if all bitrates in command line are <= max_kbps. - - For bitrate switching tests (e.g., 'from_32_kbps_to_96_kbps'), this checks - that the upper (target) bitrate does not exceed max_kbps.""" - values = self._extractKbpsValues(rawCmdline) - return bool(values) and max(values) <= float(max_kbps) - - def _isBitrateAtMost80(self, rawCmdline: str) -> bool: - return self._isBitrateAtMost(rawCmdline, 80.0) - - def _isBitrateAtMost192(self, rawCmdline: str) -> bool: - return self._isBitrateAtMost(rawCmdline, 192.0) - def _outputFormatsInCommand(self, rawCmdline: str) -> set[str]: text = rawCmdline.upper() formats = set() -- GitLab From 67b055954c4b1feb1c29a9e3923ac34a36ac50bf Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Fri, 1 May 2026 07:30:24 +0200 Subject: [PATCH 4/7] pyenv install instructions --- scripts/ivas_conformance/README_Package_Gen.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/ivas_conformance/README_Package_Gen.md b/scripts/ivas_conformance/README_Package_Gen.md index ed313db63..157bb6efc 100644 --- a/scripts/ivas_conformance/README_Package_Gen.md +++ b/scripts/ivas_conformance/README_Package_Gen.md @@ -42,12 +42,11 @@ This folder contains scripts for running IVAS conformance tests. sudo apt install python3.13 python3.13-venv ``` -- Create virtual environment for Python 3.13 and install requirements +- Create virtual environment for Python 3.13 and install requirements from the root folder of the IVAS codebase ```shell python3.13 -m venv pyConformance source pyConformance/bin/activate - cd ivas-codec python -m pip install -r tests/requirements.txt ``` -- GitLab From a89e902a9fb7231f46a822645554d49e44db3a6c Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Fri, 1 May 2026 08:52:22 +0200 Subject: [PATCH 5/7] corrections to MASA MD parsing --- scripts/ivas_conformance/runConformance.py | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index b1c6ac62e..84a13ed25 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -1136,6 +1136,30 @@ class MLDConformance: def analyseOneCommandFromTuple(self, args): return self.analyseOneCommand(*args) + def _expectedMasaMDMetFiles(self, tag: str) -> list[str]: + """Return expected DUT-side .met files for a tag's selected tests.""" + if tag not in {"ENC", "DEC", "REND"}: + return [] + + met_files = [] + for pyTestsTag in self.getSelectedTestsForTag(tag): + testDesc = self.TestDesc[tag][pyTestsTag] + + if tag == "ENC": + if not isinstance(testDesc, BitstreamTestDescriptor): + continue + dut_dec_output = testDesc.dutOutput.replace(".192", "_CUT_REFDECODED.wav") + met_files.append(dut_dec_output + ".met") + else: + if not isinstance(testDesc, TestDescriptor): + continue + met_files.append(testDesc.dutOutput + ".met") + + return met_files + + def _hasAnyProducedMasaMDMetFile(self, tag: str) -> bool: + return any(os.path.exists(path) for path in self._expectedMasaMDMetFiles(tag)) + def _outputFormatsInCommand(self, rawCmdline: str) -> set[str]: text = rawCmdline.upper() formats = set() -- GitLab From 87e2f087170981f35a59dc0ea83235b8e3300411 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Fri, 1 May 2026 09:02:56 +0200 Subject: [PATCH 6/7] fix MASA MD comaprison prinouts --- scripts/ivas_conformance/runConformance.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 84a13ed25..049c541e3 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -1959,6 +1959,7 @@ class MLDConformance: max_mld_value = None keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: + mld_header_printed = False mdlCutWithTags = None if os.path.exists(self.mldcsv[tag]) and os.path.getsize(self.mldcsv[tag]) > 0: mdlCutWithTags = np.loadtxt( @@ -2003,6 +2004,7 @@ class MLDConformance: m5 = np.sum(mdlValues <= 5.0) PCNT = lambda num: int(1000 * num / N) / 10.0 + mld_header_printed = True print(f"\n##########################################################") print(f"<{tag}> Total Frames: {N}") print(f"<{tag}> MAX MLD across all frames : {mdlValues.max()}") @@ -2075,6 +2077,15 @@ class MLDConformance: ) masaMDCutWithTags = np.atleast_1d(masaMDCutWithTags) + will_print_masa = ( + tag in ReferenceMasaMDFiles + and masaMDCutWithTags is not None + and masaMDCutWithTags.size > 0 + ) + if mld_header_printed and not will_print_masa: + print("##########################################################\n") + mld_header_printed = False + if tag in ReferenceMasaMDFiles: if masaMDCutWithTags is not None and masaMDCutWithTags.size > 0: masaMDValues = masaMDCutWithTags["MASA"] @@ -2100,6 +2111,7 @@ class MLDConformance: print( f"<{tag}> Frames with MASA metadata diff <= 1 : {m1} frames ({PCNT(m1)}%)" ) + mld_header_printed = False print("##########################################################\n") if self.args.regenerate_mld_ref: -- GitLab From be38c8dcd9004d7393df241d3ef9ea8e8eb8bbd7 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Fri, 1 May 2026 09:23:09 +0200 Subject: [PATCH 7/7] do not issue warning in case of clipping --- scripts/ivas_conformance/runConformance.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index 049c541e3..87150fa9e 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -43,7 +43,6 @@ import sys import time from typing import Tuple from multiprocessing import Pool -import warnings import math from dataclasses import dataclass from typing import Union @@ -189,7 +188,7 @@ def writefile(filename: str, x: np.ndarray, fs: int = 48000) -> None: np.logical_or(x < np.iinfo(np.int16).min, x > np.iinfo(np.int16).max) ) if clipped_samples > 0: - warnings.warn(f" Warning: {clipped_samples} samples clipped") + # do not issue warning for clipping as it is expected that some implementations may produce out of bound values and the reference implementation clips them. Instead, just count the number of clipped samples and clip the values to int16 range before writing to file to avoid overflow issues. x = np.clip(x, np.iinfo(np.int16).min, np.iinfo(np.int16).max) if file_extension == ".wav": @@ -1037,7 +1036,8 @@ class MLDConformance: self, tag: str, pyTestsTag: str, testIndex: int = 0, totalTests: int = 0 ): # Run CUT Cmdline - testPrefix = f"[{tag} {testIndex}/{totalTests}]" + _w = len(str(totalTests)) + testPrefix = f"[{tag} {testIndex:{_w}}/{totalTests}]" self.appendRunlog( context=self.formatTestHeader(testPrefix, "Running test", pyTestsTag) ) @@ -1059,7 +1059,8 @@ class MLDConformance: def analyseOneCommand( self, tag: str, pyTestsTag: str, testIndex: int = 0, totalTests: int = 0 ): - testPrefix = f"[{tag} {testIndex}/{totalTests}]" + _w = len(str(totalTests)) + testPrefix = f"[{tag} {testIndex:{_w}}/{totalTests}]" header = self.formatTestHeader(testPrefix, "Analyzing test", pyTestsTag) self.appendRunlog(context=header) non_be = None -- GitLab