Loading scripts/ivas_conformance/README.md +40 −6 Original line number Diff line number Diff line Loading @@ -224,12 +224,12 @@ Summary of results: ## Executing specific tests only Use `--filter` to control test levels, tags and optional text/out_format filtering. Use `--filter` to control test levels, tags or output format filtering. - Levels: - `LEVEL3` (default): no level-based restrictions. - `LEVEL3` (default): no bitrate restrictions. - `LEVEL1`: applies the restrictions below. - `LEVEL2`: currently unsupported; script exits with an error. - `LEVEL2`: same rules as `LEVEL1`, but with bitrate cap increased from 80 kbps to 192 kbps. - Tag selection (case-insensitive): `ENC`, `DEC`, `REND`, `ISAR`, `ISAR_ENC`. - Command substring filtering is case-insensitive. - Plain substring terms are restrictive and combined using logical AND. Loading @@ -239,7 +239,11 @@ Use `--filter` to control test levels, tags and optional text/out_format filteri - `-TERM` removes matching tests from the selection. - Example: `--filter DEC -JBM` runs decoder tests except those containing `JBM`. ### LEVEL1 behavior ### LEVEL1, LEVEL2 and LEVEL3 behavior For all levels, the default tag baseline is `ENC` + `DEC`. `REND`, `ISAR`, and `ISAR_ENC` are optional and are only included if explicitly selected as plain tags or added via `+REND` and/or `+ISAR`. When `--filter LEVEL1` is specified, the following default tests are run: Loading @@ -260,14 +264,38 @@ When `--filter LEVEL1` is specified, the following default tests are run: - Add `+REND` and/or `+ISAR` in `--filter` to include them. - If `+ISAR` is provided, both `ISAR` and `ISAR_ENC` test groups are run. When `--filter LEVEL2` is specified, all selection rules above remain the same, except bitrate caps are relaxed to 192 kbps: - Encoder (`ENC`) tests: only tests with bitrate up to 192 kbps (inclusive). - Decoder (`DEC`) tests: - `EXT` output format: only bitrate up to 192 kbps (inclusive). - `MONO` output format: all bitrates. - `STEREO` output format: all bitrates. When `--filter LEVEL3` is specified, all selection rules above remain the same, with no bitrate caps: - Encoder (`ENC`) tests: all bitrates. - Decoder (`DEC`) tests: - `EXT` output format: all bitrates. - `MONO` output format: all bitrates. - `STEREO` output format: all bitrates. Examples (non-BE): - Default behavior (same as LEVEL3): run all tags/tests - Default behavior (same as LEVEL3 baseline): run only ENC and DEC test groups ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse ``` - LEVEL3 plus renderer and ISAR test groups ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL3 +REND +ISAR ``` - LEVEL1 baseline (ENC+DEC with LEVEL1 restrictions) ```shell Loading Loading @@ -298,8 +326,14 @@ Examples (non-BE): PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 DEC JBM +BINAURAL ``` - Unsupported LEVEL2 example (will fail) - LEVEL2 baseline (ENC+DEC with LEVEL2 restrictions) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL2 ``` - LEVEL2 plus renderer and ISAR test groups ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL2 +REND +ISAR ``` scripts/ivas_conformance/runConformance.py +55 −33 Original line number Diff line number Diff line Loading @@ -1015,13 +1015,19 @@ class MLDConformance: values.append(float(match.replace("_", "."))) return values def _isBitrateAtMost80(self, rawCmdline: str) -> bool: """Check if all bitrates in command line are <= 80 kbps. def _isBitrateAtMost(self, rawCmdline: str, max_kbps: float) -> bool: """Check if all bitrates in command line are <= max_kbps. For bitrate switching tests (e.g., 'from_32_kbps_to_96_kbps'), this checks that the upper (target) bitrate does not exceed 80 kbps.""" that the upper (target) bitrate does not exceed max_kbps.""" values = self._extractKbpsValues(rawCmdline) return bool(values) and max(values) <= 80.0 return bool(values) and max(values) <= float(max_kbps) def _isBitrateAtMost80(self, rawCmdline: str) -> bool: return self._isBitrateAtMost(rawCmdline, 80.0) def _isBitrateAtMost192(self, rawCmdline: str) -> bool: return self._isBitrateAtMost(rawCmdline, 192.0) def _outputFormatsInCommand(self, rawCmdline: str) -> set[str]: text = rawCmdline.upper() Loading Loading @@ -1077,6 +1083,28 @@ class MLDConformance: # For REND/ISAR/ISAR_ENC under LEVEL1, tag-level inclusion is decided at testTags parsing. return True def _matchesLevel2(self, tag: str, rawCmdline: str) -> bool: if tag == "ENC": return self._isBitrateAtMost192(rawCmdline) if tag == "DEC": formats = self._outputFormatsInCommand(rawCmdline) requested_formats = set(getattr(self.args, "filter_decoder_formats", [])) ext_ok = "EXT" in formats and self._isBitrateAtMost192(rawCmdline) mono_ok = "MONO" in formats stereo_ok = "STEREO" in formats default_level2_dec_ok = ext_ok or mono_ok or stereo_ok if requested_formats: # Plain decoder format tokens are restrictive under LEVEL2. return default_level2_dec_ok and bool(formats.intersection(requested_formats)) return default_level2_dec_ok # For REND/ISAR/ISAR_ENC under LEVEL2, tag-level inclusion is decided at testTags parsing. return True def _testPassesFilter(self, tag: str, rawCmdline: str) -> bool: level = getattr(self.args, "filter_level", "LEVEL3") restrictive_terms = getattr(self.args, "filter_restrictive_terms", []) Loading @@ -1091,6 +1119,8 @@ class MLDConformance: passes_level = True if level == "LEVEL1": passes_level = self._matchesLevel1(tag, rawCmdline) elif level == "LEVEL2": passes_level = self._matchesLevel2(tag, rawCmdline) passes_requested_formats = True if tag in {"ENC", "DEC"} and requested_formats: Loading Loading @@ -1842,8 +1872,9 @@ if __name__ == "__main__": metavar="TOKEN", help=( "Select which tests to run. Accepts any combination of the following tokens: " "(1) Level: LEVEL1, LEVEL3 (default: LEVEL3 = no restrictions). " "(1) Level: LEVEL1, LEVEL2, LEVEL3 (default: LEVEL3 = no bitrate restrictions). " "LEVEL1 restricts ENC to <=80 kbps and DEC to EXT<=80 kbps, MONO and STEREO outputs. " "LEVEL2 uses the same rules as LEVEL1 but with <=192 kbps for ENC and DEC EXT. " "(2) Tag: ENC, DEC, REND, ISAR, ISAR_ENC — selects specific test groups. " "Prefix with + to add groups and with - to remove groups at any level. " "Under LEVEL1, the default tag baseline is ENC+DEC when no plain tag is provided. " Loading @@ -1851,16 +1882,19 @@ if __name__ == "__main__": "BINAURAL_ROOM_REVERB, 5_1, 7_1, 5_1_4, 5_1_2, 7_1_4, FOA, HOA2, HOA3, EXT. " "Aliases: HOA -> HOA2+HOA3, SBA -> FOA+HOA2+HOA3, MC -> 5_1+7_1+5_1_4+5_1_2+7_1_4. " "Wildcard prefixes are also supported for tags/formats (e.g., ISAR*, BINAURAL*). " "These tokens restrict ENC/DEC tests at any level by output format; under LEVEL1 they apply on top of LEVEL1 constraints. " "These tokens restrict ENC/DEC tests at any level by output format; under LEVEL1/LEVEL2 they apply on top of level constraints. " "(4) Substring: any other plain token is matched case-insensitively against the test " "command line. Multiple plain tokens are combined with logical AND. " "Prefix a token with + to add matching tests to the selection, " "or with - to remove matching tests from the selection. " "Examples: " "'--filter LEVEL1' — run LEVEL1 ENC+DEC; " "'--filter LEVEL2' — run LEVEL2 ENC+DEC; " "'--filter LEVEL1 DEC MONO' — run only MONO tests from the LEVEL1 DEC set; " "'--filter +REND -ISAR' — at default LEVEL3, add all REND tests and remove all ISAR tests; " "'--filter LEVEL2 DEC MONO' — run only MONO tests from the LEVEL2 DEC set; " "'--filter +REND -ISAR' — at default LEVEL3 baseline (ENC+DEC), add all REND tests and remove all ISAR tests; " "'--filter LEVEL1 +REND' — run LEVEL1 ENC+DEC and also all REND tests; " "'--filter LEVEL2 +REND' — run LEVEL2 ENC+DEC and also all REND tests; " "'--filter DEC HOA' — run DEC tests with HOA2/HOA3 outputs; " "'--filter DEC SBA' — run DEC tests with FOA/HOA2/HOA3 outputs; " "'--filter DEC MC' — run DEC tests with multichannel outputs (5_1, 7_1, 5_1_4, 5_1_2, 7_1_4); " Loading Loading @@ -1923,7 +1957,7 @@ if __name__ == "__main__": unescaped_filter.append(tok) args.filter = unescaped_filter # Explicit LEVEL3 means no restrictions: keep default full tag set. # Explicit LEVEL3 means default level behavior (same as no explicit level token). if args.filter and len(args.filter) == 1 and args.filter[0].upper() == "LEVEL3": args.filter = None Loading Loading @@ -2031,8 +2065,6 @@ if __name__ == "__main__": parser.error("Multiple filter levels specified. Use only one of LEVEL1, LEVEL2, LEVEL3.") filter_level = level_tokens[0] if level_tokens else "LEVEL3" if filter_level == "LEVEL2": parser.error("--filter LEVEL2 is currently unsupported.") # Preserve order while removing duplicates. tag_tokens = list(dict.fromkeys(tag_tokens)) Loading @@ -2048,8 +2080,8 @@ if __name__ == "__main__": if "ISAR" in tag_remove_tokens and "ISAR_ENC" not in tag_remove_tokens: tag_remove_tokens.append("ISAR_ENC") if filter_level == "LEVEL1": # LEVEL1 baseline: ENC + DEC; REND/ISAR only if explicitly requested. # All levels share the same default tag baseline: ENC + DEC. # REND/ISAR/ISAR_ENC are optional and must be explicitly selected or added. # Plain tag_tokens restrict the baseline; +tag_tokens add beyond it. if tag_tokens: selected_tag_set = set(tag_tokens) Loading @@ -2060,16 +2092,6 @@ if __name__ == "__main__": for tag in tag_remove_tokens: selected_tag_set.discard(tag) testTags = [tag for tag in IVAS_Bins.keys() if tag in selected_tag_set] else: if tag_tokens: selected_tag_set = set(tag_tokens) else: selected_tag_set = set(IVAS_Bins.keys()) for tag in tag_add_tokens: selected_tag_set.add(tag) for tag in tag_remove_tokens: selected_tag_set.discard(tag) testTags = [tag for tag in IVAS_Bins.keys() if tag in selected_tag_set] args.filter_display = raw_filter if raw_filter else None args.filter_restrictive_terms = restrictive_terms Loading Loading
scripts/ivas_conformance/README.md +40 −6 Original line number Diff line number Diff line Loading @@ -224,12 +224,12 @@ Summary of results: ## Executing specific tests only Use `--filter` to control test levels, tags and optional text/out_format filtering. Use `--filter` to control test levels, tags or output format filtering. - Levels: - `LEVEL3` (default): no level-based restrictions. - `LEVEL3` (default): no bitrate restrictions. - `LEVEL1`: applies the restrictions below. - `LEVEL2`: currently unsupported; script exits with an error. - `LEVEL2`: same rules as `LEVEL1`, but with bitrate cap increased from 80 kbps to 192 kbps. - Tag selection (case-insensitive): `ENC`, `DEC`, `REND`, `ISAR`, `ISAR_ENC`. - Command substring filtering is case-insensitive. - Plain substring terms are restrictive and combined using logical AND. Loading @@ -239,7 +239,11 @@ Use `--filter` to control test levels, tags and optional text/out_format filteri - `-TERM` removes matching tests from the selection. - Example: `--filter DEC -JBM` runs decoder tests except those containing `JBM`. ### LEVEL1 behavior ### LEVEL1, LEVEL2 and LEVEL3 behavior For all levels, the default tag baseline is `ENC` + `DEC`. `REND`, `ISAR`, and `ISAR_ENC` are optional and are only included if explicitly selected as plain tags or added via `+REND` and/or `+ISAR`. When `--filter LEVEL1` is specified, the following default tests are run: Loading @@ -260,14 +264,38 @@ When `--filter LEVEL1` is specified, the following default tests are run: - Add `+REND` and/or `+ISAR` in `--filter` to include them. - If `+ISAR` is provided, both `ISAR` and `ISAR_ENC` test groups are run. When `--filter LEVEL2` is specified, all selection rules above remain the same, except bitrate caps are relaxed to 192 kbps: - Encoder (`ENC`) tests: only tests with bitrate up to 192 kbps (inclusive). - Decoder (`DEC`) tests: - `EXT` output format: only bitrate up to 192 kbps (inclusive). - `MONO` output format: all bitrates. - `STEREO` output format: all bitrates. When `--filter LEVEL3` is specified, all selection rules above remain the same, with no bitrate caps: - Encoder (`ENC`) tests: all bitrates. - Decoder (`DEC`) tests: - `EXT` output format: all bitrates. - `MONO` output format: all bitrates. - `STEREO` output format: all bitrates. Examples (non-BE): - Default behavior (same as LEVEL3): run all tags/tests - Default behavior (same as LEVEL3 baseline): run only ENC and DEC test groups ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse ``` - LEVEL3 plus renderer and ISAR test groups ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL3 +REND +ISAR ``` - LEVEL1 baseline (ENC+DEC with LEVEL1 restrictions) ```shell Loading Loading @@ -298,8 +326,14 @@ Examples (non-BE): PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL1 DEC JBM +BINAURAL ``` - Unsupported LEVEL2 example (will fail) - LEVEL2 baseline (ENC+DEC with LEVEL2 restrictions) ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL2 ``` - LEVEL2 plus renderer and ISAR test groups ```shell PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter LEVEL2 +REND +ISAR ```
scripts/ivas_conformance/runConformance.py +55 −33 Original line number Diff line number Diff line Loading @@ -1015,13 +1015,19 @@ class MLDConformance: values.append(float(match.replace("_", "."))) return values def _isBitrateAtMost80(self, rawCmdline: str) -> bool: """Check if all bitrates in command line are <= 80 kbps. def _isBitrateAtMost(self, rawCmdline: str, max_kbps: float) -> bool: """Check if all bitrates in command line are <= max_kbps. For bitrate switching tests (e.g., 'from_32_kbps_to_96_kbps'), this checks that the upper (target) bitrate does not exceed 80 kbps.""" that the upper (target) bitrate does not exceed max_kbps.""" values = self._extractKbpsValues(rawCmdline) return bool(values) and max(values) <= 80.0 return bool(values) and max(values) <= float(max_kbps) def _isBitrateAtMost80(self, rawCmdline: str) -> bool: return self._isBitrateAtMost(rawCmdline, 80.0) def _isBitrateAtMost192(self, rawCmdline: str) -> bool: return self._isBitrateAtMost(rawCmdline, 192.0) def _outputFormatsInCommand(self, rawCmdline: str) -> set[str]: text = rawCmdline.upper() Loading Loading @@ -1077,6 +1083,28 @@ class MLDConformance: # For REND/ISAR/ISAR_ENC under LEVEL1, tag-level inclusion is decided at testTags parsing. return True def _matchesLevel2(self, tag: str, rawCmdline: str) -> bool: if tag == "ENC": return self._isBitrateAtMost192(rawCmdline) if tag == "DEC": formats = self._outputFormatsInCommand(rawCmdline) requested_formats = set(getattr(self.args, "filter_decoder_formats", [])) ext_ok = "EXT" in formats and self._isBitrateAtMost192(rawCmdline) mono_ok = "MONO" in formats stereo_ok = "STEREO" in formats default_level2_dec_ok = ext_ok or mono_ok or stereo_ok if requested_formats: # Plain decoder format tokens are restrictive under LEVEL2. return default_level2_dec_ok and bool(formats.intersection(requested_formats)) return default_level2_dec_ok # For REND/ISAR/ISAR_ENC under LEVEL2, tag-level inclusion is decided at testTags parsing. return True def _testPassesFilter(self, tag: str, rawCmdline: str) -> bool: level = getattr(self.args, "filter_level", "LEVEL3") restrictive_terms = getattr(self.args, "filter_restrictive_terms", []) Loading @@ -1091,6 +1119,8 @@ class MLDConformance: passes_level = True if level == "LEVEL1": passes_level = self._matchesLevel1(tag, rawCmdline) elif level == "LEVEL2": passes_level = self._matchesLevel2(tag, rawCmdline) passes_requested_formats = True if tag in {"ENC", "DEC"} and requested_formats: Loading Loading @@ -1842,8 +1872,9 @@ if __name__ == "__main__": metavar="TOKEN", help=( "Select which tests to run. Accepts any combination of the following tokens: " "(1) Level: LEVEL1, LEVEL3 (default: LEVEL3 = no restrictions). " "(1) Level: LEVEL1, LEVEL2, LEVEL3 (default: LEVEL3 = no bitrate restrictions). " "LEVEL1 restricts ENC to <=80 kbps and DEC to EXT<=80 kbps, MONO and STEREO outputs. " "LEVEL2 uses the same rules as LEVEL1 but with <=192 kbps for ENC and DEC EXT. " "(2) Tag: ENC, DEC, REND, ISAR, ISAR_ENC — selects specific test groups. " "Prefix with + to add groups and with - to remove groups at any level. " "Under LEVEL1, the default tag baseline is ENC+DEC when no plain tag is provided. " Loading @@ -1851,16 +1882,19 @@ if __name__ == "__main__": "BINAURAL_ROOM_REVERB, 5_1, 7_1, 5_1_4, 5_1_2, 7_1_4, FOA, HOA2, HOA3, EXT. " "Aliases: HOA -> HOA2+HOA3, SBA -> FOA+HOA2+HOA3, MC -> 5_1+7_1+5_1_4+5_1_2+7_1_4. " "Wildcard prefixes are also supported for tags/formats (e.g., ISAR*, BINAURAL*). " "These tokens restrict ENC/DEC tests at any level by output format; under LEVEL1 they apply on top of LEVEL1 constraints. " "These tokens restrict ENC/DEC tests at any level by output format; under LEVEL1/LEVEL2 they apply on top of level constraints. " "(4) Substring: any other plain token is matched case-insensitively against the test " "command line. Multiple plain tokens are combined with logical AND. " "Prefix a token with + to add matching tests to the selection, " "or with - to remove matching tests from the selection. " "Examples: " "'--filter LEVEL1' — run LEVEL1 ENC+DEC; " "'--filter LEVEL2' — run LEVEL2 ENC+DEC; " "'--filter LEVEL1 DEC MONO' — run only MONO tests from the LEVEL1 DEC set; " "'--filter +REND -ISAR' — at default LEVEL3, add all REND tests and remove all ISAR tests; " "'--filter LEVEL2 DEC MONO' — run only MONO tests from the LEVEL2 DEC set; " "'--filter +REND -ISAR' — at default LEVEL3 baseline (ENC+DEC), add all REND tests and remove all ISAR tests; " "'--filter LEVEL1 +REND' — run LEVEL1 ENC+DEC and also all REND tests; " "'--filter LEVEL2 +REND' — run LEVEL2 ENC+DEC and also all REND tests; " "'--filter DEC HOA' — run DEC tests with HOA2/HOA3 outputs; " "'--filter DEC SBA' — run DEC tests with FOA/HOA2/HOA3 outputs; " "'--filter DEC MC' — run DEC tests with multichannel outputs (5_1, 7_1, 5_1_4, 5_1_2, 7_1_4); " Loading Loading @@ -1923,7 +1957,7 @@ if __name__ == "__main__": unescaped_filter.append(tok) args.filter = unescaped_filter # Explicit LEVEL3 means no restrictions: keep default full tag set. # Explicit LEVEL3 means default level behavior (same as no explicit level token). if args.filter and len(args.filter) == 1 and args.filter[0].upper() == "LEVEL3": args.filter = None Loading Loading @@ -2031,8 +2065,6 @@ if __name__ == "__main__": parser.error("Multiple filter levels specified. Use only one of LEVEL1, LEVEL2, LEVEL3.") filter_level = level_tokens[0] if level_tokens else "LEVEL3" if filter_level == "LEVEL2": parser.error("--filter LEVEL2 is currently unsupported.") # Preserve order while removing duplicates. tag_tokens = list(dict.fromkeys(tag_tokens)) Loading @@ -2048,8 +2080,8 @@ if __name__ == "__main__": if "ISAR" in tag_remove_tokens and "ISAR_ENC" not in tag_remove_tokens: tag_remove_tokens.append("ISAR_ENC") if filter_level == "LEVEL1": # LEVEL1 baseline: ENC + DEC; REND/ISAR only if explicitly requested. # All levels share the same default tag baseline: ENC + DEC. # REND/ISAR/ISAR_ENC are optional and must be explicitly selected or added. # Plain tag_tokens restrict the baseline; +tag_tokens add beyond it. if tag_tokens: selected_tag_set = set(tag_tokens) Loading @@ -2060,16 +2092,6 @@ if __name__ == "__main__": for tag in tag_remove_tokens: selected_tag_set.discard(tag) testTags = [tag for tag in IVAS_Bins.keys() if tag in selected_tag_set] else: if tag_tokens: selected_tag_set = set(tag_tokens) else: selected_tag_set = set(IVAS_Bins.keys()) for tag in tag_add_tokens: selected_tag_set.add(tag) for tag in tag_remove_tokens: selected_tag_set.discard(tag) testTags = [tag for tag in IVAS_Bins.keys() if tag in selected_tag_set] args.filter_display = raw_filter if raw_filter else None args.filter_restrictive_terms = restrictive_terms Loading