Loading scripts/ivas_conformance/runConformance.py +62 −13 Original line number Diff line number Diff line Loading @@ -255,6 +255,29 @@ IVAS_Bins = { "ISAR": "ISAR_post_rend", } DECODER_OUTPUT_FORMATS = { "MONO", "STEREO", "BINAURAL", "BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB", "5_1", "7_1", "5_1_4", "5_1_2", "7_1_4", "FOA", "HOA2", "HOA3", "EXT", } DECODER_OUTPUT_FORMAT_ALIASES = { "HOA": {"HOA2", "HOA3"}, "SBA": {"FOA", "HOA2", "HOA3"}, "MC": {"5_1", "7_1", "5_1_4", "5_1_2", "7_1_4"}, } def validate_build_binaries(parser, build_path: str, build_label: str) -> None: """Validate that a build path exists and contains all IVAS binaries.""" Loading Loading @@ -1000,16 +1023,20 @@ class MLDConformance: values = self._extractKbpsValues(rawCmdline) return bool(values) and max(values) <= 80.0 def _decoderFormatsInCommand(self, rawCmdline: str) -> set[str]: def _outputFormatsInCommand(self, rawCmdline: str) -> set[str]: text = rawCmdline.upper() formats = set() if "MONO" in text: formats.add("MONO") if "STEREO" in text: formats.add("STEREO") # Match format token between '_out_' and '_out' to avoid matching input-format words. # Example: '..._48kHz_out_MONO_out_...' -> captures MONO only. for match in re.finditer(r"_OUT_([A-Z0-9_]+?)_OUT(?:\b|_)", text): fmt = match.group(1) if fmt == "EXTERNAL": fmt = "EXT" if fmt in DECODER_OUTPUT_FORMATS: formats.add(fmt) # Avoid matching words like e.g. EXTENDED - do keying based on _OUT output only # Avoid matching words like EXTENDED; key on EXT_OUT style output naming. if ( "_EXT_OUT" in text or "_EXTERNAL_OUT" in text Loading @@ -1033,7 +1060,7 @@ class MLDConformance: return self._isBitrateAtMost80(rawCmdline) if tag == "DEC": formats = self._decoderFormatsInCommand(rawCmdline) formats = self._outputFormatsInCommand(rawCmdline) requested_formats = set(getattr(self.args, "filter_decoder_formats", [])) ext_ok = "EXT" in formats and self._isBitrateAtMost80(rawCmdline) Loading @@ -1055,6 +1082,7 @@ class MLDConformance: restrictive_terms = getattr(self.args, "filter_restrictive_terms", []) additive_terms = getattr(self.args, "filter_add_terms", []) subtractive_terms = getattr(self.args, "filter_remove_terms", []) requested_formats = set(getattr(self.args, "filter_decoder_formats", [])) # '-' terms always remove tests from the final selection. if subtractive_terms and self._matchesAnyTerm(rawCmdline, subtractive_terms): Loading @@ -1064,8 +1092,13 @@ class MLDConformance: if level == "LEVEL1": passes_level = self._matchesLevel1(tag, rawCmdline) passes_requested_formats = True if tag in {"ENC", "DEC"} and requested_formats: cmd_formats = self._outputFormatsInCommand(rawCmdline) passes_requested_formats = bool(cmd_formats.intersection(requested_formats)) passes_restrictive_terms = self._matchesAllTerms(rawCmdline, restrictive_terms) base_selected = passes_level and passes_restrictive_terms base_selected = passes_level and passes_restrictive_terms and passes_requested_formats if base_selected: return True Loading Loading @@ -1574,6 +1607,12 @@ class MLDConformance: keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: if os.path.exists(self.BEcsv[tag]): # For filtered runs it is valid to have zero selected tests for a tag. # In that case the BE csv contains only the header; skip loadtxt to avoid warnings. with open(self.BEcsv[tag], "r") as f: non_empty_lines = [line for line in f if line.strip()] if len(non_empty_lines) <= 1: continue BEresult = np.loadtxt( self.BEcsv[tag], delimiter=",", Loading Loading @@ -1793,9 +1832,12 @@ if __name__ == "__main__": "(1) Level: LEVEL1, LEVEL3 (default: LEVEL3 = no restrictions). " "LEVEL1 restricts ENC to <=80 kbps and DEC to EXT<=80 kbps, MONO and STEREO outputs. " "(2) Tag: ENC, DEC, REND, ISAR, ISAR_ENC — selects specific test groups. " "Under LEVEL1, +REND or +ISAR adds those groups to the default ENC+DEC set. " "(3) Decoder format (LEVEL1 DEC only): EXT, MONO, STEREO — further restricts which " "output formats are included within the already-restricted LEVEL1 DEC set. " "Prefix with + to add groups and with - to remove groups at any level. " "Under LEVEL1, the default tag baseline is ENC+DEC when no plain tag is provided. " "(3) Output format (ENC/DEC): MONO, STEREO, BINAURAL, BINAURAL_ROOM_IR, " "BINAURAL_ROOM_REVERB, 5_1, 7_1, 5_1_4, 5_1_2, 7_1_4, FOA, HOA2, HOA3, EXT. " "Aliases: HOA -> HOA2+HOA3, SBA -> FOA+HOA2+HOA3, MC -> 5_1+7_1+5_1_4+5_1_2+7_1_4. " "These tokens restrict ENC/DEC tests at any level by output format; under LEVEL1 they apply on top of LEVEL1 constraints. " "(4) Substring: any other plain token is matched case-insensitively against the test " "command line. Multiple plain tokens are combined with logical AND. " "Prefix a token with + to add matching tests to the selection, " Loading @@ -1803,7 +1845,11 @@ if __name__ == "__main__": "Examples: " "'--filter LEVEL1' — run LEVEL1 ENC+DEC; " "'--filter LEVEL1 DEC MONO' — run only MONO tests from the LEVEL1 DEC set; " "'--filter +REND -ISAR' — at default LEVEL3, add all REND tests and remove all ISAR tests; " "'--filter LEVEL1 +REND' — run LEVEL1 ENC+DEC and also all REND tests; " "'--filter DEC HOA' — run DEC tests with HOA2/HOA3 outputs; " "'--filter DEC SBA' — run DEC tests with FOA/HOA2/HOA3 outputs; " "'--filter DEC MC' — run DEC tests with multichannel outputs (5_1, 7_1, 5_1_4, 5_1_2, 7_1_4); " "'--filter DEC JBM' — run DEC tests containing 'JBM'; " "'--filter DEC +BINAURAL' — run all DEC tests plus those containing 'BINAURAL'; " "'--filter DEC -voip' — run all DEC tests except those containing 'voip'." Loading Loading @@ -1877,7 +1923,9 @@ if __name__ == "__main__": valid_tags = set(IVAS_Bins.keys()) valid_levels = {"LEVEL1", "LEVEL2", "LEVEL3"} valid_decoder_formats = {"EXT", "MONO", "STEREO"} valid_decoder_formats = set(DECODER_OUTPUT_FORMATS).union( DECODER_OUTPUT_FORMAT_ALIASES.keys() ) level_tokens = [] tag_tokens = [] Loading Loading @@ -1909,12 +1957,13 @@ if __name__ == "__main__": else: tag_tokens.append(upper_tok) elif upper_tok in valid_decoder_formats: expanded_formats = DECODER_OUTPUT_FORMAT_ALIASES.get(upper_tok, {upper_tok}) if sign == "+": additive_terms.append(base_tok) elif sign == "-": subtractive_terms.append(base_tok) else: decoder_format_tokens.append(upper_tok) decoder_format_tokens.extend(sorted(expanded_formats)) else: if sign == "+": additive_terms.append(base_tok) Loading Loading
scripts/ivas_conformance/runConformance.py +62 −13 Original line number Diff line number Diff line Loading @@ -255,6 +255,29 @@ IVAS_Bins = { "ISAR": "ISAR_post_rend", } DECODER_OUTPUT_FORMATS = { "MONO", "STEREO", "BINAURAL", "BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB", "5_1", "7_1", "5_1_4", "5_1_2", "7_1_4", "FOA", "HOA2", "HOA3", "EXT", } DECODER_OUTPUT_FORMAT_ALIASES = { "HOA": {"HOA2", "HOA3"}, "SBA": {"FOA", "HOA2", "HOA3"}, "MC": {"5_1", "7_1", "5_1_4", "5_1_2", "7_1_4"}, } def validate_build_binaries(parser, build_path: str, build_label: str) -> None: """Validate that a build path exists and contains all IVAS binaries.""" Loading Loading @@ -1000,16 +1023,20 @@ class MLDConformance: values = self._extractKbpsValues(rawCmdline) return bool(values) and max(values) <= 80.0 def _decoderFormatsInCommand(self, rawCmdline: str) -> set[str]: def _outputFormatsInCommand(self, rawCmdline: str) -> set[str]: text = rawCmdline.upper() formats = set() if "MONO" in text: formats.add("MONO") if "STEREO" in text: formats.add("STEREO") # Match format token between '_out_' and '_out' to avoid matching input-format words. # Example: '..._48kHz_out_MONO_out_...' -> captures MONO only. for match in re.finditer(r"_OUT_([A-Z0-9_]+?)_OUT(?:\b|_)", text): fmt = match.group(1) if fmt == "EXTERNAL": fmt = "EXT" if fmt in DECODER_OUTPUT_FORMATS: formats.add(fmt) # Avoid matching words like e.g. EXTENDED - do keying based on _OUT output only # Avoid matching words like EXTENDED; key on EXT_OUT style output naming. if ( "_EXT_OUT" in text or "_EXTERNAL_OUT" in text Loading @@ -1033,7 +1060,7 @@ class MLDConformance: return self._isBitrateAtMost80(rawCmdline) if tag == "DEC": formats = self._decoderFormatsInCommand(rawCmdline) formats = self._outputFormatsInCommand(rawCmdline) requested_formats = set(getattr(self.args, "filter_decoder_formats", [])) ext_ok = "EXT" in formats and self._isBitrateAtMost80(rawCmdline) Loading @@ -1055,6 +1082,7 @@ class MLDConformance: restrictive_terms = getattr(self.args, "filter_restrictive_terms", []) additive_terms = getattr(self.args, "filter_add_terms", []) subtractive_terms = getattr(self.args, "filter_remove_terms", []) requested_formats = set(getattr(self.args, "filter_decoder_formats", [])) # '-' terms always remove tests from the final selection. if subtractive_terms and self._matchesAnyTerm(rawCmdline, subtractive_terms): Loading @@ -1064,8 +1092,13 @@ class MLDConformance: if level == "LEVEL1": passes_level = self._matchesLevel1(tag, rawCmdline) passes_requested_formats = True if tag in {"ENC", "DEC"} and requested_formats: cmd_formats = self._outputFormatsInCommand(rawCmdline) passes_requested_formats = bool(cmd_formats.intersection(requested_formats)) passes_restrictive_terms = self._matchesAllTerms(rawCmdline, restrictive_terms) base_selected = passes_level and passes_restrictive_terms base_selected = passes_level and passes_restrictive_terms and passes_requested_formats if base_selected: return True Loading Loading @@ -1574,6 +1607,12 @@ class MLDConformance: keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] for tag in keys: if os.path.exists(self.BEcsv[tag]): # For filtered runs it is valid to have zero selected tests for a tag. # In that case the BE csv contains only the header; skip loadtxt to avoid warnings. with open(self.BEcsv[tag], "r") as f: non_empty_lines = [line for line in f if line.strip()] if len(non_empty_lines) <= 1: continue BEresult = np.loadtxt( self.BEcsv[tag], delimiter=",", Loading Loading @@ -1793,9 +1832,12 @@ if __name__ == "__main__": "(1) Level: LEVEL1, LEVEL3 (default: LEVEL3 = no restrictions). " "LEVEL1 restricts ENC to <=80 kbps and DEC to EXT<=80 kbps, MONO and STEREO outputs. " "(2) Tag: ENC, DEC, REND, ISAR, ISAR_ENC — selects specific test groups. " "Under LEVEL1, +REND or +ISAR adds those groups to the default ENC+DEC set. " "(3) Decoder format (LEVEL1 DEC only): EXT, MONO, STEREO — further restricts which " "output formats are included within the already-restricted LEVEL1 DEC set. " "Prefix with + to add groups and with - to remove groups at any level. " "Under LEVEL1, the default tag baseline is ENC+DEC when no plain tag is provided. " "(3) Output format (ENC/DEC): MONO, STEREO, BINAURAL, BINAURAL_ROOM_IR, " "BINAURAL_ROOM_REVERB, 5_1, 7_1, 5_1_4, 5_1_2, 7_1_4, FOA, HOA2, HOA3, EXT. " "Aliases: HOA -> HOA2+HOA3, SBA -> FOA+HOA2+HOA3, MC -> 5_1+7_1+5_1_4+5_1_2+7_1_4. " "These tokens restrict ENC/DEC tests at any level by output format; under LEVEL1 they apply on top of LEVEL1 constraints. " "(4) Substring: any other plain token is matched case-insensitively against the test " "command line. Multiple plain tokens are combined with logical AND. " "Prefix a token with + to add matching tests to the selection, " Loading @@ -1803,7 +1845,11 @@ if __name__ == "__main__": "Examples: " "'--filter LEVEL1' — run LEVEL1 ENC+DEC; " "'--filter LEVEL1 DEC MONO' — run only MONO tests from the LEVEL1 DEC set; " "'--filter +REND -ISAR' — at default LEVEL3, add all REND tests and remove all ISAR tests; " "'--filter LEVEL1 +REND' — run LEVEL1 ENC+DEC and also all REND tests; " "'--filter DEC HOA' — run DEC tests with HOA2/HOA3 outputs; " "'--filter DEC SBA' — run DEC tests with FOA/HOA2/HOA3 outputs; " "'--filter DEC MC' — run DEC tests with multichannel outputs (5_1, 7_1, 5_1_4, 5_1_2, 7_1_4); " "'--filter DEC JBM' — run DEC tests containing 'JBM'; " "'--filter DEC +BINAURAL' — run all DEC tests plus those containing 'BINAURAL'; " "'--filter DEC -voip' — run all DEC tests except those containing 'voip'." Loading Loading @@ -1877,7 +1923,9 @@ if __name__ == "__main__": valid_tags = set(IVAS_Bins.keys()) valid_levels = {"LEVEL1", "LEVEL2", "LEVEL3"} valid_decoder_formats = {"EXT", "MONO", "STEREO"} valid_decoder_formats = set(DECODER_OUTPUT_FORMATS).union( DECODER_OUTPUT_FORMAT_ALIASES.keys() ) level_tokens = [] tag_tokens = [] Loading Loading @@ -1909,12 +1957,13 @@ if __name__ == "__main__": else: tag_tokens.append(upper_tok) elif upper_tok in valid_decoder_formats: expanded_formats = DECODER_OUTPUT_FORMAT_ALIASES.get(upper_tok, {upper_tok}) if sign == "+": additive_terms.append(base_tok) elif sign == "-": subtractive_terms.append(base_tok) else: decoder_format_tokens.append(upper_tok) decoder_format_tokens.extend(sorted(expanded_formats)) else: if sign == "+": additive_terms.append(base_tok) Loading