Commit 690f2be4 authored by Vladimir Malenovsky's avatar Vladimir Malenovsky
Browse files

add support for adding/removoing tests using + or - symbol

parent f70a9b64
Loading
Loading
Loading
Loading
+28 −9
Original line number Diff line number Diff line
@@ -247,28 +247,41 @@ MLD Corridor passed for ISAR with max MLD diff of 0.0

## Executing specific tests only

Use `--filter` to control test levels, tags and optional text/format filtering.
Use `--filter` to control test levels, tags and optional text/out_format filtering.

- Levels:
  - `LEVEL3` (default): no level-based restrictions.
  - `LEVEL1`: applies the restrictions below.
  - `LEVEL2`: currently unsupported; script exits with an error.
- Tag selection (case-insensitive): `ENC`, `DEC`, `REND`, `ISAR`, `ISAR_ENC`.
- Command substring filter is case-insensitive.
- Command substring filtering is case-insensitive.
- Plain substring terms are restrictive and combined using logical AND.
  - Example: `--filter DEC JBM` runs only decoder tests that contain `JBM`.
- `+TERM` adds matching tests to the selection.
  - Example: `--filter LEVEL1 DEC +JBM` adds all `JBM` decoder tests to the default `LEVEL1 DEC` selection.
- `-TERM` removes matching tests from the selection.
  - Example: `--filter DEC -JBM` runs decoder tests except those containing `JBM`.

### LEVEL1 behavior

When `--filter LEVEL1` is specified, the following default tests are run:

- Encoder (`ENC`) tests: only tests with bitrate up to 80 kbps (inclusive).
- Decoder (`DEC`) tests by default:
- Decoder (`DEC`) tests:
  - `EXT` output format: only bitrate up to 80 kbps (inclusive).
  - `MONO` output format: all bitrates.
  - `STEREO` output format: all bitrates.
- Decoder format tokens are additive to the default LEVEL1 decoder set.
- Any token containing `BINAURAL` matches decoder tests whose command contains `BINAURAL`
  (for example, `BINAURAL`, `BINAURAL_IR`, `BINAURAL_ROOM_IR`, `BINAURAL_REVERB`).

- The default `LEVEL1` tests may be restricted by adding more terms (acting as logical AND).
  - Example: `--filter LEVEL1 DEC MONO` keeps only `MONO` tests from the LEVEL1-eligible DEC set.
  - Example: `--filter LEVEL1 JBM` keeps all LEVEL1-eligible ENC tests but only JBM tests from the LEVEL1-eligible DEC tests.
- `+TERM` adds tests to the final LEVEL1 selection, even if they would otherwise be restricted.
  - Example: `--filter LEVEL1 DEC JBM +BINAURAL` runs only JBM-matching LEVEL1 DEC tests and additionally includes DEC tests containing `BINAURAL` keyword, i.e. `BINAURAL`, `BINAURAL_IR`, `BINAURAL_ROOM_IR`, `BINAURAL_REVERB`.
- `-TERM` removes matching tests from the final LEVEL1 selection (including tests added via `+TERM`).
  - Example: `--filter LEVEL1 DEC +JBM -VOIP` adds JBM-matching DEC tests and then excludes any DEC tests containing the keyword `VOIP`.
- Renderer and ISAR tests are not run by default in `LEVEL1`.
  - Add `REND` and/or `ISAR` in `--filter` to include them.
  - If `ISAR` is provided, both `ISAR` and `ISAR_ENC` test groups are run.
  - Add `+REND` and/or `+ISAR` in `--filter` to include them.
  - If `+ISAR` is provided, both `ISAR` and `ISAR_ENC` test groups are run.

Examples:

@@ -299,7 +312,13 @@ Examples:
- LEVEL1 with additive BINAURAL decoder matching

  ```shell
  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --filter LEVEL1 BINAURAL
  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --filter LEVEL1 +BINAURAL
  ```

- LEVEL1 with restrictive and additive terms together

  ```shell
  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --filter LEVEL1 DEC JBM +BINAURAL
  ```

- Unsupported LEVEL2 example (will fail)
+89 −28
Original line number Diff line number Diff line
@@ -945,8 +945,6 @@ class MLDConformance:
        text = rawCmdline.upper()
        formats = set()

        if "BINAURAL" in text:
            formats.add("BINAURAL")
        if "MONO" in text:
            formats.add("MONO")
        if "STEREO" in text:
@@ -963,11 +961,13 @@ class MLDConformance:

        return formats

    def _matchesSubstringFilter(self, rawCmdline: str) -> bool:
        filter_substring = getattr(self.args, "filter_substring", None)
        if not filter_substring:
            return True
        return filter_substring.lower() in rawCmdline.lower()
    def _matchesAllTerms(self, rawCmdline: str, terms: list[str]) -> bool:
        text = rawCmdline.lower()
        return all(term.lower() in text for term in terms)

    def _matchesAnyTerm(self, rawCmdline: str, terms: list[str]) -> bool:
        text = rawCmdline.lower()
        return any(term.lower() in text for term in terms)

    def _matchesLevel1(self, tag: str, rawCmdline: str) -> bool:
        if tag == "ENC":
@@ -983,12 +983,8 @@ class MLDConformance:
            default_level1_dec_ok = ext_ok or mono_ok or stereo_ok

            if requested_formats:
                requested_match = False
                if "BINAURAL" in requested_formats and "BINAURAL" in rawCmdline.upper():
                    requested_match = True
                elif formats.intersection(requested_formats):
                    requested_match = True
                return default_level1_dec_ok or requested_match
                # Plain decoder format tokens are restrictive under LEVEL1.
                return default_level1_dec_ok and bool(formats.intersection(requested_formats))

            return default_level1_dec_ok

@@ -997,11 +993,29 @@ class MLDConformance:

    def _testPassesFilter(self, tag: str, rawCmdline: str) -> bool:
        level = getattr(self.args, "filter_level", "LEVEL3")
        restrictive_terms = getattr(self.args, "filter_restrictive_terms", [])
        additive_terms = getattr(self.args, "filter_add_terms", [])
        subtractive_terms = getattr(self.args, "filter_remove_terms", [])

        if level == "LEVEL1" and not self._matchesLevel1(tag, rawCmdline):
        # '-' terms always remove tests from the final selection.
        if subtractive_terms and self._matchesAnyTerm(rawCmdline, subtractive_terms):
            return False

        return self._matchesSubstringFilter(rawCmdline)
        passes_level = True
        if level == "LEVEL1":
            passes_level = self._matchesLevel1(tag, rawCmdline)

        passes_restrictive_terms = self._matchesAllTerms(rawCmdline, restrictive_terms)
        base_selected = passes_level and passes_restrictive_terms

        if base_selected:
            return True

        # '+' terms add tests even if they fail restrictive filters.
        if additive_terms and self._matchesAnyTerm(rawCmdline, additive_terms):
            return True

        return False

    def getSelectedTestsForTag(self, tag: str) -> list[str]:
        selected = []
@@ -1704,8 +1718,9 @@ if __name__ == "__main__":
        default=None,
        help=(
            "Filter tests. Supports levels LEVEL1 LEVEL2 LEVEL3 and tags ENC DEC REND ISAR ISAR_ENC "
            "(comma/space-separated), decoder formats EXT MONO STEREO BINAURAL, "
            "and case-insensitive command substring filtering. "
            "(comma/space-separated), decoder formats EXT MONO STEREO, "
            "and case-insensitive command substring filtering. Plain terms are restrictive (AND). "
            "Use +TERM to add matches and -TERM to remove matches. "
            "LEVEL3 is default. LEVEL2 is currently unsupported."
        ),
    )
@@ -1764,25 +1779,51 @@ if __name__ == "__main__":

    valid_tags = set(IVAS_Bins.keys())
    valid_levels = {"LEVEL1", "LEVEL2", "LEVEL3"}
    valid_decoder_formats = {"EXT", "MONO", "STEREO", "BINAURAL"}
    valid_decoder_formats = {"EXT", "MONO", "STEREO"}

    level_tokens = []
    tag_tokens = []
    tag_add_tokens = []
    tag_remove_tokens = []
    decoder_format_tokens = []
    non_special_tokens = []
    restrictive_terms = []
    additive_terms = []
    subtractive_terms = []

    for tok in filter_tokens:
        upper_tok = tok.upper()
        sign = ""
        base_tok = tok
        if len(tok) > 1 and tok[0] in {"+", "-"}:
            sign = tok[0]
            base_tok = tok[1:]

        upper_tok = base_tok.upper()

        if upper_tok in valid_levels:
            if sign:
                parser.error(f"Level token '{tok}' cannot be prefixed with '+' or '-'.")
            level_tokens.append(upper_tok)
        elif upper_tok in valid_tags:
            if sign == "+":
                tag_add_tokens.append(upper_tok)
            elif sign == "-":
                tag_remove_tokens.append(upper_tok)
            else:
                tag_tokens.append(upper_tok)
        elif "BINAURAL" in upper_tok:
            decoder_format_tokens.append("BINAURAL")
        elif upper_tok in valid_decoder_formats:
            if sign == "+":
                additive_terms.append(base_tok)
            elif sign == "-":
                subtractive_terms.append(base_tok)
            else:
                decoder_format_tokens.append(upper_tok)
        else:
            non_special_tokens.append(tok)
            if sign == "+":
                additive_terms.append(base_tok)
            elif sign == "-":
                subtractive_terms.append(base_tok)
            else:
                restrictive_terms.append(base_tok)

    if len(set(level_tokens)) > 1:
        parser.error("Multiple filter levels specified. Use only one of LEVEL1, LEVEL2, LEVEL3.")
@@ -1793,23 +1834,43 @@ if __name__ == "__main__":

    # Preserve order while removing duplicates.
    tag_tokens = list(dict.fromkeys(tag_tokens))
    tag_add_tokens = list(dict.fromkeys(tag_add_tokens))
    tag_remove_tokens = list(dict.fromkeys(tag_remove_tokens))
    decoder_format_tokens = list(dict.fromkeys(decoder_format_tokens))

    # If ISAR is requested, run both ISAR and ISAR_ENC.
    # If ISAR is requested/added/removed, apply same action to ISAR_ENC.
    if "ISAR" in tag_tokens and "ISAR_ENC" not in tag_tokens:
        tag_tokens.append("ISAR_ENC")
    if "ISAR" in tag_add_tokens and "ISAR_ENC" not in tag_add_tokens:
        tag_add_tokens.append("ISAR_ENC")
    if "ISAR" in tag_remove_tokens and "ISAR_ENC" not in tag_remove_tokens:
        tag_remove_tokens.append("ISAR_ENC")

    if filter_level == "LEVEL1":
        # LEVEL1 baseline: ENC + DEC; REND/ISAR only if explicitly requested.
        selected_tag_set = {"ENC", "DEC"}
        for tag in tag_tokens:
            selected_tag_set.add(tag)
        for tag in tag_add_tokens:
            selected_tag_set.add(tag)
        for tag in tag_remove_tokens:
            selected_tag_set.discard(tag)
        testTags = [tag for tag in IVAS_Bins.keys() if tag in selected_tag_set]
    else:
        testTags = tag_tokens if tag_tokens else list(IVAS_Bins.keys())
        if tag_tokens:
            selected_tag_set = set(tag_tokens)
        else:
            selected_tag_set = set(IVAS_Bins.keys())
        for tag in tag_add_tokens:
            selected_tag_set.add(tag)
        for tag in tag_remove_tokens:
            selected_tag_set.discard(tag)
        testTags = [tag for tag in IVAS_Bins.keys() if tag in selected_tag_set]

    args.filter_display = raw_filter if raw_filter else None
    args.filter_substring = " ".join(non_special_tokens) if non_special_tokens else None
    args.filter_restrictive_terms = restrictive_terms
    args.filter_add_terms = additive_terms
    args.filter_remove_terms = subtractive_terms
    args.filter_decoder_formats = decoder_format_tokens
    args.filter_level = filter_level