Commit 5404bc76 authored by Vladimir Malenovsky's avatar Vladimir Malenovsky
Browse files

add support for LEVEL1 and LEVEL3

parent 60781006

scripts/ivas_conformance/README.md

+33 −13
Original line number Diff line number Diff line
@@ -247,34 +247,54 @@ MLD Corridor passed for ISAR with max MLD diff of 0.0 


## Executing specific tests only



Use `--filter` to run or analyze a subset of tests.



- Tag selection is supported with comma-separated tags: `ENC`, `DEC`, `REND`, `ISAR`, `ISAR_ENC`.

- Command substring filtering is also supported.

- Matching is case-insensitive.

Use `--filter` to control test levels, tags and optional text/format filtering.



- Levels:

  - `LEVEL3` (default): no level-based restrictions.

  - `LEVEL1`: applies the restrictions below.

  - `LEVEL2`: currently unsupported; script exits with an error.

- Tag selection (case-insensitive): `ENC`, `DEC`, `REND`, `ISAR`, `ISAR_ENC`.

- Command substring filter is case-insensitive.



### LEVEL1 behavior



- Encoder (`ENC`) tests: only tests with bitrate up to 80 kbps (inclusive).

- Decoder (`DEC`) tests by default:

  - `EXT` output format: only bitrate up to 80 kbps (inclusive).

  - `MONO` output format: all bitrates.

  - `STEREO` output format: all bitrates.

- Renderer and ISAR tests are not run by default in `LEVEL1`.

  - Add `REND` and/or `ISAR` in `--filter` to include them.

  - If `ISAR` is provided, both `ISAR` and `ISAR_ENC` test groups are run.



Examples:



- Run all renderer tests only:

- Default behavior (same as LEVEL3): run all tags/tests



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR

  ```



- LEVEL1 baseline (ENC+DEC with LEVEL1 restrictions)



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --filter rend

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --filter LEVEL1

  ```



- Analyze BE conformance for encoder tests only:

- LEVEL1 plus renderer and ISAR test groups



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --be-test --filter enc

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --filter LEVEL1 REND ISAR

  ```



- Analyze non-BE conformance for DEC and REND tests that include `voip` in the command:

- LEVEL1 with additional case-insensitive command substring filtering



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --analyse --filter dec,rend voip

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --filter LEVEL1 DEC voip

  ```



- Run only ISAR encoder tests:

- Unsupported LEVEL2 example (will fail)



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --filter isar_enc

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR --filter LEVEL2

  ```

scripts/ivas_conformance/runConformance.py

+118 −37
Original line number Diff line number Diff line
@@ -359,7 +359,7 @@ class MLDConformance: 


    def setupDUT(self):

        self.cut_build_path = self.args.cut_build_path

        self.filter = self.args.filter

        self.filter = getattr(self.args, "filter_display", self.args.filter)

        exe_platform = platform.system()

        if exe_platform == "Windows":

            exe_platform = "Win64"
@@ -931,16 +931,75 @@ class MLDConformance: 
    def analyseOneCommandFromTuple(self, args):

        return self.analyseOneCommand(*args)



    def _extractKbpsValues(self, rawCmdline: str) -> list[float]:

        values = []

        for match in re.findall(r"(\d+(?:_\d+)?)_kbps", rawCmdline.lower()):

            values.append(float(match.replace("_", ".")))

        return values



    def _isBitrateAtMost80(self, rawCmdline: str) -> bool:

        values = self._extractKbpsValues(rawCmdline)

        return bool(values) and max(values) <= 80.0



    def _decoderFormatsInCommand(self, rawCmdline: str) -> set[str]:

        text = rawCmdline.upper()

        formats = set()



        if "MONO" in text:

            formats.add("MONO")

        if "STEREO" in text:

            formats.add("STEREO")



        # Avoid matching words like e.g. EXTENDED  - do keying based on _OUT output only  

        if (

            "_EXT_OUT" in text

            or "_EXTERNAL_OUT" in text

            or " EXT_OUT" in text

            or " EXTERNAL_OUT" in text

        ):

            formats.add("EXT")



        return formats



    def _matchesSubstringFilter(self, rawCmdline: str) -> bool:

        filter_substring = getattr(self.args, "filter_substring", None)

        if not filter_substring:

            return True

        return filter_substring.lower() in rawCmdline.lower()



    def _matchesLevel1(self, tag: str, rawCmdline: str) -> bool:

        if tag == "ENC":

            return self._isBitrateAtMost80(rawCmdline)



        if tag == "DEC":

            formats = self._decoderFormatsInCommand(rawCmdline)

            ext_ok = "EXT" in formats and self._isBitrateAtMost80(rawCmdline)

            mono_ok = "MONO" in formats

            stereo_ok = "STEREO" in formats

            return ext_ok or mono_ok or stereo_ok



        # For REND/ISAR/ISAR_ENC under LEVEL1, tag-level inclusion is decided at testTags parsing.

        return True



    def _testPassesFilter(self, tag: str, rawCmdline: str) -> bool:

        level = getattr(self.args, "filter_level", "LEVEL3")



        if level == "LEVEL1" and not self._matchesLevel1(tag, rawCmdline):

            return False



        return self._matchesSubstringFilter(rawCmdline)



    def getSelectedTestsForTag(self, tag: str) -> list[str]:

        selected = []

        for pyTestsTag in self.TestDesc[tag].keys():

            rawCmdline = self.TestDesc[tag][pyTestsTag].rawCmdline

            if self._testPassesFilter(tag, rawCmdline):

                selected.append(pyTestsTag)

        return selected



    def runTag(self, tag: str) -> bool:

        failed_before = self.getFailedCommandCount()

        selectedTests = list()

        if self.filter:

            filter_text = self.filter.lower()

            for pyTestsTag in self.TestDesc[tag].keys():

                if filter_text in self.TestDesc[tag][pyTestsTag].rawCmdline.lower():

                    selectedTests.append(pyTestsTag)

        else:

            selectedTests = list(self.TestDesc[tag].keys())

        selectedTests = self.getSelectedTestsForTag(tag)



        self.totalTests = len(selectedTests)

        print(
@@ -1022,14 +1081,7 @@ class MLDConformance: 
            with open(self.sampleStats[tag], "w") as f:

                f.write(f"PYTESTTAG, MAXDIFF, RMSdB, BEFRAMES_PERCENT, MAX_MLD\n")



        selectedTests = []

        if self.filter:

            filter_text = self.filter.lower()

            for pyTestsTag in self.TestDesc[tag].keys():

                if filter_text in self.TestDesc[tag][pyTestsTag].rawCmdline.lower():

                    selectedTests.append(pyTestsTag)

        else:

            selectedTests = list(self.TestDesc[tag].keys())

        selectedTests = self.getSelectedTestsForTag(tag)



        self.totalTests = len(selectedTests)

        print(
@@ -1637,8 +1689,9 @@ if __name__ == "__main__": 
        type=str,

        default=None,

        help=(

            "Filter tests. Supports tags ENC DEC REND ISAR ISAR_ENC (comma/space-separated) "

            "and case-insensitive command substring filtering. Examples: '--filter rend' or '--filter REND binaural'."

            "Filter tests. Supports levels LEVEL1 LEVEL2 LEVEL3 and tags ENC DEC REND ISAR ISAR_ENC "

            "(comma/space-separated), and case-insensitive command substring filtering. "

            "LEVEL3 is default. LEVEL2 is currently unsupported."

        ),

    )

    parser.add_argument(
@@ -1690,36 +1743,64 @@ if __name__ == "__main__": 
    if args.cut_build_path:

        validate_build_binaries(parser, args.cut_build_path, "CUT")



    conformance = MLDConformance(args)

    conformance.accumulateCommands()



    if args.regenerate_enc_refs:

        conformance.runReferenceGeneration(encTag="ISAR_ENC")

        conformance.runReferenceGeneration(encTag="ENC")

        sys.exit(0)



    # If --regenerate-mld-ref is set, treat as --analyse

    if args.regenerate_mld_ref:

        args.analyse = True



    # Parse --filter into optional tag selection + optional command substring filter.

    # Parse --filter into level + optional tag selection + optional format/substring filters.

    raw_filter = args.filter or ""

    filter_tokens = [tok for tok in re.split(r"[\s,]+", raw_filter.strip()) if tok]



    valid_tags = set(IVAS_Bins.keys())

    valid_levels = {"LEVEL1", "LEVEL2", "LEVEL3"}



    level_tokens = []

    tag_tokens = []

    non_tag_tokens = []

    non_special_tokens = []



    for tok in filter_tokens:

        upper_tok = tok.upper()

        if upper_tok in valid_tags:

        if upper_tok in valid_levels:

            level_tokens.append(upper_tok)

        elif upper_tok in valid_tags:

            tag_tokens.append(upper_tok)

        else:

            non_tag_tokens.append(tok)

            non_special_tokens.append(tok)



    if len(set(level_tokens)) > 1:

        parser.error("Multiple filter levels specified. Use only one of LEVEL1, LEVEL2, LEVEL3.")



    filter_level = level_tokens[0] if level_tokens else "LEVEL3"

    if filter_level == "LEVEL2":

        parser.error("--filter LEVEL2 is currently unsupported.")



    # Preserve order while removing duplicates.

    tag_tokens = list(dict.fromkeys(tag_tokens))



    # If ISAR is requested, run both ISAR and ISAR_ENC.

    if "ISAR" in tag_tokens and "ISAR_ENC" not in tag_tokens:

        tag_tokens.append("ISAR_ENC")



    if filter_level == "LEVEL1":

        # LEVEL1 baseline: ENC + DEC; REND/ISAR only if explicitly requested.

        selected_tag_set = {"ENC", "DEC"}

        for tag in tag_tokens:

            selected_tag_set.add(tag)

        testTags = [tag for tag in IVAS_Bins.keys() if tag in selected_tag_set]

    else:

        testTags = tag_tokens if tag_tokens else list(IVAS_Bins.keys())

    args.filter = " ".join(non_tag_tokens) if non_tag_tokens else None



    args.filter_display = raw_filter if raw_filter else None

    args.filter_substring = " ".join(non_special_tokens) if non_special_tokens else None

    args.filter_level = filter_level



    conformance = MLDConformance(args)

    conformance.accumulateCommands()



    if args.regenerate_enc_refs:

        conformance.runReferenceGeneration(encTag="ISAR_ENC")

        conformance.runReferenceGeneration(encTag="ENC")

        sys.exit(0)



    # If --regenerate-mld-ref is set, treat as --analyse

    if args.regenerate_mld_ref:

        args.analyse = True



    tag_results = {}

    for tag in testTags: