Commit 1d249385 authored by Vladimir Malenovsky's avatar Vladimir Malenovsky
Browse files

conformance: report MASA results and enable REND met-trigger

parent 3dad267c

scripts/ivas_conformance/runConformance.py

+36 −12
Original line number Diff line number Diff line
@@ -696,12 +696,19 @@ class MLDConformance: 
                if mld_error is not None:

                    return (None, None, mld_error, None)



                if tag in {"DEC", "REND"} and "EXT" in self._outputFormatsInCommand(testDesc.rawCmdline):

                refMdFile = testDesc.refOutput + ".met"

                dutMdFile = testDesc.dutOutput + ".met"

                runMasaMD = (

                    (tag == "DEC" and "EXT" in self._outputFormatsInCommand(testDesc.rawCmdline))

                    or (tag == "REND" and (os.path.exists(refMdFile) or os.path.exists(dutMdFile)))

                )



                if runMasaMD:

                    masaMD_scores, masaMD_error = self.masaMD(

                        tag,

                        dutPytestTag,

                        refMdFile=testDesc.refOutput + ".met",

                        dutMdFile=testDesc.dutOutput + ".met",

                        refMdFile=refMdFile,

                        dutMdFile=dutMdFile,

                    )

                    if masaMD_error is not None:

                        return (None, None, masaMD_error, None)
@@ -1324,21 +1331,33 @@ class MLDConformance: 
            self.doBEanalysis(selectTag=tag)

            analysis_ok = True

            corridor_fail_count = 0

            masa_comparison_done = False

            masa_corridor_fail_count = 0

        else:

            analysis_ok, corridor_fail_count, _ = self.doAnalysis(selectTag=tag)

            (

                analysis_ok,

                corridor_fail_count,

                _,

                masa_comparison_done,

                masa_corridor_fail_count,

            ) = self.doAnalysis(selectTag=tag)



        if self.args.regenerate_mld_ref:

            return command_fail_count == 0 and analysis_ok



        if command_fail_count == 0 and failure_count == 0 and analysis_ok:

            print(

                f"[{tag}] OK (ERRORS={command_fail_count}, BE={be_count}, NON-BE={non_be_count}, MLD CORRIDOR FAILURES={failure_count})\n"

        result_details = (

            f"ERRORS={command_fail_count}, BE={be_count}, NON-BE={non_be_count}, "

            f"MLD CORRIDOR FAILURES={failure_count}"

        )

        if masa_comparison_done:

            result_details += f", MASA MD CORRIDOR FAILURES={masa_corridor_fail_count}"



        print()

        if command_fail_count == 0 and failure_count == 0 and analysis_ok:

            print(f"[{tag}] OK ({result_details})\n")

            return True



        print(

            f"[{tag}] FAILED (ERRORS={command_fail_count}, BE={be_count}, NON-BE={non_be_count}, MLD CORRIDOR FAILURES={failure_count})"

        )

        print(f"[{tag}] FAILED ({result_details})")

        if worst_failure is not None:

            print(

                f"[{tag}] Worst MLD corridor failure: {worst_failure['prefix']} {worst_failure['tag']} "
@@ -1814,6 +1833,8 @@ class MLDConformance: 
    def doAnalysis(self, selectTag="all"):

        all_ok = True

        corridor_fail_count = 0

        masa_corridor_fail_count = 0

        masa_comparison_done = False

        max_mld_value = None

        keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag]

        for tag in keys:
@@ -1972,6 +1993,7 @@ class MLDConformance: 
                            if masaMDCutWithTags is not None and masaMDCutWithTags.size > 0:

                                np.savetxt(f, masaMDCutWithTags, fmt="%s", delimiter=",")

                elif masaMDCutWithTags is not None and masaMDCutWithTags.size > 0:

                    masa_comparison_done = True

                    refMasaMDFile = os.path.join(

                        self.testvDir, "mld_ref", ReferenceMasaMDFiles[tag]

                    )
@@ -1991,6 +2013,7 @@ class MLDConformance: 
                        )

                        all_ok = all_ok and corridor_ok

                        corridor_fail_count += int(not corridor_ok)

                        masa_corridor_fail_count += int(not corridor_ok)

                    else:

                        missing_msg = f"Missing reference MASA file for {tag} : {refMasaMDFile}"

                        print(f"\033[91m{missing_msg} \033[00m")
@@ -1998,8 +2021,9 @@ class MLDConformance: 
                        self.appendFailed(context=missing_msg)

                        all_ok = False

                        corridor_fail_count += 1

                        masa_corridor_fail_count += 1



                return all_ok, corridor_fail_count, max_mld_value

        return all_ok, corridor_fail_count, max_mld_value, masa_comparison_done, masa_corridor_fail_count





if __name__ == "__main__":
@@ -2133,7 +2157,7 @@ if __name__ == "__main__": 
    tag_results = {}

    for tag in testTags:

        if args.report_only:

            tag_ok, _, _ = conformance.doAnalysis(selectTag=tag)

            tag_ok, _, _, _, _ = conformance.doAnalysis(selectTag=tag)

        elif not args.analyse:

            tag_ok = conformance.runTag(tag)

        else: