Merge branch 'basop-ci/check-for-regressions-in-mr-pl' into 'main'

#! /usr/bin/env python3

"""

(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,

Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

contributors to this repository. All Rights Reserved.

This software is protected by copyright law and by international treaties.

The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,

Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

contributors to this repository retain full ownership rights in their respective contributions in

the software. This notice grants no license of any kind, including but not limited to patent

license, nor is any license granted by implication, estoppel or otherwise.

Contributors are required to enter into the IVAS codec Public Collaboration agreement before making

contributions.

This software is provided "AS IS", without any express or implied warranties. The software is in the

development stage. It is intended exclusively for experts who have experience with such software and

solely for the purpose of inspection. All implied warranties of non-infringement, merchantability

and fitness for a particular purpose are hereby disclaimed and excluded.

Any dispute, controversy or claim arising under or in relation to providing this software shall be

submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in

accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and

the United Nations Convention on Contracts on the International Sales of Goods.

"""

import pandas as pd

import argparse

import sys

import os

import pathlib

# set positive threshold for "lower is better" metrics, negative for "higher is better"

COLS_2_THRESHOLDS = {

    "MLD": float(os.environ.get("CI_REGRESSION_THRESH_MLD", 0.1)),

    "MAXIMUM ABS DIFF": float(os.environ.get("CI_REGRESSION_THRESH_MAX_ABS_DIFF", 5)),

    "MIN_SSNR": float(os.environ.get("CI_REGRESSION_THRESH_SSNR", -1)),

    "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)),

}

def main(args):

    df_curr = pd.read_csv(args.csv_current, sep=";")

    df_prev = pd.read_csv(args.csv_previous, sep=";")

    df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"])

    # remove leading path from testcase names for better readability

    df_merged["testcase"] = [pathlib.Path(tc).name for tc in df_merged["testcase"]]

    # this is for printing the whole testcase names

    pd.options.display.max_colwidth = 200

    regressions_found = False

    # check for newly introduced crashes

    col_curr = "Result-curr"

    col_prev = "Result-prev"

    mask_crash_introduced = (df_merged[col_curr] == "ERROR") & (

        df_merged[col_prev] != "ERROR"

    )

    mask_crash_fixed = (df_merged[col_curr] != "ERROR") & (

        df_merged[col_prev] == "ERROR"

    )

    display_cols = ["testcase", col_curr, col_prev]

    if sum(mask_crash_introduced) > 0:

        regressions_found = True

        print("---------------Testcases that introduced new crashes---------------")

        print(df_merged[mask_crash_introduced][display_cols].reset_index(drop=True))

        print()

    if args.show_improvements and sum(mask_crash_fixed) > 0:

        print("---------------Testcases that fixed crashes---------------")

        print(df_merged[mask_crash_fixed][display_cols].reset_index(drop=True))

        print()

    # remove columns with ERRORs in any of the csv files before comparing the numerical columns

    mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR")

    df_merged = df_merged[mask_no_errors].reset_index(drop=True)

    # check numeric columns and compare diff to thresholds

    for col in args.columns_to_compare:

        col_curr = f"{col}-curr"

        col_prev = f"{col}-prev"

        diff = df_merged[col_curr] - df_merged[col_prev]

        thresh = COLS_2_THRESHOLDS[col]

        # invert sign of difference for "higher is better" metrics

        if thresh < 0:

            diff *= -1

        thresh = abs(thresh)

        mask_worse = diff > thresh

        mask_better = diff < -thresh

        display_cols = ["testcase", col_curr, col_prev]

        if sum(mask_worse) > 0:

            regressions_found = True

            print(

                f"---------------Testcases that got worse wrt to {col}---------------"

            )

            print(df_merged[mask_worse][display_cols].reset_index(drop=True))

            print()

        if args.show_improvements and sum(mask_better) > 0:

            print(

                f"---------------Testcases that got better wrt to {col}---------------"

            )

            print(df_merged[mask_better][display_cols].reset_index(drop=True))

            print()

    return int(regressions_found)

if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("csv_current")

    parser.add_argument("csv_previous")

    parser.add_argument(

        "--columns_to_compare",

        nargs="+",

        choices=COLS_2_THRESHOLDS.keys(),

        default=COLS_2_THRESHOLDS.keys(),

    )

    parser.add_argument("--show_improvements", action="store_true")

    args = parser.parse_args()

    sys.exit(main(args))