diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py new file mode 100644 index 0000000000000000000000000000000000000000..d0c5bed7a2b11e51cec100315ac760377148e240 --- /dev/null +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -0,0 +1,134 @@ +#! /usr/bin/env python3 + +""" +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. +""" + +import pandas as pd +import argparse +import sys +import os +import pathlib + + +# set positive threshold for "lower is better" metrics, negative for "higher is better" +COLS_2_THRESHOLDS = { + "MLD": float(os.environ.get("CI_REGRESSION_THRESH_MLD", 0.1)), + "MAXIMUM ABS DIFF": float(os.environ.get("CI_REGRESSION_THRESH_MAX_ABS_DIFF", 5)), + "MIN_SSNR": float(os.environ.get("CI_REGRESSION_THRESH_SSNR", -1)), + "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), +} + + +def main(args): + df_curr = pd.read_csv(args.csv_current, sep=";") + df_prev = pd.read_csv(args.csv_previous, sep=";") + df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"]) + + # remove leading path from testcase names for better readability + df_merged["testcase"] = [pathlib.Path(tc).name for tc in df_merged["testcase"]] + + # this is for printing the whole testcase names + pd.options.display.max_colwidth = 200 + regressions_found = False + + # check for newly introduced crashes + col_curr = "Result-curr" + col_prev = "Result-prev" + mask_crash_introduced = (df_merged[col_curr] == "ERROR") & ( + df_merged[col_prev] != "ERROR" + ) + mask_crash_fixed = (df_merged[col_curr] != "ERROR") & ( + df_merged[col_prev] == "ERROR" + ) + + display_cols = ["testcase", col_curr, col_prev] + if sum(mask_crash_introduced) > 0: + regressions_found = True + print("---------------Testcases that introduced new crashes---------------") + print(df_merged[mask_crash_introduced][display_cols].reset_index(drop=True)) + print() + + if args.show_improvements and sum(mask_crash_fixed) > 0: + print("---------------Testcases that fixed crashes---------------") + print(df_merged[mask_crash_fixed][display_cols].reset_index(drop=True)) + print() + + # remove columns with ERRORs in any of the csv files before comparing the numerical columns + mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") + df_merged = df_merged[mask_no_errors].reset_index(drop=True) + + # check numeric columns and compare diff to thresholds + for col in args.columns_to_compare: + col_curr = f"{col}-curr" + col_prev = f"{col}-prev" + diff = df_merged[col_curr] - df_merged[col_prev] + + thresh = COLS_2_THRESHOLDS[col] + # invert sign of difference for "higher is better" metrics + if thresh < 0: + diff *= -1 + + thresh = abs(thresh) + mask_worse = diff > thresh + mask_better = diff < -thresh + + display_cols = ["testcase", col_curr, col_prev] + if sum(mask_worse) > 0: + regressions_found = True + print( + f"---------------Testcases that got worse wrt to {col}---------------" + ) + print(df_merged[mask_worse][display_cols].reset_index(drop=True)) + print() + + if args.show_improvements and sum(mask_better) > 0: + print( + f"---------------Testcases that got better wrt to {col}---------------" + ) + print(df_merged[mask_better][display_cols].reset_index(drop=True)) + print() + + return int(regressions_found) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("csv_current") + parser.add_argument("csv_previous") + parser.add_argument( + "--columns_to_compare", + nargs="+", + choices=COLS_2_THRESHOLDS.keys(), + default=COLS_2_THRESHOLDS.keys(), + ) + parser.add_argument("--show_improvements", action="store_true") + + args = parser.parse_args() + sys.exit(main(args))