From a0424eeddf51bd641ef60d299cae358d475fbdad Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 11:38:03 +0200 Subject: [PATCH 1/9] add script to check for regressions in basop CI MR pipelines --- scripts/basop_check_for_regressions.py | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 scripts/basop_check_for_regressions.py diff --git a/scripts/basop_check_for_regressions.py b/scripts/basop_check_for_regressions.py new file mode 100644 index 0000000000..ea9f1d3eaa --- /dev/null +++ b/scripts/basop_check_for_regressions.py @@ -0,0 +1,52 @@ +#! /usr/bin/env python3 + +import pandas as pd +import argparse +import sys + + +# set positive threshold for "lower is better" metrics, negative for "higher is better" +COLS_2_THRESHOLDS = { + "MLD": 0.1, + "MAXIMUM ABS DIFF": 5, + "MIN_SSNR": -1, + "MIN_ODG": -0.05, +} + + +def main(args): + df_curr = pd.read_csv(args.csv_current, sep=";").sort_values(by="testcase") + df_prev = pd.read_csv(args.csv_previous, sep=";").sort_values(by="testcase") + + pd.options.display.max_colwidth = 200 + regressions_found = False + for col, thresh in COLS_2_THRESHOLDS.items(): + diff = df_curr[col] - df_prev[col] + + # invert sign of difference for "higher is better" metrics + if thresh < 0: + diff *= -1 + + thresh = abs(thresh) + mask_worse = diff > thresh + mask_better = diff < -thresh + + if sum(mask_worse) > 0: + regressions_found = True + print(f"Testcases that got worse wrt to {col}:") + print(df_curr[mask_worse]["testcase"]) + + if sum(mask_better) > 0: + print(f"Testcases that got better wrt to {col}:") + print(df_curr[mask_better]["testcase"]) + + return int(regressions_found) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("csv_current") + parser.add_argument("csv_previous") + + args = parser.parse_args() + sys.exit(main(args)) -- GitLab From e9d7afef11f9d045cb430bf87c1313d6b02f4eec Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 11:53:31 +0200 Subject: [PATCH 2/9] improve printout --- scripts/basop_check_for_regressions.py | 28 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/scripts/basop_check_for_regressions.py b/scripts/basop_check_for_regressions.py index ea9f1d3eaa..6d1b5b9aeb 100644 --- a/scripts/basop_check_for_regressions.py +++ b/scripts/basop_check_for_regressions.py @@ -3,6 +3,7 @@ import pandas as pd import argparse import sys +import pathlib # set positive threshold for "lower is better" metrics, negative for "higher is better" @@ -15,13 +16,19 @@ COLS_2_THRESHOLDS = { def main(args): - df_curr = pd.read_csv(args.csv_current, sep=";").sort_values(by="testcase") - df_prev = pd.read_csv(args.csv_previous, sep=";").sort_values(by="testcase") + df_curr = pd.read_csv(args.csv_current, sep=";") + df_prev = pd.read_csv(args.csv_previous, sep=";") + df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"]) + + # remove leading path from testcase names for better readability + df_merged["testcase"] = [pathlib.Path(tc).name for tc in df_merged["testcase"]] pd.options.display.max_colwidth = 200 regressions_found = False for col, thresh in COLS_2_THRESHOLDS.items(): - diff = df_curr[col] - df_prev[col] + col_curr = f"{col}-curr" + col_prev = f"{col}-prev" + diff = df_merged[col_curr] - df_merged[col_prev] # invert sign of difference for "higher is better" metrics if thresh < 0: @@ -31,14 +38,21 @@ def main(args): mask_worse = diff > thresh mask_better = diff < -thresh + display_cols = ["testcase", col_curr, col_prev] if sum(mask_worse) > 0: regressions_found = True - print(f"Testcases that got worse wrt to {col}:") - print(df_curr[mask_worse]["testcase"]) + print( + f"---------------Testcases that got worse wrt to {col}---------------" + ) + print(df_merged[mask_worse][display_cols].reset_index(drop=True)) + print() if sum(mask_better) > 0: - print(f"Testcases that got better wrt to {col}:") - print(df_curr[mask_better]["testcase"]) + print( + f"---------------Testcases that got better wrt to {col}---------------" + ) + print(df_merged[mask_better][display_cols].reset_index(drop=True)) + print() return int(regressions_found) -- GitLab From 2bb4c286e3add418b74ffdc7267c6ae72162f93e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 12:00:31 +0200 Subject: [PATCH 3/9] add argument for selecting the columns to compare on --- scripts/basop_check_for_regressions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/basop_check_for_regressions.py b/scripts/basop_check_for_regressions.py index 6d1b5b9aeb..166e96fd84 100644 --- a/scripts/basop_check_for_regressions.py +++ b/scripts/basop_check_for_regressions.py @@ -25,11 +25,12 @@ def main(args): pd.options.display.max_colwidth = 200 regressions_found = False - for col, thresh in COLS_2_THRESHOLDS.items(): + for col in args.columns_to_compare: col_curr = f"{col}-curr" col_prev = f"{col}-prev" diff = df_merged[col_curr] - df_merged[col_prev] + thresh = COLS_2_THRESHOLDS[col] # invert sign of difference for "higher is better" metrics if thresh < 0: diff *= -1 @@ -61,6 +62,12 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("csv_current") parser.add_argument("csv_previous") + parser.add_argument( + "--columns_to_compare", + nargs="+", + choices=COLS_2_THRESHOLDS.keys(), + default=COLS_2_THRESHOLDS.keys(), + ) args = parser.parse_args() sys.exit(main(args)) -- GitLab From 2677852c0b6d1b5c0d0636caf13ce99415cf1ed8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 12:30:17 +0200 Subject: [PATCH 4/9] add copyright header --- scripts/basop_check_for_regressions.py | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/scripts/basop_check_for_regressions.py b/scripts/basop_check_for_regressions.py index 166e96fd84..20880f9743 100644 --- a/scripts/basop_check_for_regressions.py +++ b/scripts/basop_check_for_regressions.py @@ -1,5 +1,35 @@ #! /usr/bin/env python3 +""" + (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. +""" + import pandas as pd import argparse import sys -- GitLab From 0842301ef4da52b38154cd6ed407ed46b0110bcd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 12:32:43 +0200 Subject: [PATCH 5/9] get thresholds from env vars for easier adjustment in ci setup --- scripts/basop_check_for_regressions.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/basop_check_for_regressions.py b/scripts/basop_check_for_regressions.py index 20880f9743..2b1353182e 100644 --- a/scripts/basop_check_for_regressions.py +++ b/scripts/basop_check_for_regressions.py @@ -33,15 +33,16 @@ import pandas as pd import argparse import sys +import os import pathlib # set positive threshold for "lower is better" metrics, negative for "higher is better" COLS_2_THRESHOLDS = { - "MLD": 0.1, - "MAXIMUM ABS DIFF": 5, - "MIN_SSNR": -1, - "MIN_ODG": -0.05, + "MLD": os.environ.get("CI_REGRESSION_THRESH_MLD", 0.1), + "MAXIMUM ABS DIFF": os.environ.get("CI_REGRESSION_THRESH_MAX_ABS_DIFF", 5), + "MIN_SSNR": os.environ.get("CI_REGRESSION_THRESH_SSNR", -1), + "MIN_ODG": os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05), } -- GitLab From 9a475a93f7d91c47eac7db5f49b0ff71331a0d94 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 13:58:07 +0200 Subject: [PATCH 6/9] convert environment variable threshold values to float numbers --- scripts/basop_check_for_regressions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/basop_check_for_regressions.py b/scripts/basop_check_for_regressions.py index 2b1353182e..9e28743b1c 100644 --- a/scripts/basop_check_for_regressions.py +++ b/scripts/basop_check_for_regressions.py @@ -39,10 +39,10 @@ import pathlib # set positive threshold for "lower is better" metrics, negative for "higher is better" COLS_2_THRESHOLDS = { - "MLD": os.environ.get("CI_REGRESSION_THRESH_MLD", 0.1), - "MAXIMUM ABS DIFF": os.environ.get("CI_REGRESSION_THRESH_MAX_ABS_DIFF", 5), - "MIN_SSNR": os.environ.get("CI_REGRESSION_THRESH_SSNR", -1), - "MIN_ODG": os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05), + "MLD": float(os.environ.get("CI_REGRESSION_THRESH_MLD", 0.1)), + "MAXIMUM ABS DIFF": float(os.environ.get("CI_REGRESSION_THRESH_MAX_ABS_DIFF", 5)), + "MIN_SSNR": float(os.environ.get("CI_REGRESSION_THRESH_SSNR", -1)), + "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), } -- GitLab From 01e941b9d68b44f23328fd1d6d58ab87247a3267 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 14:22:58 +0200 Subject: [PATCH 7/9] add stage that checks for introduced/fixed crashes --- scripts/basop_check_for_regressions.py | 83 +++++++++++++++++--------- 1 file changed, 56 insertions(+), 27 deletions(-) diff --git a/scripts/basop_check_for_regressions.py b/scripts/basop_check_for_regressions.py index 9e28743b1c..d60004346d 100644 --- a/scripts/basop_check_for_regressions.py +++ b/scripts/basop_check_for_regressions.py @@ -1,33 +1,33 @@ #! /usr/bin/env python3 """ - (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import pandas as pd @@ -54,8 +54,37 @@ def main(args): # remove leading path from testcase names for better readability df_merged["testcase"] = [pathlib.Path(tc).name for tc in df_merged["testcase"]] + # this is for printing the whole testcase names pd.options.display.max_colwidth = 200 regressions_found = False + + # check for newly introduced crashes + col_curr = "Result-curr" + col_prev = "Result-prev" + mask_crash_introduced = (df_merged[col_curr] == "ERROR") & ( + df_merged[col_prev] != "ERROR" + ) + mask_crash_fixed = (df_merged[col_curr] != "ERROR") & ( + df_merged[col_prev] == "ERROR" + ) + + display_cols = ["testcase", col_curr, col_prev] + if sum(mask_crash_introduced) > 0: + regressions_found = True + print("---------------Testcases that introduced new crashes---------------") + print(df_merged[mask_crash_introduced][display_cols].reset_index(drop=True)) + print() + + if sum(mask_crash_fixed) > 0: + print("---------------Testcases that fixed crashes---------------") + print(df_merged[mask_crash_fixed][display_cols].reset_index(drop=True)) + print() + + # remove columns with ERRORs in any of the csv files before comparing the numerical columns + mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") + df_merged = df_merged[mask_no_errors].reset_index(drop=True) + + # check numeric columns and compare diff to thresholds for col in args.columns_to_compare: col_curr = f"{col}-curr" col_prev = f"{col}-prev" -- GitLab From 682eaf71f66664b41c0f88e0eb6a5588b3e87152 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 14:24:50 +0200 Subject: [PATCH 8/9] rename script to better reflect that it also checks for improvements --- ...for_regressions.py => basop_check_for_changes_in_testcases.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{basop_check_for_regressions.py => basop_check_for_changes_in_testcases.py} (100%) diff --git a/scripts/basop_check_for_regressions.py b/scripts/basop_check_for_changes_in_testcases.py similarity index 100% rename from scripts/basop_check_for_regressions.py rename to scripts/basop_check_for_changes_in_testcases.py -- GitLab From a9deec66dbc9769a363070aa08bf350272218a81 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 14:27:06 +0200 Subject: [PATCH 9/9] show improvements only when flag is given --- scripts/basop_check_for_changes_in_testcases.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index d60004346d..d0c5bed7a2 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -75,7 +75,7 @@ def main(args): print(df_merged[mask_crash_introduced][display_cols].reset_index(drop=True)) print() - if sum(mask_crash_fixed) > 0: + if args.show_improvements and sum(mask_crash_fixed) > 0: print("---------------Testcases that fixed crashes---------------") print(df_merged[mask_crash_fixed][display_cols].reset_index(drop=True)) print() @@ -108,7 +108,7 @@ def main(args): print(df_merged[mask_worse][display_cols].reset_index(drop=True)) print() - if sum(mask_better) > 0: + if args.show_improvements and sum(mask_better) > 0: print( f"---------------Testcases that got better wrt to {col}---------------" ) @@ -128,6 +128,7 @@ if __name__ == "__main__": choices=COLS_2_THRESHOLDS.keys(), default=COLS_2_THRESHOLDS.keys(), ) + parser.add_argument("--show_improvements", action="store_true") args = parser.parse_args() sys.exit(main(args)) -- GitLab