From b99a2e5f599bb210acc4f00766e6ded59be87721 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Feb 2025 10:23:58 +0100 Subject: [PATCH 1/2] report regressions and improvements in separate files --- .../basop_check_for_changes_in_testcases.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index b47f0d6cc2..1ac3dfca18 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -37,7 +37,6 @@ import os import pathlib import re import xml.etree.ElementTree as ET -from typing import Tuple # set positive threshold for "lower is better" metrics, negative for "higher is better" @@ -48,8 +47,10 @@ COLS_2_THRESHOLDS = { "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), } -OUTFILE_CRASHES = "changes_crashes.csv" -OUTFILE_SCORES = "changes_{}.csv" +OUTFILE_CRASHES_FIXED = "improvements_crashes.csv" +OUTFILE_CRASHES_ADDED = "regressions_crashes.csv" +OUTFILE_SCORES_IMPROVEMENTS = "improvements_{}.csv" +OUTFILE_SCORES_REGRESSION = "regressions_{}.csv" PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:" PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:" @@ -102,7 +103,7 @@ def main(args): df_crashes_introduced = pd.merge( df_crashes_introduced, cmdlines_crashes_introduced, on="testcase" ) - df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") + df_crashes_introduced.to_csv(OUTFILE_CRASHES_ADDED, sep=";") if args.show_improvements and sum(mask_crash_fixed) > 0: df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index( @@ -119,7 +120,7 @@ def main(args): df_crashes_fixed = pd.merge( df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") + df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, mode="a", sep=";") # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") @@ -140,7 +141,7 @@ def main(args): mask_better = (df_merged[col_diff] * fac) < -thresh display_cols = ["testcase", col_curr, col_prev, col_diff] - outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) + outfile_regressions = OUTFILE_SCORES_REGRESSION.format(col.replace(" ", "_")) df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) if sum(mask_worse) > 0: regressions_found = True @@ -155,7 +156,7 @@ def main(args): df_worse["testcase"], xml_report, args.inject_cwd ) df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase") - df_worse.to_csv(outfile, sep=";") + df_worse.to_csv(outfile_regressions, sep=";") if args.show_improvements and sum(mask_better) > 0: df_better = df_merged[mask_better][display_cols].reset_index(drop=True) @@ -170,7 +171,10 @@ def main(args): df_better["testcase"], xml_report, args.inject_cwd ) df_better = pd.merge(df_better, cmdlines_better, on="testcase") - df_better.to_csv(outfile, mode="a", sep=";") + outfile_improvements = OUTFILE_SCORES_IMPROVEMENTS.format( + col.replace(" ", "_") + ) + df_better.to_csv(outfile_improvements, mode="a", sep=";") return int(regressions_found) -- GitLab From de027e8b9a58935378d96c61edcbafd9cf495aea Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Feb 2025 10:37:28 +0100 Subject: [PATCH 2/2] remove now obsolete append mode --- scripts/basop_check_for_changes_in_testcases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 1ac3dfca18..315afd0a88 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -120,7 +120,7 @@ def main(args): df_crashes_fixed = pd.merge( df_crashes_fixed, cmdlines_crashes_fixed, on="testcase" ) - df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, mode="a", sep=";") + df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, sep=";") # remove columns with ERRORs in any of the csv files before comparing the numerical columns mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR") @@ -174,7 +174,7 @@ def main(args): outfile_improvements = OUTFILE_SCORES_IMPROVEMENTS.format( col.replace(" ", "_") ) - df_better.to_csv(outfile_improvements, mode="a", sep=";") + df_better.to_csv(outfile_improvements, sep=";") return int(regressions_found) -- GitLab