Commit e164f2a5 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'basop-ci/improve-regression-check-scripts' into 'basop-ci-branch' 

[BASOP-CI] report regressions and improvements in separate files

See merge request !2030
parents 6d36acce de027e8b

scripts/basop_check_for_changes_in_testcases.py

+12 −8
Original line number Diff line number Diff line
@@ -37,7 +37,6 @@ import os 
import pathlib

import re

import xml.etree.ElementTree as ET

from typing import Tuple





# set positive threshold for "lower is better" metrics, negative for "higher is better"
@@ -48,8 +47,10 @@ COLS_2_THRESHOLDS = { 
    "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)),

}



OUTFILE_CRASHES = "changes_crashes.csv"

OUTFILE_SCORES = "changes_{}.csv"

OUTFILE_CRASHES_FIXED = "improvements_crashes.csv"

OUTFILE_CRASHES_ADDED = "regressions_crashes.csv"

OUTFILE_SCORES_IMPROVEMENTS = "improvements_{}.csv"

OUTFILE_SCORES_REGRESSION = "regressions_{}.csv"



PATTERN_ENC = r"... encoder command:\s*(.*)\s*... encoder stdout:"

PATTERN_DEC = r"... decoder command:\s*(.*)\s*... decoder stdout:"
@@ -102,7 +103,7 @@ def main(args): 
            df_crashes_introduced = pd.merge(

                df_crashes_introduced, cmdlines_crashes_introduced, on="testcase"

            )

    df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";")

    df_crashes_introduced.to_csv(OUTFILE_CRASHES_ADDED, sep=";")



    if args.show_improvements and sum(mask_crash_fixed) > 0:

        df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index(
@@ -119,7 +120,7 @@ def main(args): 
            df_crashes_fixed = pd.merge(

                df_crashes_fixed, cmdlines_crashes_fixed, on="testcase"

            )

        df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";")

        df_crashes_fixed.to_csv(OUTFILE_CRASHES_FIXED, sep=";")



    # remove columns with ERRORs in any of the csv files before comparing the numerical columns

    mask_no_errors = (df_merged[col_curr] != "ERROR") & (df_merged[col_prev] != "ERROR")
@@ -140,7 +141,7 @@ def main(args): 
        mask_better = (df_merged[col_diff] * fac) < -thresh



        display_cols = ["testcase", col_curr, col_prev, col_diff]

        outfile = OUTFILE_SCORES.format(col.replace(" ", "_"))

        outfile_regressions = OUTFILE_SCORES_REGRESSION.format(col.replace(" ", "_"))

        df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True)

        if sum(mask_worse) > 0:

            regressions_found = True
@@ -155,7 +156,7 @@ def main(args): 
                    df_worse["testcase"], xml_report, args.inject_cwd

                )

                df_worse = pd.merge(df_worse, cmdlines_worse, on="testcase")

        df_worse.to_csv(outfile, sep=";")

        df_worse.to_csv(outfile_regressions, sep=";")



        if args.show_improvements and sum(mask_better) > 0:

            df_better = df_merged[mask_better][display_cols].reset_index(drop=True)
@@ -170,7 +171,10 @@ def main(args): 
                    df_better["testcase"], xml_report, args.inject_cwd

                )

                df_better = pd.merge(df_better, cmdlines_better, on="testcase")

            df_better.to_csv(outfile, mode="a", sep=";")

            outfile_improvements = OUTFILE_SCORES_IMPROVEMENTS.format(

                col.replace(" ", "_")

            )

            df_better.to_csv(outfile_improvements, sep=";")



    return int(regressions_found)