Commit 7299ac91 authored by Jan Kiene's avatar Jan Kiene
Browse files

shorten testcase names and write out results to csv files

parent 8e02a759

scripts/basop_check_for_changes_in_testcases.py

+25 −6
Original line number Diff line number Diff line
@@ -45,6 +45,9 @@ COLS_2_THRESHOLDS = { 
    "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)),

}



OUTFILE_CRASHES = "changes_crashes.csv"

OUTFILE_SCORES = "changes_{}.csv"





def main(args):

    df_curr = pd.read_csv(args.csv_current, sep=";")
@@ -52,7 +55,9 @@ def main(args): 
    df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"])



    # remove leading path from testcase names for better readability

    df_merged["testcase"] = [pathlib.Path(tc).name for tc in df_merged["testcase"]]

    df_merged["testcase"] = [

        pathlib.Path(tc).name.split("::")[-1] for tc in df_merged["testcase"]

    ]



    # this is for printing the whole testcase names

    pd.options.display.max_colwidth = 200
@@ -69,15 +74,24 @@ def main(args): 
    )



    display_cols = ["testcase", col_curr, col_prev]

    if sum(mask_crash_introduced) > 0:

    df_crashes_introduced = df_merged[mask_crash_introduced][display_cols].reset_index(

        drop=True

    )

    df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";")



    if len(df_crashes_introduced) > 0:

        regressions_found = True

        print("---------------Testcases that introduced new crashes---------------")

        print(df_merged[mask_crash_introduced][display_cols].reset_index(drop=True))

        print(df_crashes_introduced)

        print()



    if args.show_improvements and sum(mask_crash_fixed) > 0:

        df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index(

            drop=True

        )

        df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";")

        print("---------------Testcases that fixed crashes---------------")

        print(df_merged[mask_crash_fixed][display_cols].reset_index(drop=True))

        print(df_crashes_fixed)

        print()



    # remove columns with ERRORs in any of the csv files before comparing the numerical columns
@@ -100,19 +114,24 @@ def main(args): 
        mask_better = diff < -thresh



        display_cols = ["testcase", col_curr, col_prev]

        outfile = OUTFILE_SCORES.format(col)

        df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True)

        df_worse.to_csv(outfile, sep=";")

        if sum(mask_worse) > 0:

            regressions_found = True

            print(

                f"---------------Testcases that got worse wrt to {col}---------------"

            )

            print(df_merged[mask_worse][display_cols].reset_index(drop=True))

            print(df_worse)

            print()



        if args.show_improvements and sum(mask_better) > 0:

            df_better = df_merged[mask_better][display_cols].reset_index(drop=True)

            df_better.to_csv(outfile, mode="a", sep=";")

            print(

                f"---------------Testcases that got better wrt to {col}---------------"

            )

            print(df_merged[mask_better][display_cols].reset_index(drop=True))

            print(df_better)

            print()



    return int(regressions_found)