Commit 91b20f49 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'basop-ci-branch' into basop-ci/handle-changes-in-testcases

parents e4ac4575 720ad874

ci/complexity_measurements/parseNewsletterRam.py

+9 −9
Original line number Diff line number Diff line
@@ -48,17 +48,17 @@ if __name__ == "__main__": 
    shortDate = sys.argv[5]

    fullDate = sys.argv[6]



max_total_enc = ["", 0]

max_total_dec = ["", 0]

max_total_encdec = ["", 0]

max_total_enc = ["None", 0]

max_total_dec = ["None", 0]

max_total_encdec = ["None", 0]



max_stack_enc = ["", 0]

max_stack_dec = ["", 0]

max_stack_encdec = ["", 0]

max_stack_enc = ["None", 0]

max_stack_dec = ["None", 0]

max_stack_encdec = ["None", 0]



max_heap_enc = ["", 0]

max_heap_dec = ["", 0]

max_heap_encdec = ["", 0]

max_heap_enc = ["None", 0]

max_heap_dec = ["None", 0]

max_heap_encdec = ["None", 0]



ram_table = {}

ci/complexity_measurements/parseNewsletterRom.py

+13 −13
Original line number Diff line number Diff line
@@ -49,19 +49,19 @@ if __name__ == "__main__": 
    shortDate = sys.argv[5]

    fullDate = sys.argv[6]



max_prom_enc = ["", 0]

max_prom_dec = ["", 0]

max_prom_com = ["", 0]

max_prom_rend = ["", 0]

max_prom_total = ["", 0]



max_trom_enc = ["", 0]

max_trom_dec = ["", 0]

max_trom_com = ["", 0]

max_trom_rend = ["", 0]

max_trom_total = ["", 0]



max_total_encdec = ["", 0]

max_prom_enc = ["None", 0]

max_prom_dec = ["None", 0]

max_prom_com = ["None", 0]

max_prom_rend = ["None", 0]

max_prom_total = ["None", 0]



max_trom_enc = ["None", 0]

max_trom_dec = ["None", 0]

max_trom_com = ["None", 0]

max_trom_rend = ["None", 0]

max_trom_total = ["None", 0]



max_total_encdec = ["None", 0]



rom_table = {}

ci/complexity_measurements/parseNewsletterWmops.py

+3 −3
Original line number Diff line number Diff line
@@ -47,9 +47,9 @@ if __name__ == "__main__": 
    shortDate = sys.argv[4]

    fullDate = sys.argv[5]



max_enc = ["", 0]

max_dec = ["", 0]

max_total = ["", 0]

max_enc = ["None", 0]

max_dec = ["None", 0]

max_total = ["None", 0]

fixedpointScalingFac = 1.0



with open(newsletterFilename, "r") as csvfile:

ci/setup_pages.py

+17 −0
Original line number Diff line number Diff line
@@ -49,6 +49,23 @@ JOBS_BASOP_REPO = { 
    "ivas-pytest-compare_ref-long-enc": "Pytest encoder compare to ref LTV",

    "ivas-pytest-compare_ref-long-enc-lev+10": "Pytest encoder compare to ref LTV +10dB",

    "ivas-pytest-compare_ref-long-enc-lev-10": "Pytest encoder compare to ref LTV -10dB",

    "complexity-ism-in-binaural-out": "ISM in, BINAURAL out",

    "complexity-ism-in-binaural_room_ir-out": "ISM in, BINAURAL_ROOM_IR out",

    "complexity-ism-in-ext-out": "ISM in, EXT out",

    "complexity-sba-hoa3-in-hoa3-out": "HOA3 in, HOA3 out",

    "complexity-sba-hoa3-in-binaural-out": "HOA3 in, BINAURAL out",

    "complexity-sba-hoa3-in-binaural_room_ir-out": "HOA3 in, BINAURAL_ROOM_IR out",

    "complexity-mc-in-7_1_4-out": "MC in, 7_1_4 out",

    "complexity-mc-in-binaural-out": "MC in, BINAURAL out",

    "complexity-mc-in-binaural_room_ir-out": "MC in, BINAURAL_ROOM_IR out",

    "complexity-masa-in-ext-out": "MASA in, EXT out",

    "complexity-masa-in-binaural-out": "MASA in, BINAURAL out",

    "complexity-masa-in-hoa3-out": "MASA in, HOA3 out",

    "complexity-omasa-in-binaural-out": "OMASA in, BINAURAL out",

    "complexity-omasa-in-hoa3-out": "OMASA in HOA3 out",

    "complexity-StereoDmxEVS-stereo-in-mono-out": "StereoDmxEVS",

    "complexity-osba-in-binaural-out": "OSBA in, BINAURAL out",

    "complexity-osba-in-binaural_room_ir-out": "OSBA in, BINAURAL_ROOM_IR out",

    "complexity-stereo-in-stereo-out": "Stereo in, Stereo out",

}

scripts/basop_check_for_changes_in_testcases.py

+30 −12
Original line number Diff line number Diff line
@@ -45,6 +45,9 @@ COLS_2_THRESHOLDS = { 
    "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)),

}



OUTFILE_CRASHES = "changes_crashes.csv"

OUTFILE_SCORES = "changes_{}.csv"





def main(args):

    df_curr = pd.read_csv(args.csv_current, sep=";")
@@ -52,7 +55,9 @@ def main(args): 
    df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"])



    # remove leading path from testcase names for better readability

    df_merged["testcase"] = [pathlib.Path(tc).name for tc in df_merged["testcase"]]

    df_merged["testcase"] = [

        pathlib.Path(tc).name.split("::")[-1] for tc in df_merged["testcase"]

    ]



    # this is for printing the whole testcase names

    pd.options.display.max_colwidth = 200
@@ -69,15 +74,24 @@ def main(args): 
    )



    display_cols = ["testcase", col_curr, col_prev]

    df_crashes_introduced = df_merged[mask_crash_introduced][display_cols].reset_index(

        drop=True

    )

    df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";")



    if sum(mask_crash_introduced) > 0:

        regressions_found = True

        print("---------------Testcases that introduced new crashes---------------")

        print(df_merged[mask_crash_introduced][display_cols].reset_index(drop=True))

        print(df_crashes_introduced)

        print()



    if args.show_improvements and sum(mask_crash_fixed) > 0:

        df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index(

            drop=True

        )

        df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";")

        print("---------------Testcases that fixed crashes---------------")

        print(df_merged[mask_crash_fixed][display_cols].reset_index(drop=True))

        print(df_crashes_fixed)

        print()



    # remove columns with ERRORs in any of the csv files before comparing the numerical columns
@@ -88,31 +102,35 @@ def main(args): 
    for col in args.columns_to_compare:

        col_curr = f"{col}-curr"

        col_prev = f"{col}-prev"

        diff = df_merged[col_curr] - df_merged[col_prev]

        col_diff = f"{col}-diff"

        df_merged[col_diff] = df_merged[col_curr] - df_merged[col_prev]



        thresh = COLS_2_THRESHOLDS[col]

        # invert sign of difference for "higher is better" metrics

        if thresh < 0:

            diff *= -1



        fac = -1 if thresh < 0 else 1

        thresh = abs(thresh)

        mask_worse = diff > thresh

        mask_better = diff < -thresh

        mask_worse = (df_merged[col_diff] * fac) > thresh

        mask_better = (df_merged[col_diff] * fac) < -thresh



        display_cols = ["testcase", col_curr, col_prev]

        display_cols = ["testcase", col_curr, col_prev, col_diff]

        outfile = OUTFILE_SCORES.format(col.replace(" ", "_"))

        df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True)

        df_worse.to_csv(outfile, sep=";")

        if sum(mask_worse) > 0:

            regressions_found = True

            print(

                f"---------------Testcases that got worse wrt to {col}---------------"

            )

            print(df_merged[mask_worse][display_cols].reset_index(drop=True))

            print(df_worse)

            print()



        if args.show_improvements and sum(mask_better) > 0:

            df_better = df_merged[mask_better][display_cols].reset_index(drop=True)

            df_better.to_csv(outfile, mode="a", sep=";")

            print(

                f"---------------Testcases that got better wrt to {col}---------------"

            )

            print(df_merged[mask_better][display_cols].reset_index(drop=True))

            print(df_better)

            print()



    return int(regressions_found)