diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index af937355ef4054fbf24561bf0aee9aa0285ee776..cf3a3a6472347271149589643cb7275ddf4ee7f3 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -199,11 +199,15 @@ def tr_from_row(row, id_current, id_previous): if float(curr) > float(prev): curr += f" {ARROW_UP}" # increase is bad -> mark in red, execpt for SNR for which it is good -> mark in green - td_tmpl_curr = TD_TMPL_REDUCE if c == "MIN_SSNR" else TD_TMPL_INCREASE + td_tmpl_curr = ( + TD_TMPL_REDUCE if c == "MIN_SSNR" else TD_TMPL_INCREASE + ) elif float(curr) < float(prev): curr += f" {ARROW_DOWN}" # reduce is good -> mark in green, execpt for SNR for which it is bad -> mark in red - td_tmpl_curr = TD_TMPL_INCREASE if c == "MIN_SSNR" else TD_TMPL_REDUCE + td_tmpl_curr = ( + TD_TMPL_INCREASE if c == "MIN_SSNR" else TD_TMPL_REDUCE + ) except ValueError: # if we land here, one of the cells is not a number, this indicates a crash # or some error in the scripts, so mark with red as well @@ -295,22 +299,25 @@ def merge_tables(tbl1, tbl2, suffix1, suffix2, merge_key, other_keys): for key in other_keys: new_row[f"{key}-{suffix1}"] = row1[key] + found_merge_key_in_both_tbls = False for row2 in tbl2: if row1[merge_key] == row2[merge_key]: new_row[merge_key] = row1[merge_key] for key in other_keys: - if key in row2: # In case key is missing, just insert a blank + if key in row2: # In case key is missing, just insert a blank new_row[f"{key}-{suffix2}"] = row2[key] else: new_row[f"{key}-{suffix2}"] = "" + + found_merge_key_in_both_tbls = True break - merged.append(new_row) + if found_merge_key_in_both_tbls: + merged.append(new_row) return merged - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("html_out") diff --git a/ci/complexity_measurements/parseNewsletterRam.py b/ci/complexity_measurements/parseNewsletterRam.py index 6fe1cd899329675d10dc47f7130ef28638dcc314..55eecff5738264d933b13470ee0f35789c138853 100755 --- a/ci/complexity_measurements/parseNewsletterRam.py +++ b/ci/complexity_measurements/parseNewsletterRam.py @@ -48,17 +48,17 @@ if __name__ == "__main__": shortDate = sys.argv[5] fullDate = sys.argv[6] -max_total_enc = ["", 0] -max_total_dec = ["", 0] -max_total_encdec = ["", 0] +max_total_enc = ["None", 0] +max_total_dec = ["None", 0] +max_total_encdec = ["None", 0] -max_stack_enc = ["", 0] -max_stack_dec = ["", 0] -max_stack_encdec = ["", 0] +max_stack_enc = ["None", 0] +max_stack_dec = ["None", 0] +max_stack_encdec = ["None", 0] -max_heap_enc = ["", 0] -max_heap_dec = ["", 0] -max_heap_encdec = ["", 0] +max_heap_enc = ["None", 0] +max_heap_dec = ["None", 0] +max_heap_encdec = ["None", 0] ram_table = {} diff --git a/ci/complexity_measurements/parseNewsletterRom.py b/ci/complexity_measurements/parseNewsletterRom.py index 2416a6fd0b8977db3a14f06b8864ea2df973447a..a4a3df4ecfe0da6763668dd4e4e5639ebbb087aa 100755 --- a/ci/complexity_measurements/parseNewsletterRom.py +++ b/ci/complexity_measurements/parseNewsletterRom.py @@ -49,19 +49,19 @@ if __name__ == "__main__": shortDate = sys.argv[5] fullDate = sys.argv[6] -max_prom_enc = ["", 0] -max_prom_dec = ["", 0] -max_prom_com = ["", 0] -max_prom_rend = ["", 0] -max_prom_total = ["", 0] - -max_trom_enc = ["", 0] -max_trom_dec = ["", 0] -max_trom_com = ["", 0] -max_trom_rend = ["", 0] -max_trom_total = ["", 0] - -max_total_encdec = ["", 0] +max_prom_enc = ["None", 0] +max_prom_dec = ["None", 0] +max_prom_com = ["None", 0] +max_prom_rend = ["None", 0] +max_prom_total = ["None", 0] + +max_trom_enc = ["None", 0] +max_trom_dec = ["None", 0] +max_trom_com = ["None", 0] +max_trom_rend = ["None", 0] +max_trom_total = ["None", 0] + +max_total_encdec = ["None", 0] rom_table = {} diff --git a/ci/complexity_measurements/parseNewsletterWmops.py b/ci/complexity_measurements/parseNewsletterWmops.py index 3f884fd3aac12b6017de3b0c212924ebbe6a25a1..a00171ead797c3117aaf8e571cd88415480ad47d 100755 --- a/ci/complexity_measurements/parseNewsletterWmops.py +++ b/ci/complexity_measurements/parseNewsletterWmops.py @@ -47,9 +47,9 @@ if __name__ == "__main__": shortDate = sys.argv[4] fullDate = sys.argv[5] -max_enc = ["", 0] -max_dec = ["", 0] -max_total = ["", 0] +max_enc = ["None", 0] +max_dec = ["None", 0] +max_total = ["None", 0] fixedpointScalingFac = 1.0 with open(newsletterFilename, "r") as csvfile: diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 77986464d9b3f14e1ba769af757615af5b11f124..7e426ac0662fc629ef907779fd126e4b03ffb1de 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -49,6 +49,23 @@ JOBS_BASOP_REPO = { "ivas-pytest-compare_ref-long-enc": "Pytest encoder compare to ref LTV", "ivas-pytest-compare_ref-long-enc-lev+10": "Pytest encoder compare to ref LTV +10dB", "ivas-pytest-compare_ref-long-enc-lev-10": "Pytest encoder compare to ref LTV -10dB", + "complexity-ism-in-binaural-out": "ISM in, BINAURAL out", + "complexity-ism-in-binaural_room_ir-out": "ISM in, BINAURAL_ROOM_IR out", + "complexity-ism-in-ext-out": "ISM in, EXT out", + "complexity-sba-hoa3-in-hoa3-out": "HOA3 in, HOA3 out", + "complexity-sba-hoa3-in-binaural-out": "HOA3 in, BINAURAL out", + "complexity-sba-hoa3-in-binaural_room_ir-out": "HOA3 in, BINAURAL_ROOM_IR out", + "complexity-mc-in-7_1_4-out": "MC in, 7_1_4 out", + "complexity-mc-in-binaural-out": "MC in, BINAURAL out", + "complexity-mc-in-binaural_room_ir-out": "MC in, BINAURAL_ROOM_IR out", + "complexity-masa-in-ext-out": "MASA in, EXT out", + "complexity-masa-in-binaural-out": "MASA in, BINAURAL out", + "complexity-masa-in-hoa3-out": "MASA in, HOA3 out", + "complexity-omasa-in-binaural-out": "OMASA in, BINAURAL out", + "complexity-omasa-in-hoa3-out": "OMASA in HOA3 out", + "complexity-StereoDmxEVS-stereo-in-mono-out": "StereoDmxEVS", + "complexity-osba-in-binaural-out": "OSBA in, BINAURAL out", + "complexity-osba-in-binaural_room_ir-out": "OSBA in, BINAURAL_ROOM_IR out", "complexity-stereo-in-stereo-out": "Stereo in, Stereo out", } diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index d0c5bed7a2b11e51cec100315ac760377148e240..e4116cb76d25dfda0d126c1acc5cf203015d5a58 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -45,6 +45,9 @@ COLS_2_THRESHOLDS = { "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), } +OUTFILE_CRASHES = "changes_crashes.csv" +OUTFILE_SCORES = "changes_{}.csv" + def main(args): df_curr = pd.read_csv(args.csv_current, sep=";") @@ -52,7 +55,9 @@ def main(args): df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"]) # remove leading path from testcase names for better readability - df_merged["testcase"] = [pathlib.Path(tc).name for tc in df_merged["testcase"]] + df_merged["testcase"] = [ + pathlib.Path(tc).name.split("::")[-1] for tc in df_merged["testcase"] + ] # this is for printing the whole testcase names pd.options.display.max_colwidth = 200 @@ -69,15 +74,24 @@ def main(args): ) display_cols = ["testcase", col_curr, col_prev] + df_crashes_introduced = df_merged[mask_crash_introduced][display_cols].reset_index( + drop=True + ) + df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") + if sum(mask_crash_introduced) > 0: regressions_found = True print("---------------Testcases that introduced new crashes---------------") - print(df_merged[mask_crash_introduced][display_cols].reset_index(drop=True)) + print(df_crashes_introduced) print() if args.show_improvements and sum(mask_crash_fixed) > 0: + df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index( + drop=True + ) + df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") print("---------------Testcases that fixed crashes---------------") - print(df_merged[mask_crash_fixed][display_cols].reset_index(drop=True)) + print(df_crashes_fixed) print() # remove columns with ERRORs in any of the csv files before comparing the numerical columns @@ -88,31 +102,35 @@ def main(args): for col in args.columns_to_compare: col_curr = f"{col}-curr" col_prev = f"{col}-prev" - diff = df_merged[col_curr] - df_merged[col_prev] + col_diff = f"{col}-diff" + df_merged[col_diff] = df_merged[col_curr] - df_merged[col_prev] thresh = COLS_2_THRESHOLDS[col] # invert sign of difference for "higher is better" metrics - if thresh < 0: - diff *= -1 - + fac = -1 if thresh < 0 else 1 thresh = abs(thresh) - mask_worse = diff > thresh - mask_better = diff < -thresh + mask_worse = (df_merged[col_diff] * fac) > thresh + mask_better = (df_merged[col_diff] * fac) < -thresh - display_cols = ["testcase", col_curr, col_prev] + display_cols = ["testcase", col_curr, col_prev, col_diff] + outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) + df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) + df_worse.to_csv(outfile, sep=";") if sum(mask_worse) > 0: regressions_found = True print( f"---------------Testcases that got worse wrt to {col}---------------" ) - print(df_merged[mask_worse][display_cols].reset_index(drop=True)) + print(df_worse) print() if args.show_improvements and sum(mask_better) > 0: + df_better = df_merged[mask_better][display_cols].reset_index(drop=True) + df_better.to_csv(outfile, mode="a", sep=";") print( f"---------------Testcases that got better wrt to {col}---------------" ) - print(df_merged[mask_better][display_cols].reset_index(drop=True)) + print(df_better) print() return int(regressions_found) diff --git a/scripts/prepare_instrumentation.sh b/scripts/prepare_instrumentation.sh index 281b4fba87c2ed3bd1ea38c474814bcf9bc943ce..f72c3f28d573dbd765342fd14252bac2247d6394 100755 --- a/scripts/prepare_instrumentation.sh +++ b/scripts/prepare_instrumentation.sh @@ -49,13 +49,13 @@ while getopts "m:p:h" OPTIONS; do case ${OPTIONS} in m) MODE=${OPTARG^^} - if [ "$MODE" != "FULL" && "$MODE" != "MEM_ONLY" ]; then + if [ "$MODE" != "FULL" -a "$MODE" != "MEM_ONLY" ]; then usage fi ;; p) PROJECT=${OPTARG^^} - if [ "$PROJECT" != "FLOAT" && "$PROJECT" != "BASOP" ]; then + if [ "$PROJECT" != "FLOAT" -a "$PROJECT" != "BASOP" ]; then usage fi ;; @@ -108,7 +108,9 @@ mkdir $targetdir cp -R ../lib_* $targetdir cp -R ../apps $targetdir cp -R ../Makefile $targetdir -cp -R ../CMakeLists.txt $targetdir +if [ "$PROJECT" = "FLOAT" ]; then + cp -R ../CMakeLists.txt $targetdir +fi cp -R ../Workspace_msvc $targetdir # back up #ifdef-list @@ -165,11 +167,14 @@ if coan_exists; then sed -i "/-DWMOPS/d" $ifdef_list sed -i "/-UMEM_COUNT_DETAILS/d" $ifdef_list - coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,isar,rend,util,debug}/!(wmc_auto*).[hc] coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/apps/*.[hc] if [ "$PROJECT" = "FLOAT" ]; then + coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,isar,rend,util,debug}/!(wmc_auto*).[hc] coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/!(wmc_auto*).[hc] coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/fft/!(wmc_auto*).[hc] + else + # same as first call from if, but without "isar" and "debug" to avoid coan warning + coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,rend,util}/!(wmc_auto*).[hc] fi else ./strip_defines_cppp.sh $targetdir $ifdef_list