From 7299ac9133de87fd9e8f47f6e13d4e3fb0ff07eb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 31 Jul 2024 10:20:49 +0200 Subject: [PATCH 01/10] shorten testcase names and write out results to csv files --- .../basop_check_for_changes_in_testcases.py | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index d0c5bed7a2..54a636ab2a 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -45,6 +45,9 @@ COLS_2_THRESHOLDS = { "MIN_ODG": float(os.environ.get("CI_REGRESSION_THRESH_ODG", -0.05)), } +OUTFILE_CRASHES = "changes_crashes.csv" +OUTFILE_SCORES = "changes_{}.csv" + def main(args): df_curr = pd.read_csv(args.csv_current, sep=";") @@ -52,7 +55,9 @@ def main(args): df_merged = pd.merge(df_curr, df_prev, on="testcase", suffixes=["-curr", "-prev"]) # remove leading path from testcase names for better readability - df_merged["testcase"] = [pathlib.Path(tc).name for tc in df_merged["testcase"]] + df_merged["testcase"] = [ + pathlib.Path(tc).name.split("::")[-1] for tc in df_merged["testcase"] + ] # this is for printing the whole testcase names pd.options.display.max_colwidth = 200 @@ -69,15 +74,24 @@ def main(args): ) display_cols = ["testcase", col_curr, col_prev] - if sum(mask_crash_introduced) > 0: + df_crashes_introduced = df_merged[mask_crash_introduced][display_cols].reset_index( + drop=True + ) + df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") + + if len(df_crashes_introduced) > 0: regressions_found = True print("---------------Testcases that introduced new crashes---------------") - print(df_merged[mask_crash_introduced][display_cols].reset_index(drop=True)) + print(df_crashes_introduced) print() if args.show_improvements and sum(mask_crash_fixed) > 0: + df_crashes_fixed = df_merged[mask_crash_fixed][display_cols].reset_index( + drop=True + ) + df_crashes_fixed.to_csv(OUTFILE_CRASHES, mode="a", sep=";") print("---------------Testcases that fixed crashes---------------") - print(df_merged[mask_crash_fixed][display_cols].reset_index(drop=True)) + print(df_crashes_fixed) print() # remove columns with ERRORs in any of the csv files before comparing the numerical columns @@ -100,19 +114,24 @@ def main(args): mask_better = diff < -thresh display_cols = ["testcase", col_curr, col_prev] + outfile = OUTFILE_SCORES.format(col) + df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) + df_worse.to_csv(outfile, sep=";") if sum(mask_worse) > 0: regressions_found = True print( f"---------------Testcases that got worse wrt to {col}---------------" ) - print(df_merged[mask_worse][display_cols].reset_index(drop=True)) + print(df_worse) print() if args.show_improvements and sum(mask_better) > 0: + df_better = df_merged[mask_better][display_cols].reset_index(drop=True) + df_better.to_csv(outfile, mode="a", sep=";") print( f"---------------Testcases that got better wrt to {col}---------------" ) - print(df_merged[mask_better][display_cols].reset_index(drop=True)) + print(df_better) print() return int(regressions_found) -- GitLab From 8269885adf9f37e7b249ff46394763994cb75894 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 31 Jul 2024 10:30:17 +0200 Subject: [PATCH 02/10] fix for whitespace in col name --- scripts/basop_check_for_changes_in_testcases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 54a636ab2a..4936dd7acc 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -114,7 +114,7 @@ def main(args): mask_better = diff < -thresh display_cols = ["testcase", col_curr, col_prev] - outfile = OUTFILE_SCORES.format(col) + outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) df_worse.to_csv(outfile, sep=";") if sum(mask_worse) > 0: -- GitLab From a44e75459af5826ec2f900b1efd8afa38a516d6f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 31 Jul 2024 11:03:59 +0200 Subject: [PATCH 03/10] add difference of scores to printout and csv output --- scripts/basop_check_for_changes_in_testcases.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/basop_check_for_changes_in_testcases.py b/scripts/basop_check_for_changes_in_testcases.py index 4936dd7acc..e4116cb76d 100644 --- a/scripts/basop_check_for_changes_in_testcases.py +++ b/scripts/basop_check_for_changes_in_testcases.py @@ -79,7 +79,7 @@ def main(args): ) df_crashes_introduced.to_csv(OUTFILE_CRASHES, sep=";") - if len(df_crashes_introduced) > 0: + if sum(mask_crash_introduced) > 0: regressions_found = True print("---------------Testcases that introduced new crashes---------------") print(df_crashes_introduced) @@ -102,18 +102,17 @@ def main(args): for col in args.columns_to_compare: col_curr = f"{col}-curr" col_prev = f"{col}-prev" - diff = df_merged[col_curr] - df_merged[col_prev] + col_diff = f"{col}-diff" + df_merged[col_diff] = df_merged[col_curr] - df_merged[col_prev] thresh = COLS_2_THRESHOLDS[col] # invert sign of difference for "higher is better" metrics - if thresh < 0: - diff *= -1 - + fac = -1 if thresh < 0 else 1 thresh = abs(thresh) - mask_worse = diff > thresh - mask_better = diff < -thresh + mask_worse = (df_merged[col_diff] * fac) > thresh + mask_better = (df_merged[col_diff] * fac) < -thresh - display_cols = ["testcase", col_curr, col_prev] + display_cols = ["testcase", col_curr, col_prev, col_diff] outfile = OUTFILE_SCORES.format(col.replace(" ", "_")) df_worse = df_merged[mask_worse][display_cols].reset_index(drop=True) df_worse.to_csv(outfile, sep=";") -- GitLab From 0c59250f401e6d8e879d80444f5d43f96455b176 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 5 Aug 2024 17:28:18 +0200 Subject: [PATCH 04/10] fix warnings from bash and coan --- scripts/prepare_instrumentation.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_instrumentation.sh b/scripts/prepare_instrumentation.sh index 281b4fba87..18750001aa 100755 --- a/scripts/prepare_instrumentation.sh +++ b/scripts/prepare_instrumentation.sh @@ -49,13 +49,13 @@ while getopts "m:p:h" OPTIONS; do case ${OPTIONS} in m) MODE=${OPTARG^^} - if [ "$MODE" != "FULL" && "$MODE" != "MEM_ONLY" ]; then + if [ "$MODE" != "FULL" -a "$MODE" != "MEM_ONLY" ]; then usage fi ;; p) PROJECT=${OPTARG^^} - if [ "$PROJECT" != "FLOAT" && "$PROJECT" != "BASOP" ]; then + if [ "$PROJECT" != "FLOAT" -a "$PROJECT" != "BASOP" ]; then usage fi ;; @@ -165,11 +165,14 @@ if coan_exists; then sed -i "/-DWMOPS/d" $ifdef_list sed -i "/-UMEM_COUNT_DETAILS/d" $ifdef_list - coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,isar,rend,util,debug}/!(wmc_auto*).[hc] coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/apps/*.[hc] if [ "$PROJECT" = "FLOAT" ]; then + coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,isar,rend,util,debug}/!(wmc_auto*).[hc] coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/!(wmc_auto*).[hc] coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/fft/!(wmc_auto*).[hc] + else + # same as first call from if, but without "isar" to avoid coan warning + coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,rend,util,debug}/!(wmc_auto*).[hc] fi else ./strip_defines_cppp.sh $targetdir $ifdef_list -- GitLab From 1e0d098697dedafd6a2ee00b74485df07e2160b4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 5 Aug 2024 17:59:32 +0200 Subject: [PATCH 05/10] more fixes to avoid warninngs and errors in prepare_instrumentation.sh --- scripts/prepare_instrumentation.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_instrumentation.sh b/scripts/prepare_instrumentation.sh index 18750001aa..f72c3f28d5 100755 --- a/scripts/prepare_instrumentation.sh +++ b/scripts/prepare_instrumentation.sh @@ -108,7 +108,9 @@ mkdir $targetdir cp -R ../lib_* $targetdir cp -R ../apps $targetdir cp -R ../Makefile $targetdir -cp -R ../CMakeLists.txt $targetdir +if [ "$PROJECT" = "FLOAT" ]; then + cp -R ../CMakeLists.txt $targetdir +fi cp -R ../Workspace_msvc $targetdir # back up #ifdef-list @@ -171,8 +173,8 @@ if coan_exists; then coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/!(wmc_auto*).[hc] coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/fft/!(wmc_auto*).[hc] else - # same as first call from if, but without "isar" to avoid coan warning - coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,rend,util,debug}/!(wmc_auto*).[hc] + # same as first call from if, but without "isar" and "debug" to avoid coan warning + coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,rend,util}/!(wmc_auto*).[hc] fi else ./strip_defines_cppp.sh $targetdir $ifdef_list -- GitLab From 455fd169d4df8441fc4e02ad01d9601cd578bed1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 9 Sep 2024 17:04:58 +0200 Subject: [PATCH 06/10] add name for default value in parsing scripts --- .../parseNewsletterRam.py | 18 ++++++------- .../parseNewsletterRom.py | 26 +++++++++---------- .../parseNewsletterWmops.py | 6 ++--- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/ci/complexity_measurements/parseNewsletterRam.py b/ci/complexity_measurements/parseNewsletterRam.py index 6fe1cd8993..55eecff573 100755 --- a/ci/complexity_measurements/parseNewsletterRam.py +++ b/ci/complexity_measurements/parseNewsletterRam.py @@ -48,17 +48,17 @@ if __name__ == "__main__": shortDate = sys.argv[5] fullDate = sys.argv[6] -max_total_enc = ["", 0] -max_total_dec = ["", 0] -max_total_encdec = ["", 0] +max_total_enc = ["None", 0] +max_total_dec = ["None", 0] +max_total_encdec = ["None", 0] -max_stack_enc = ["", 0] -max_stack_dec = ["", 0] -max_stack_encdec = ["", 0] +max_stack_enc = ["None", 0] +max_stack_dec = ["None", 0] +max_stack_encdec = ["None", 0] -max_heap_enc = ["", 0] -max_heap_dec = ["", 0] -max_heap_encdec = ["", 0] +max_heap_enc = ["None", 0] +max_heap_dec = ["None", 0] +max_heap_encdec = ["None", 0] ram_table = {} diff --git a/ci/complexity_measurements/parseNewsletterRom.py b/ci/complexity_measurements/parseNewsletterRom.py index 2416a6fd0b..a4a3df4ecf 100755 --- a/ci/complexity_measurements/parseNewsletterRom.py +++ b/ci/complexity_measurements/parseNewsletterRom.py @@ -49,19 +49,19 @@ if __name__ == "__main__": shortDate = sys.argv[5] fullDate = sys.argv[6] -max_prom_enc = ["", 0] -max_prom_dec = ["", 0] -max_prom_com = ["", 0] -max_prom_rend = ["", 0] -max_prom_total = ["", 0] - -max_trom_enc = ["", 0] -max_trom_dec = ["", 0] -max_trom_com = ["", 0] -max_trom_rend = ["", 0] -max_trom_total = ["", 0] - -max_total_encdec = ["", 0] +max_prom_enc = ["None", 0] +max_prom_dec = ["None", 0] +max_prom_com = ["None", 0] +max_prom_rend = ["None", 0] +max_prom_total = ["None", 0] + +max_trom_enc = ["None", 0] +max_trom_dec = ["None", 0] +max_trom_com = ["None", 0] +max_trom_rend = ["None", 0] +max_trom_total = ["None", 0] + +max_total_encdec = ["None", 0] rom_table = {} diff --git a/ci/complexity_measurements/parseNewsletterWmops.py b/ci/complexity_measurements/parseNewsletterWmops.py index 3f884fd3aa..a00171ead7 100755 --- a/ci/complexity_measurements/parseNewsletterWmops.py +++ b/ci/complexity_measurements/parseNewsletterWmops.py @@ -47,9 +47,9 @@ if __name__ == "__main__": shortDate = sys.argv[4] fullDate = sys.argv[5] -max_enc = ["", 0] -max_dec = ["", 0] -max_total = ["", 0] +max_enc = ["None", 0] +max_dec = ["None", 0] +max_total = ["None", 0] fixedpointScalingFac = 1.0 with open(newsletterFilename, "r") as csvfile: -- GitLab From 33d16a511ce3febe4d2123ca25a436e840c735e1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 13 Sep 2024 16:38:35 +0200 Subject: [PATCH 07/10] add all complexity jobs for BASOP --- ci/setup_pages.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 77986464d9..7e426ac066 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -49,6 +49,23 @@ JOBS_BASOP_REPO = { "ivas-pytest-compare_ref-long-enc": "Pytest encoder compare to ref LTV", "ivas-pytest-compare_ref-long-enc-lev+10": "Pytest encoder compare to ref LTV +10dB", "ivas-pytest-compare_ref-long-enc-lev-10": "Pytest encoder compare to ref LTV -10dB", + "complexity-ism-in-binaural-out": "ISM in, BINAURAL out", + "complexity-ism-in-binaural_room_ir-out": "ISM in, BINAURAL_ROOM_IR out", + "complexity-ism-in-ext-out": "ISM in, EXT out", + "complexity-sba-hoa3-in-hoa3-out": "HOA3 in, HOA3 out", + "complexity-sba-hoa3-in-binaural-out": "HOA3 in, BINAURAL out", + "complexity-sba-hoa3-in-binaural_room_ir-out": "HOA3 in, BINAURAL_ROOM_IR out", + "complexity-mc-in-7_1_4-out": "MC in, 7_1_4 out", + "complexity-mc-in-binaural-out": "MC in, BINAURAL out", + "complexity-mc-in-binaural_room_ir-out": "MC in, BINAURAL_ROOM_IR out", + "complexity-masa-in-ext-out": "MASA in, EXT out", + "complexity-masa-in-binaural-out": "MASA in, BINAURAL out", + "complexity-masa-in-hoa3-out": "MASA in, HOA3 out", + "complexity-omasa-in-binaural-out": "OMASA in, BINAURAL out", + "complexity-omasa-in-hoa3-out": "OMASA in HOA3 out", + "complexity-StereoDmxEVS-stereo-in-mono-out": "StereoDmxEVS", + "complexity-osba-in-binaural-out": "OSBA in, BINAURAL out", + "complexity-osba-in-binaural_room_ir-out": "OSBA in, BINAURAL_ROOM_IR out", "complexity-stereo-in-stereo-out": "Stereo in, Stereo out", } -- GitLab From 3419f8c2b245a9f45092ad8e691fbb9c35a9a2e2 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 19 Sep 2024 09:33:43 +0200 Subject: [PATCH 08/10] Update remove_unsupported_testcases.py --- ci/remove_unsupported_testcases.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index 2a1e9ed755..39ecf34d53 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -61,9 +61,13 @@ TESTCASES = [ "Multi-channel 5_1 at 512 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (CREND)", "Multi-channel 5_1 at 64 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (FastConv)", "Multi-channel 5_1 at 32 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (ParamBin)", + "MASA 1dir 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration", + "MASA 1dir 1 TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", + "MASA 1dir 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration", ] + def remove_testcases(cfg: Path, testcases: list): """ Go through file line by line and copy all testcases except the given ones -- GitLab From 912672cbaca95f50f6f48902dbfabf21617f8373 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 20 Sep 2024 14:25:45 +0200 Subject: [PATCH 09/10] only merge on merge_keys present in both tables --- ci/basop-pages/create_report_pages.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index af937355ef..0a0dcfdb34 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -295,6 +295,7 @@ def merge_tables(tbl1, tbl2, suffix1, suffix2, merge_key, other_keys): for key in other_keys: new_row[f"{key}-{suffix1}"] = row1[key] + found_merge_key_in_both_tbls = False for row2 in tbl2: if row1[merge_key] == row2[merge_key]: new_row[merge_key] = row1[merge_key] @@ -303,9 +304,12 @@ def merge_tables(tbl1, tbl2, suffix1, suffix2, merge_key, other_keys): new_row[f"{key}-{suffix2}"] = row2[key] else: new_row[f"{key}-{suffix2}"] = "" + + found_merge_key_in_both_tbls = True break - merged.append(new_row) + if found_merge_key_in_both_tbls: + merged.append(new_row) return merged -- GitLab From d506919e4055b237e2bf571593e92d7bd79abd8b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 20 Sep 2024 14:26:40 +0200 Subject: [PATCH 10/10] run formatter --- ci/basop-pages/create_report_pages.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 0a0dcfdb34..cf3a3a6472 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -199,11 +199,15 @@ def tr_from_row(row, id_current, id_previous): if float(curr) > float(prev): curr += f" {ARROW_UP}" # increase is bad -> mark in red, execpt for SNR for which it is good -> mark in green - td_tmpl_curr = TD_TMPL_REDUCE if c == "MIN_SSNR" else TD_TMPL_INCREASE + td_tmpl_curr = ( + TD_TMPL_REDUCE if c == "MIN_SSNR" else TD_TMPL_INCREASE + ) elif float(curr) < float(prev): curr += f" {ARROW_DOWN}" # reduce is good -> mark in green, execpt for SNR for which it is bad -> mark in red - td_tmpl_curr = TD_TMPL_INCREASE if c == "MIN_SSNR" else TD_TMPL_REDUCE + td_tmpl_curr = ( + TD_TMPL_INCREASE if c == "MIN_SSNR" else TD_TMPL_REDUCE + ) except ValueError: # if we land here, one of the cells is not a number, this indicates a crash # or some error in the scripts, so mark with red as well @@ -300,7 +304,7 @@ def merge_tables(tbl1, tbl2, suffix1, suffix2, merge_key, other_keys): if row1[merge_key] == row2[merge_key]: new_row[merge_key] = row1[merge_key] for key in other_keys: - if key in row2: # In case key is missing, just insert a blank + if key in row2: # In case key is missing, just insert a blank new_row[f"{key}-{suffix2}"] = row2[key] else: new_row[f"{key}-{suffix2}"] = "" @@ -314,7 +318,6 @@ def merge_tables(tbl1, tbl2, suffix1, suffix2, merge_key, other_keys): return merged - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("html_out") -- GitLab