diff --git a/ci/basop-pages/basop_index.html b/ci/basop-pages/basop_index.html index 4c5202ad352c5821de0e106eee7100ac95d8fcd5..8688232fe49e0da273958c53986e9d3bcdafc2c2 100644 --- a/ci/basop-pages/basop_index.html +++ b/ci/basop-pages/basop_index.html @@ -13,10 +13,4 @@
  • ivas-pytest-mld-long-dec-lev-10
    • -

    Test Coverage

    - -
    - tbd... -
    - diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index d8c3373b5432dfdfd371e30d42452d4e438ffa15..e3557e252ba32c9b33962f0f7cad677d05d1624d 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -1,6 +1,7 @@ import csv import pathlib import argparse +from functools import partial CSV_DELIM = ";" @@ -11,11 +12,14 @@ SUBPAGE_TMPL_CSS = """ overflow:hidden;padding:10px 5px;word-break:normal;} .tbase th{border-color:black;border-style:solid;border-width:1px;font-family:sans-serif;font-size:14px; font-weight:normal;overflow:hidden;padding:10px 5px;word-break:normal;} -.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;vertical-align:top} -.tbase .tcenter{font-weight:bold;text-align:center;vertical-align:top} -.tbase .tleft{text-align:left;vertical-align:top} -.tbase .tincrease{text-align:left;vertical-align:top;background-color:#ff0000;border-color:inherit;font-weight:bold;} -.tbase .treduce{text-align:left;vertical-align:top;background-color:#00ff00;border-color:inherit;font-weight:bold;} +.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;} +.tbase .tcenter{font-weight:bold;text-align:center;} +.tbase .tleft{text-align:left;horizontal-align:bottom} +.tbase .tincrease{text-align:left;background-color:#ff5500;border-color:inherit;font-weight:bold;} +.tbase .treduce{text-align:left;background-color:#acff00;border-color:inherit;font-weight:bold;} + +.arrowup {font-weight:bold;font-size:200%;} +.arrowdown {font-weight:bold;font-size:200%;} """ @@ -27,12 +31,41 @@ Comparing:
    -Table is sorted by Difference in MLD with ERRORs or missing values ("None", "") being on top additionally. +How is the table sorted? + +
    +What do the colours indicate +
    +How to interpret the Result column? + + @@ -54,12 +87,15 @@ TH_TMPL_GLOBAL = '' TH_TMPL_DIFFERENTIAL = '' TH_TMPL_SECOND_ROW = '' +ARROW_UP = '⬀' +ARROW_DOWN = '⬂' + # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] COLUMNS_GLOBAL = COLUMNS[:1] COLUMNS_DIFFERENTIAL = COLUMNS[1:] - +COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:] def create_subpage( html_out, @@ -78,8 +114,8 @@ def create_subpage( table_header_a = "".join([TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL]) table_header_b = list() for c in COLUMNS_DIFFERENTIAL: - table_header_b.append(TH_TMPL_SECOND_ROW.format(id_previous)) - table_header_b.append(TH_TMPL_SECOND_ROW.format(id_current)) + table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Previous Run
    ID: {id_previous}")) + table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Current Run
    ID: {id_current}")) table_header_b = "".join(table_header_b) table_body = "\n".join( tr_from_row(row, id_current, id_previous) for row in merged_reports @@ -127,20 +163,30 @@ def tr_from_row(row, id_current, id_previous): prev = row[f"{c}-{id_previous}"] curr = row[f"{c}-{id_current}"] - # use red background if increase, green if decrease, white if same - td_tmpl = TD_TMPL_NORMAL - try: - if float(curr) > float(prev): - td_tmpl = TD_TMPL_INCREASE - if float(curr) < float(prev): - td_tmpl = TD_TMPL_REDUCE - except ValueError: - # if we land here, one of the cells is not a number, this indicates a crash - # or some error in the scripts, so mark with red as well - td_tmpl = TD_TMPL_INCREASE - - tr.append(td_tmpl.format(row[f"{c}-{id_previous}"])) - tr.append(td_tmpl.format(row[f"{c}-{id_current}"])) + if c == "Result": + # print errors in bold red font + td_tmpl = TD_TMPL_INCREASE if prev == "ERROR" else TD_TMPL_NORMAL + tr.append(td_tmpl.format(prev)) + td_tmpl = TD_TMPL_INCREASE if curr == "ERROR" else TD_TMPL_NORMAL + tr.append(td_tmpl.format(curr)) + else: + td_tmpl_curr = TD_TMPL_NORMAL + td_tmpl_prev = TD_TMPL_NORMAL + try: + if float(curr) > float(prev): + curr += f" {ARROW_UP}" + td_tmpl_curr = TD_TMPL_INCREASE + elif float(curr) < float(prev): + curr += f" {ARROW_DOWN}" + td_tmpl_curr = TD_TMPL_REDUCE + except ValueError: + # if we land here, one of the cells is not a number, this indicates a crash + # or some error in the scripts, so mark with red as well + td_tmpl_curr = TD_TMPL_INCREASE + td_tmpl_prev = TD_TMPL_INCREASE + + tr.append(td_tmpl_prev.format(prev)) + tr.append(td_tmpl_curr.format(curr)) return TR_TMPL.format("\n".join(tr)) @@ -165,17 +211,38 @@ def merge_and_cleanup_mld_reports( mld_col_curr = f"MLD-{id_current}" mld_col_prev = f"MLD-{id_previous}" - # sort based on difference in MLD between current and previous run - # put cases with "None" at the top of the list - def sort_func(x): - vals_missing = ["None", ""] - - if x[mld_col_curr] in vals_missing or x[mld_col_prev] in vals_missing: + def sort_func(x, other_col_pairs): + """ + Sort function for the rows. Puts missing or invalid values on top as those usually + indicate crashes. Then sorts by MLD difference in descending order. MLD diffs of zero + are uninteresting and are put last. + """ + try: + float(x[mld_col_curr]) + float(x[mld_col_prev]) + except ValueError: + # Value is no valid floating point value return float("inf") - return float(x[mld_col_curr]) - float(x[mld_col_prev]) + diff = float(x[mld_col_curr]) - float(x[mld_col_prev]) + + # if no diff in mld col found, check if there is a diff in any other measure + if diff == 0: + diff = float("-inf") + + diff_other = 0 + for col_pair in other_col_pairs: + col_prev = col_pair[0] + col_curr = col_pair[1] + diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + + if diff_other > 0: + diff = -1000000 + + return diff - merged = sorted(merged, key=sort_func, reverse=True) + other_col_pairs = [(f"{col}-{id_previous}", f"{col}-{id_current}") for col in COLUMNS_DIFFERENTIAL_NOT_MLD] + merged = sorted(merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True) # remove the unecessary whole path from the testcase names for row in merged: diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 4754d09f5c8d834e9f664aa63a73377b60473668..a28c12675e6d223ce6474265c05f2af932879cef 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -23,6 +23,8 @@ JOBS_FLOAT_REPO = [ ] JOBS_BASOP_REPO = [ "ivas-pytest-mld-long-dec", + "ivas-pytest-mld-long-dec-lev+10", + "ivas-pytest-mld-long-dec-lev-10", ] JOBS_FOR_PROJECT_ID = {
    {}{}{}