From e4836a178f4a740febc3fac726961466d4ed1a31 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 29 May 2024 17:00:34 +0200 Subject: [PATCH 01/12] skip aeid test as not supported in BASOP --- tests/renderer/test_renderer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 3c9b95dbaf..0c1ce804b8 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -127,6 +127,7 @@ def test_ambisonics_binaural_headrotation( ) +@pytest.mark.skip(reason="Not supported for BASOP code currently") @pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -- GitLab From 26bed8066cd6c47f508ee457746c292d07833a12 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 3 Jun 2024 15:27:30 +0200 Subject: [PATCH 02/12] update name of merged csv file --- ci/basop-pages/create_report_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index d8c3373b54..065a2eced8 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -27,7 +27,7 @@ Comparing:
-- GitLab From cdca0f45157d512124e66f168c4760fa163e1696 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 08:06:41 +0200 Subject: [PATCH 03/12] remove anyway empty coverage section for now --- ci/basop-pages/basop_index.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/basop-pages/basop_index.html b/ci/basop-pages/basop_index.html index 4c5202ad35..8688232fe4 100644 --- a/ci/basop-pages/basop_index.html +++ b/ci/basop-pages/basop_index.html @@ -13,10 +13,4 @@
  • ivas-pytest-mld-long-dec-lev-10
    • -

    Test Coverage

    - -
    - tbd... -
    - -- GitLab From b43ffb28de0eec4a7e57a9fefec819223ed279e2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:02:57 +0200 Subject: [PATCH 04/12] change colouring and add arrows --- ci/basop-pages/create_report_pages.py | 56 ++++++++++++++++++--------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 065a2eced8..a21d4a84ed 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -11,11 +11,14 @@ SUBPAGE_TMPL_CSS = """ overflow:hidden;padding:10px 5px;word-break:normal;} .tbase th{border-color:black;border-style:solid;border-width:1px;font-family:sans-serif;font-size:14px; font-weight:normal;overflow:hidden;padding:10px 5px;word-break:normal;} -.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;vertical-align:top} -.tbase .tcenter{font-weight:bold;text-align:center;vertical-align:top} -.tbase .tleft{text-align:left;vertical-align:top} -.tbase .tincrease{text-align:left;vertical-align:top;background-color:#ff0000;border-color:inherit;font-weight:bold;} -.tbase .treduce{text-align:left;vertical-align:top;background-color:#00ff00;border-color:inherit;font-weight:bold;} +.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;} +.tbase .tcenter{font-weight:bold;text-align:center;} +.tbase .tleft{text-align:left;horizontal-align:bottom} +.tbase .tincrease{text-align:left;background-color:#ff0000;border-color:inherit;font-weight:bold;} +.tbase .treduce{text-align:left;background-color:#00ff00;border-color:inherit;font-weight:bold;} + +.arrowup {font-weight:bold;background-color:#ff0000;font-size:200%;} +.arrowdown {font-weight:bold;background-color:#00ff00;font-size:200%;} """ @@ -54,6 +57,11 @@ TH_TMPL_GLOBAL = '{}' TH_TMPL_DIFFERENTIAL = '{}' TH_TMPL_SECOND_ROW = '{}' +ARROW_UP = '⬀' +ARROW_DOWN = '⬂' +# ARROW_UP = '⬀' +# ARROW_DOWN = '⬂' + # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] @@ -127,20 +135,30 @@ def tr_from_row(row, id_current, id_previous): prev = row[f"{c}-{id_previous}"] curr = row[f"{c}-{id_current}"] - # use red background if increase, green if decrease, white if same - td_tmpl = TD_TMPL_NORMAL - try: - if float(curr) > float(prev): - td_tmpl = TD_TMPL_INCREASE - if float(curr) < float(prev): - td_tmpl = TD_TMPL_REDUCE - except ValueError: - # if we land here, one of the cells is not a number, this indicates a crash - # or some error in the scripts, so mark with red as well - td_tmpl = TD_TMPL_INCREASE - - tr.append(td_tmpl.format(row[f"{c}-{id_previous}"])) - tr.append(td_tmpl.format(row[f"{c}-{id_current}"])) + if c == "Result": + # print errors in bold red font + td_tmpl = TD_TMPL_INCREASE if prev == "ERROR" else TD_TMPL_NORMAL + tr.append(td_tmpl.format(prev)) + td_tmpl = TD_TMPL_INCREASE if curr == "ERROR" else TD_TMPL_NORMAL + tr.append(td_tmpl.format(curr)) + else: + td_tmpl_curr = TD_TMPL_NORMAL + td_tmpl_prev = TD_TMPL_NORMAL + try: + if float(curr) > float(prev): + curr += f" {ARROW_UP}" + td_tmpl_curr = TD_TMPL_INCREASE + elif float(curr) < float(prev): + curr += f" {ARROW_DOWN}" + td_tmpl_curr = TD_TMPL_REDUCE + except ValueError: + # if we land here, one of the cells is not a number, this indicates a crash + # or some error in the scripts, so mark with red as well + td_tmpl_curr = TD_TMPL_INCREASE + td_tmpl_prev = TD_TMPL_INCREASE + + tr.append(td_tmpl_prev.format(prev)) + tr.append(td_tmpl_curr.format(curr)) return TR_TMPL.format("\n".join(tr)) -- GitLab From fcd3bb481eb98e1fa7e815c99263cc9093560790 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:04:44 +0200 Subject: [PATCH 05/12] remove obsolete bg color from arrows --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index a21d4a84ed..d5eaf5841d 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -17,8 +17,8 @@ SUBPAGE_TMPL_CSS = """ .tbase .tincrease{text-align:left;background-color:#ff0000;border-color:inherit;font-weight:bold;} .tbase .treduce{text-align:left;background-color:#00ff00;border-color:inherit;font-weight:bold;} -.arrowup {font-weight:bold;background-color:#ff0000;font-size:200%;} -.arrowdown {font-weight:bold;background-color:#00ff00;font-size:200%;} +.arrowup {font-weight:bold;font-size:200%;} +.arrowdown {font-weight:bold;font-size:200%;} """ -- GitLab From 225ca8c57923d21d789c196269633ddb768660ce Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:10:03 +0200 Subject: [PATCH 06/12] change colours for better readability --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index d5eaf5841d..129dac100b 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -14,8 +14,8 @@ SUBPAGE_TMPL_CSS = """ .tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;} .tbase .tcenter{font-weight:bold;text-align:center;} .tbase .tleft{text-align:left;horizontal-align:bottom} -.tbase .tincrease{text-align:left;background-color:#ff0000;border-color:inherit;font-weight:bold;} -.tbase .treduce{text-align:left;background-color:#00ff00;border-color:inherit;font-weight:bold;} +.tbase .tincrease{text-align:left;background-color:#ff5500;border-color:inherit;font-weight:bold;} +.tbase .treduce{text-align:left;background-color:#acff00;border-color:inherit;font-weight:bold;} .arrowup {font-weight:bold;font-size:200%;} .arrowdown {font-weight:bold;font-size:200%;} -- GitLab From ac4c0d3d9d7154209ce9bb0be66ab8f001139efd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:33:47 +0200 Subject: [PATCH 07/12] add explicit headings --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 129dac100b..316cf9a6a2 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -86,8 +86,8 @@ def create_subpage( table_header_a = "".join([TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL]) table_header_b = list() for c in COLUMNS_DIFFERENTIAL: - table_header_b.append(TH_TMPL_SECOND_ROW.format(id_previous)) - table_header_b.append(TH_TMPL_SECOND_ROW.format(id_current)) + table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Previous Run
    ID: {id_previous}")) + table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Current Run
    ID: {id_current}")) table_header_b = "".join(table_header_b) table_body = "\n".join( tr_from_row(row, id_current, id_previous) for row in merged_reports -- GitLab From 965d19b84144c8bf8cab30c0d7dffa4cbc0d4364 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 17:00:23 +0200 Subject: [PATCH 08/12] add explanation and better sort --- ci/basop-pages/create_report_pages.py | 54 ++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 316cf9a6a2..a03747c9b3 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -34,8 +34,37 @@ Comparing:
    -Table is sorted by Difference in MLD with ERRORs or missing values ("None", "") being on top additionally. +How is the table sorted? +
      +
    • Cases with result ERROR or invalid/missing values for the numerical measures are given first
      • +
    • Next are all cases with MLD(current) != MLD(previous), sorted in descending order (biggest MLD increase comes first)
      • +
    • All cases with no difference in MLD are at the bottom
      • +
    +
    +What do the colours indicate +
      +
    • + Red background: +
        +
      • This testcases either triggered an ERROR in the pytest run (probably a crash in the codec) or did not give valid values for the numerical measures (probably an error in the scripts).
        • +
      • For this testcase the MLD increased wrt the previous run - MLD(current) > MLD(previous)
        • +
      +
      • +
    • + Green background: +
        +
      • For this testcase the MLD decreased wrt the previous run - MLD(current) < MLD(previous)
        • +
      +
      • +

    +How to interpret the Result column? +
      +
    • ERROR: An error occured during test run. It should be checked if a crash in the codec occured.
      • +
    • FAIL: An MLD value > 0 was reported. This is to be expected, the test just reports a failure due to how the tests are currently implemented.
      • +
    • PASS: MLD value of 0 was reported. This should only be the case for some special operating points and could indicate that parts of the codec are not being converted yet.
      • +
    + @@ -59,8 +88,6 @@ TH_TMPL_SECOND_ROW = '' ARROW_UP = '⬀' ARROW_DOWN = '⬂' -# ARROW_UP = '⬀' -# ARROW_DOWN = '⬂' # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table @@ -183,15 +210,24 @@ def merge_and_cleanup_mld_reports( mld_col_curr = f"MLD-{id_current}" mld_col_prev = f"MLD-{id_previous}" - # sort based on difference in MLD between current and previous run - # put cases with "None" at the top of the list def sort_func(x): - vals_missing = ["None", ""] - - if x[mld_col_curr] in vals_missing or x[mld_col_prev] in vals_missing: + """ + Sort function for the rows. Puts missing or invalid values on top as those usually + indicate crashes. Then sorts by MLD difference in descending order. MLD diffs of zero + are uninteresting and are put last. + """ + try: + float(x[mld_col_curr]) + float(x[mld_col_prev]) + except ValueError: + # Value is no valid floating point value return float("inf") - return float(x[mld_col_curr]) - float(x[mld_col_prev]) + diff = float(x[mld_col_curr]) - float(x[mld_col_prev]) + if diff == 0: + diff = float("-inf") + + return diff merged = sorted(merged, key=sort_func, reverse=True) -- GitLab From 53b7a93dd709dea82c70e68cebbb37fb8836ccc8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 17:14:26 +0200 Subject: [PATCH 09/12] sort so that any diffs are above no diffs --- ci/basop-pages/create_report_pages.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index a03747c9b3..6ad35c09f1 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -1,6 +1,7 @@ import csv import pathlib import argparse +from functools import partial CSV_DELIM = ";" @@ -94,7 +95,7 @@ ARROW_DOWN = '⬂' COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] COLUMNS_GLOBAL = COLUMNS[:1] COLUMNS_DIFFERENTIAL = COLUMNS[1:] - +COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:] def create_subpage( html_out, @@ -210,7 +211,7 @@ def merge_and_cleanup_mld_reports( mld_col_curr = f"MLD-{id_current}" mld_col_prev = f"MLD-{id_previous}" - def sort_func(x): + def sort_func(x, other_col_pairs): """ Sort function for the rows. Puts missing or invalid values on top as those usually indicate crashes. Then sorts by MLD difference in descending order. MLD diffs of zero @@ -224,12 +225,24 @@ def merge_and_cleanup_mld_reports( return float("inf") diff = float(x[mld_col_curr]) - float(x[mld_col_prev]) + + # if no diff in mld col found, check if there is a diff in any other measure if diff == 0: diff = float("-inf") + diff_other = 0 + for col_pair in other_col_pairs: + col_prev = col_pair[0] + col_curr = col_pair[1] + diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + + if diff_other > 0: + diff = -1000000 + return diff - merged = sorted(merged, key=sort_func, reverse=True) + other_col_pairs = [(f"{col}-{id_previous}", f"{col}-{id_current}") for col in COLUMNS_DIFFERENTIAL_NOT_MLD] + merged = sorted(merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True) # remove the unecessary whole path from the testcase names for row in merged: -- GitLab From e8dceb4d5a133dc8a112c5e63e3ad79c34629516 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Jun 2024 08:33:02 +0200 Subject: [PATCH 10/12] change small part in description --- ci/basop-pages/create_report_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 6ad35c09f1..e3557e252b 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -63,7 +63,7 @@ Comparing:
    • ERROR: An error occured during test run. It should be checked if a crash in the codec occured.
    • FAIL: An MLD value > 0 was reported. This is to be expected, the test just reports a failure due to how the tests are currently implemented.
      • -
    • PASS: MLD value of 0 was reported. This should only be the case for some special operating points and could indicate that parts of the codec are not being converted yet.
      • +
    • PASS: MLD value of 0 was reported. This should only be the case for some special operating points and could indicate that parts of the codec are not (fully) converted yet.
    -- GitLab From b862260f1e2461aaf7c95ae737b1657ea5fa676e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Jun 2024 08:55:54 +0200 Subject: [PATCH 11/12] Revert "skip aeid test as not supported in BASOP" This reverts commit e48ce10426f1fa0c286c7e3258cd6f153c34b00d. --- tests/renderer/test_renderer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 0c1ce804b8..3c9b95dbaf 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -127,7 +127,6 @@ def test_ambisonics_binaural_headrotation( ) -@pytest.mark.skip(reason="Not supported for BASOP code currently") @pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -- GitLab From 546074958fa4c4a046e896f6c5c9916a97fe8566 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Jun 2024 08:58:38 +0200 Subject: [PATCH 12/12] add scaled jobs for basop repo --- ci/setup_pages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 4754d09f5c..a28c12675e 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -23,6 +23,8 @@ JOBS_FLOAT_REPO = [ ] JOBS_BASOP_REPO = [ "ivas-pytest-mld-long-dec", + "ivas-pytest-mld-long-dec-lev+10", + "ivas-pytest-mld-long-dec-lev-10", ] JOBS_FOR_PROJECT_ID = { -- GitLab
    {}