From 13983a76c311b899c8413e4381c51279d4ae16d4 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 23 May 2024 09:57:08 +0900 Subject: [PATCH 01/10] Restore binary HRTF model files --- .../binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin | 4 ++-- .../binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin | 4 ++-- .../binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin | 4 ++-- .../ivas_binaural_51_brir-lc_16kHz.bin | 4 ++-- .../ivas_binaural_51_brir-lc_32kHz.bin | 4 ++-- .../ivas_binaural_51_brir-lc_48kHz.bin | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin index 97f7cfdf50..9a988eb588 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:41719cf6eeb2b627a48e30a9d0eb0375f2041730ef29c35e23af3162abe37ff7 -size 1971934 +oid sha256:a263c5498fc7ec0407c4f448e78e1f0aaf97163234ef44f2f79a1dfcb5adc37f +size 1968026 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin index 0c87defe8c..73f151217e 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:84545fbcc27f8123f4c98f2188794797565e00b3ac76d133f6c24eebab9f7f29 -size 2430738 +oid sha256:8622b0f7969a3126f8ab147c10392916b5e9cbe8d8bd9c04798abd3836e36c76 +size 2431246 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin index 68cae9bc10..d847e74613 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ef00e531e26972dd3788ad98378a6224f288d90ddd0536aefe546dccbd835bc2 -size 2673906 +oid sha256:30bcf48f0c851d4e676d5b92be7eb3032442f5106a1072ef8fc9b335d60d0fbf +size 2673694 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_16kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_16kHz.bin index 13787ec393..20c688d2da 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_16kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_16kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:420704e8c05cd809f8448c1797b3cd2e69df0deb7ebe229b51ff0b54bbf99339 -size 1771126 +oid sha256:c839dcbdcfb7de23b725325770a07de3fe0144dc56f0ba7b4a9627bc912c2547 +size 1771166 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_32kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_32kHz.bin index 9b7593a1d5..c92f4c2af0 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_32kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_32kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5f26a82e286d6cadac49bd448bb4b621ff6381cb9c0be91a85b6723761f1c676 -size 2107642 +oid sha256:9c88daaf248bb36ac11d0aa320b18ce87019ef6ad9fab6ac2b7f064b27048aac +size 2107682 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_48kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_48kHz.bin index 8e7ad080a4..61f226bca4 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_48kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_48kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7d72291e29de622a681635c11f8acceabbdfaec91e81a82146859b8b0d5ca2c8 -size 2343610 +oid sha256:ed7e4b9f3306e7aecd2edf19b7ca9fb62240031fa248b26ad7c606fef36a20fe +size 2343650 -- GitLab From e48ce10426f1fa0c286c7e3258cd6f153c34b00d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 29 May 2024 17:00:34 +0200 Subject: [PATCH 02/10] skip aeid test as not supported in BASOP --- tests/renderer/test_renderer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 3c9b95dbaf..0c1ce804b8 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -127,6 +127,7 @@ def test_ambisonics_binaural_headrotation( ) +@pytest.mark.skip(reason="Not supported for BASOP code currently") @pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -- GitLab From 5387710fe3e612468b9d5d5e1ca5adb441c02eec Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 3 Jun 2024 15:27:30 +0200 Subject: [PATCH 03/10] update name of merged csv file --- ci/basop-pages/create_report_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index d8c3373b54..065a2eced8 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -27,7 +27,7 @@ Comparing:
-- GitLab From 89cb800444ced18495779a11d880fd7b2694d8a7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 08:06:41 +0200 Subject: [PATCH 04/10] remove anyway empty coverage section for now --- ci/basop-pages/basop_index.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/basop-pages/basop_index.html b/ci/basop-pages/basop_index.html index 4c5202ad35..8688232fe4 100644 --- a/ci/basop-pages/basop_index.html +++ b/ci/basop-pages/basop_index.html @@ -13,10 +13,4 @@
  • ivas-pytest-mld-long-dec-lev-10
    • -

    Test Coverage

    - -
    - tbd... -
    - -- GitLab From b34be1b91e0279d77e519f600781323650c77dbd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:02:57 +0200 Subject: [PATCH 05/10] change colouring and add arrows --- ci/basop-pages/create_report_pages.py | 56 ++++++++++++++++++--------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index d8c3373b54..de3959aef9 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -11,11 +11,14 @@ SUBPAGE_TMPL_CSS = """ overflow:hidden;padding:10px 5px;word-break:normal;} .tbase th{border-color:black;border-style:solid;border-width:1px;font-family:sans-serif;font-size:14px; font-weight:normal;overflow:hidden;padding:10px 5px;word-break:normal;} -.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;vertical-align:top} -.tbase .tcenter{font-weight:bold;text-align:center;vertical-align:top} -.tbase .tleft{text-align:left;vertical-align:top} -.tbase .tincrease{text-align:left;vertical-align:top;background-color:#ff0000;border-color:inherit;font-weight:bold;} -.tbase .treduce{text-align:left;vertical-align:top;background-color:#00ff00;border-color:inherit;font-weight:bold;} +.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;} +.tbase .tcenter{font-weight:bold;text-align:center;} +.tbase .tleft{text-align:left;horizontal-align:bottom} +.tbase .tincrease{text-align:left;background-color:#ff0000;border-color:inherit;font-weight:bold;} +.tbase .treduce{text-align:left;background-color:#00ff00;border-color:inherit;font-weight:bold;} + +.arrowup {font-weight:bold;background-color:#ff0000;font-size:200%;} +.arrowdown {font-weight:bold;background-color:#00ff00;font-size:200%;} """ @@ -54,6 +57,11 @@ TH_TMPL_GLOBAL = '{}' TH_TMPL_DIFFERENTIAL = '{}' TH_TMPL_SECOND_ROW = '{}' +ARROW_UP = '⬀' +ARROW_DOWN = '⬂' +# ARROW_UP = '⬀' +# ARROW_DOWN = '⬂' + # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] @@ -127,20 +135,30 @@ def tr_from_row(row, id_current, id_previous): prev = row[f"{c}-{id_previous}"] curr = row[f"{c}-{id_current}"] - # use red background if increase, green if decrease, white if same - td_tmpl = TD_TMPL_NORMAL - try: - if float(curr) > float(prev): - td_tmpl = TD_TMPL_INCREASE - if float(curr) < float(prev): - td_tmpl = TD_TMPL_REDUCE - except ValueError: - # if we land here, one of the cells is not a number, this indicates a crash - # or some error in the scripts, so mark with red as well - td_tmpl = TD_TMPL_INCREASE - - tr.append(td_tmpl.format(row[f"{c}-{id_previous}"])) - tr.append(td_tmpl.format(row[f"{c}-{id_current}"])) + if c == "Result": + # print errors in bold red font + td_tmpl = TD_TMPL_INCREASE if prev == "ERROR" else TD_TMPL_NORMAL + tr.append(td_tmpl.format(prev)) + td_tmpl = TD_TMPL_INCREASE if curr == "ERROR" else TD_TMPL_NORMAL + tr.append(td_tmpl.format(curr)) + else: + td_tmpl_curr = TD_TMPL_NORMAL + td_tmpl_prev = TD_TMPL_NORMAL + try: + if float(curr) > float(prev): + curr += f" {ARROW_UP}" + td_tmpl_curr = TD_TMPL_INCREASE + elif float(curr) < float(prev): + curr += f" {ARROW_DOWN}" + td_tmpl_curr = TD_TMPL_REDUCE + except ValueError: + # if we land here, one of the cells is not a number, this indicates a crash + # or some error in the scripts, so mark with red as well + td_tmpl_curr = TD_TMPL_INCREASE + td_tmpl_prev = TD_TMPL_INCREASE + + tr.append(td_tmpl_prev.format(prev)) + tr.append(td_tmpl_curr.format(curr)) return TR_TMPL.format("\n".join(tr)) -- GitLab From 1a95d5c32b33696a7048b05b808b5e7dfaa3547e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:04:44 +0200 Subject: [PATCH 06/10] remove obsolete bg color from arrows --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index de3959aef9..eeef815560 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -17,8 +17,8 @@ SUBPAGE_TMPL_CSS = """ .tbase .tincrease{text-align:left;background-color:#ff0000;border-color:inherit;font-weight:bold;} .tbase .treduce{text-align:left;background-color:#00ff00;border-color:inherit;font-weight:bold;} -.arrowup {font-weight:bold;background-color:#ff0000;font-size:200%;} -.arrowdown {font-weight:bold;background-color:#00ff00;font-size:200%;} +.arrowup {font-weight:bold;font-size:200%;} +.arrowdown {font-weight:bold;font-size:200%;} """ -- GitLab From 02382e64020cd647c93a772a5126a9cdef5c1a6d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:10:03 +0200 Subject: [PATCH 07/10] change colours for better readability --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index eeef815560..7b2476d7c1 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -14,8 +14,8 @@ SUBPAGE_TMPL_CSS = """ .tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;} .tbase .tcenter{font-weight:bold;text-align:center;} .tbase .tleft{text-align:left;horizontal-align:bottom} -.tbase .tincrease{text-align:left;background-color:#ff0000;border-color:inherit;font-weight:bold;} -.tbase .treduce{text-align:left;background-color:#00ff00;border-color:inherit;font-weight:bold;} +.tbase .tincrease{text-align:left;background-color:#ff5500;border-color:inherit;font-weight:bold;} +.tbase .treduce{text-align:left;background-color:#acff00;border-color:inherit;font-weight:bold;} .arrowup {font-weight:bold;font-size:200%;} .arrowdown {font-weight:bold;font-size:200%;} -- GitLab From 5df73a612381e9d50fc44b9b2b88123263704e69 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:33:47 +0200 Subject: [PATCH 08/10] add explicit headings --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 7b2476d7c1..e3534d895b 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -86,8 +86,8 @@ def create_subpage( table_header_a = "".join([TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL]) table_header_b = list() for c in COLUMNS_DIFFERENTIAL: - table_header_b.append(TH_TMPL_SECOND_ROW.format(id_previous)) - table_header_b.append(TH_TMPL_SECOND_ROW.format(id_current)) + table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Previous Run
    ID: {id_previous}")) + table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Current Run
    ID: {id_current}")) table_header_b = "".join(table_header_b) table_body = "\n".join( tr_from_row(row, id_current, id_previous) for row in merged_reports -- GitLab From d80c699c26109b9f7e035cc99da1051f0f5eb329 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 17:00:23 +0200 Subject: [PATCH 09/10] add explanation and better sort --- ci/basop-pages/create_report_pages.py | 54 ++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index e3534d895b..f635c04614 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -34,8 +34,37 @@ Comparing:
    -Table is sorted by Difference in MLD with ERRORs or missing values ("None", "") being on top additionally. +How is the table sorted? +
      +
    • Cases with result ERROR or invalid/missing values for the numerical measures are given first
      • +
    • Next are all cases with MLD(current) != MLD(previous), sorted in descending order (biggest MLD increase comes first)
      • +
    • All cases with no difference in MLD are at the bottom
      • +
    +
    +What do the colours indicate +
      +
    • + Red background: +
        +
      • This testcases either triggered an ERROR in the pytest run (probably a crash in the codec) or did not give valid values for the numerical measures (probably an error in the scripts).
        • +
      • For this testcase the MLD increased wrt the previous run - MLD(current) > MLD(previous)
        • +
      +
      • +
    • + Green background: +
        +
      • For this testcase the MLD decreased wrt the previous run - MLD(current) < MLD(previous)
        • +
      +
      • +

    +How to interpret the Result column? +
      +
    • ERROR: An error occured during test run. It should be checked if a crash in the codec occured.
      • +
    • FAIL: An MLD value > 0 was reported. This is to be expected, the test just reports a failure due to how the tests are currently implemented.
      • +
    • PASS: MLD value of 0 was reported. This should only be the case for some special operating points and could indicate that parts of the codec are not being converted yet.
      • +
    + @@ -59,8 +88,6 @@ TH_TMPL_SECOND_ROW = '' ARROW_UP = '⬀' ARROW_DOWN = '⬂' -# ARROW_UP = '⬀' -# ARROW_DOWN = '⬂' # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table @@ -183,15 +210,24 @@ def merge_and_cleanup_mld_reports( mld_col_curr = f"MLD-{id_current}" mld_col_prev = f"MLD-{id_previous}" - # sort based on difference in MLD between current and previous run - # put cases with "None" at the top of the list def sort_func(x): - vals_missing = ["None", ""] - - if x[mld_col_curr] in vals_missing or x[mld_col_prev] in vals_missing: + """ + Sort function for the rows. Puts missing or invalid values on top as those usually + indicate crashes. Then sorts by MLD difference in descending order. MLD diffs of zero + are uninteresting and are put last. + """ + try: + float(x[mld_col_curr]) + float(x[mld_col_prev]) + except ValueError: + # Value is no valid floating point value return float("inf") - return float(x[mld_col_curr]) - float(x[mld_col_prev]) + diff = float(x[mld_col_curr]) - float(x[mld_col_prev]) + if diff == 0: + diff = float("-inf") + + return diff merged = sorted(merged, key=sort_func, reverse=True) -- GitLab From 924d6dc158a5ee25943f295fa055cc939a217f5e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 17:14:26 +0200 Subject: [PATCH 10/10] sort so that any diffs are above no diffs --- ci/basop-pages/create_report_pages.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index f635c04614..f6f2aa0778 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -1,6 +1,7 @@ import csv import pathlib import argparse +from functools import partial CSV_DELIM = ";" @@ -94,7 +95,7 @@ ARROW_DOWN = '⬂' COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] COLUMNS_GLOBAL = COLUMNS[:1] COLUMNS_DIFFERENTIAL = COLUMNS[1:] - +COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:] def create_subpage( html_out, @@ -210,7 +211,7 @@ def merge_and_cleanup_mld_reports( mld_col_curr = f"MLD-{id_current}" mld_col_prev = f"MLD-{id_previous}" - def sort_func(x): + def sort_func(x, other_col_pairs): """ Sort function for the rows. Puts missing or invalid values on top as those usually indicate crashes. Then sorts by MLD difference in descending order. MLD diffs of zero @@ -224,12 +225,24 @@ def merge_and_cleanup_mld_reports( return float("inf") diff = float(x[mld_col_curr]) - float(x[mld_col_prev]) + + # if no diff in mld col found, check if there is a diff in any other measure if diff == 0: diff = float("-inf") + diff_other = 0 + for col_pair in other_col_pairs: + col_prev = col_pair[0] + col_curr = col_pair[1] + diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + + if diff_other > 0: + diff = -1000000 + return diff - merged = sorted(merged, key=sort_func, reverse=True) + other_col_pairs = [(f"{col}-{id_previous}", f"{col}-{id_current}") for col in COLUMNS_DIFFERENTIAL_NOT_MLD] + merged = sorted(merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True) # remove the unecessary whole path from the testcase names for row in merged: -- GitLab
    {}