From 13983a76c311b899c8413e4381c51279d4ae16d4 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 23 May 2024 09:57:08 +0900 Subject: [PATCH 01/27] Restore binary HRTF model files --- .../binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin | 4 ++-- .../binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin | 4 ++-- .../binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin | 4 ++-- .../ivas_binaural_51_brir-lc_16kHz.bin | 4 ++-- .../ivas_binaural_51_brir-lc_32kHz.bin | 4 ++-- .../ivas_binaural_51_brir-lc_48kHz.bin | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin index 97f7cfdf50..9a988eb588 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:41719cf6eeb2b627a48e30a9d0eb0375f2041730ef29c35e23af3162abe37ff7 -size 1971934 +oid sha256:a263c5498fc7ec0407c4f448e78e1f0aaf97163234ef44f2f79a1dfcb5adc37f +size 1968026 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin index 0c87defe8c..73f151217e 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:84545fbcc27f8123f4c98f2188794797565e00b3ac76d133f6c24eebab9f7f29 -size 2430738 +oid sha256:8622b0f7969a3126f8ab147c10392916b5e9cbe8d8bd9c04798abd3836e36c76 +size 2431246 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin index 68cae9bc10..d847e74613 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ef00e531e26972dd3788ad98378a6224f288d90ddd0536aefe546dccbd835bc2 -size 2673906 +oid sha256:30bcf48f0c851d4e676d5b92be7eb3032442f5106a1072ef8fc9b335d60d0fbf +size 2673694 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_16kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_16kHz.bin index 13787ec393..20c688d2da 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_16kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_16kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:420704e8c05cd809f8448c1797b3cd2e69df0deb7ebe229b51ff0b54bbf99339 -size 1771126 +oid sha256:c839dcbdcfb7de23b725325770a07de3fe0144dc56f0ba7b4a9627bc912c2547 +size 1771166 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_32kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_32kHz.bin index 9b7593a1d5..c92f4c2af0 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_32kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_32kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5f26a82e286d6cadac49bd448bb4b621ff6381cb9c0be91a85b6723761f1c676 -size 2107642 +oid sha256:9c88daaf248bb36ac11d0aa320b18ce87019ef6ad9fab6ac2b7f064b27048aac +size 2107682 diff --git a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_48kHz.bin b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_48kHz.bin index 8e7ad080a4..61f226bca4 100644 --- a/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_48kHz.bin +++ b/scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_51_brir-lc_48kHz.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7d72291e29de622a681635c11f8acceabbdfaec91e81a82146859b8b0d5ca2c8 -size 2343610 +oid sha256:ed7e4b9f3306e7aecd2edf19b7ca9fb62240031fa248b26ad7c606fef36a20fe +size 2343650 -- GitLab From e48ce10426f1fa0c286c7e3258cd6f153c34b00d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 29 May 2024 17:00:34 +0200 Subject: [PATCH 02/27] skip aeid test as not supported in BASOP --- tests/renderer/test_renderer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 3c9b95dbaf..0c1ce804b8 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -127,6 +127,7 @@ def test_ambisonics_binaural_headrotation( ) +@pytest.mark.skip(reason="Not supported for BASOP code currently") @pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -- GitLab From 5387710fe3e612468b9d5d5e1ca5adb441c02eec Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 3 Jun 2024 15:27:30 +0200 Subject: [PATCH 03/27] update name of merged csv file --- ci/basop-pages/create_report_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index d8c3373b54..065a2eced8 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -27,7 +27,7 @@ Comparing:
-- GitLab From 89cb800444ced18495779a11d880fd7b2694d8a7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 08:06:41 +0200 Subject: [PATCH 04/27] remove anyway empty coverage section for now --- ci/basop-pages/basop_index.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/basop-pages/basop_index.html b/ci/basop-pages/basop_index.html index 4c5202ad35..8688232fe4 100644 --- a/ci/basop-pages/basop_index.html +++ b/ci/basop-pages/basop_index.html @@ -13,10 +13,4 @@
  • ivas-pytest-mld-long-dec-lev-10
    • -

    Test Coverage

    - -
    - tbd... -
    - -- GitLab From b34be1b91e0279d77e519f600781323650c77dbd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:02:57 +0200 Subject: [PATCH 05/27] change colouring and add arrows --- ci/basop-pages/create_report_pages.py | 56 ++++++++++++++++++--------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index d8c3373b54..de3959aef9 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -11,11 +11,14 @@ SUBPAGE_TMPL_CSS = """ overflow:hidden;padding:10px 5px;word-break:normal;} .tbase th{border-color:black;border-style:solid;border-width:1px;font-family:sans-serif;font-size:14px; font-weight:normal;overflow:hidden;padding:10px 5px;word-break:normal;} -.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;vertical-align:top} -.tbase .tcenter{font-weight:bold;text-align:center;vertical-align:top} -.tbase .tleft{text-align:left;vertical-align:top} -.tbase .tincrease{text-align:left;vertical-align:top;background-color:#ff0000;border-color:inherit;font-weight:bold;} -.tbase .treduce{text-align:left;vertical-align:top;background-color:#00ff00;border-color:inherit;font-weight:bold;} +.tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;} +.tbase .tcenter{font-weight:bold;text-align:center;} +.tbase .tleft{text-align:left;horizontal-align:bottom} +.tbase .tincrease{text-align:left;background-color:#ff0000;border-color:inherit;font-weight:bold;} +.tbase .treduce{text-align:left;background-color:#00ff00;border-color:inherit;font-weight:bold;} + +.arrowup {font-weight:bold;background-color:#ff0000;font-size:200%;} +.arrowdown {font-weight:bold;background-color:#00ff00;font-size:200%;} """ @@ -54,6 +57,11 @@ TH_TMPL_GLOBAL = '{}' TH_TMPL_DIFFERENTIAL = '{}' TH_TMPL_SECOND_ROW = '{}' +ARROW_UP = '⬀' +ARROW_DOWN = '⬂' +# ARROW_UP = '⬀' +# ARROW_DOWN = '⬂' + # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] @@ -127,20 +135,30 @@ def tr_from_row(row, id_current, id_previous): prev = row[f"{c}-{id_previous}"] curr = row[f"{c}-{id_current}"] - # use red background if increase, green if decrease, white if same - td_tmpl = TD_TMPL_NORMAL - try: - if float(curr) > float(prev): - td_tmpl = TD_TMPL_INCREASE - if float(curr) < float(prev): - td_tmpl = TD_TMPL_REDUCE - except ValueError: - # if we land here, one of the cells is not a number, this indicates a crash - # or some error in the scripts, so mark with red as well - td_tmpl = TD_TMPL_INCREASE - - tr.append(td_tmpl.format(row[f"{c}-{id_previous}"])) - tr.append(td_tmpl.format(row[f"{c}-{id_current}"])) + if c == "Result": + # print errors in bold red font + td_tmpl = TD_TMPL_INCREASE if prev == "ERROR" else TD_TMPL_NORMAL + tr.append(td_tmpl.format(prev)) + td_tmpl = TD_TMPL_INCREASE if curr == "ERROR" else TD_TMPL_NORMAL + tr.append(td_tmpl.format(curr)) + else: + td_tmpl_curr = TD_TMPL_NORMAL + td_tmpl_prev = TD_TMPL_NORMAL + try: + if float(curr) > float(prev): + curr += f" {ARROW_UP}" + td_tmpl_curr = TD_TMPL_INCREASE + elif float(curr) < float(prev): + curr += f" {ARROW_DOWN}" + td_tmpl_curr = TD_TMPL_REDUCE + except ValueError: + # if we land here, one of the cells is not a number, this indicates a crash + # or some error in the scripts, so mark with red as well + td_tmpl_curr = TD_TMPL_INCREASE + td_tmpl_prev = TD_TMPL_INCREASE + + tr.append(td_tmpl_prev.format(prev)) + tr.append(td_tmpl_curr.format(curr)) return TR_TMPL.format("\n".join(tr)) -- GitLab From 1a95d5c32b33696a7048b05b808b5e7dfaa3547e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:04:44 +0200 Subject: [PATCH 06/27] remove obsolete bg color from arrows --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index de3959aef9..eeef815560 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -17,8 +17,8 @@ SUBPAGE_TMPL_CSS = """ .tbase .tincrease{text-align:left;background-color:#ff0000;border-color:inherit;font-weight:bold;} .tbase .treduce{text-align:left;background-color:#00ff00;border-color:inherit;font-weight:bold;} -.arrowup {font-weight:bold;background-color:#ff0000;font-size:200%;} -.arrowdown {font-weight:bold;background-color:#00ff00;font-size:200%;} +.arrowup {font-weight:bold;font-size:200%;} +.arrowdown {font-weight:bold;font-size:200%;} """ -- GitLab From 02382e64020cd647c93a772a5126a9cdef5c1a6d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:10:03 +0200 Subject: [PATCH 07/27] change colours for better readability --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index eeef815560..7b2476d7c1 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -14,8 +14,8 @@ SUBPAGE_TMPL_CSS = """ .tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;} .tbase .tcenter{font-weight:bold;text-align:center;} .tbase .tleft{text-align:left;horizontal-align:bottom} -.tbase .tincrease{text-align:left;background-color:#ff0000;border-color:inherit;font-weight:bold;} -.tbase .treduce{text-align:left;background-color:#00ff00;border-color:inherit;font-weight:bold;} +.tbase .tincrease{text-align:left;background-color:#ff5500;border-color:inherit;font-weight:bold;} +.tbase .treduce{text-align:left;background-color:#acff00;border-color:inherit;font-weight:bold;} .arrowup {font-weight:bold;font-size:200%;} .arrowdown {font-weight:bold;font-size:200%;} -- GitLab From 5df73a612381e9d50fc44b9b2b88123263704e69 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 16:33:47 +0200 Subject: [PATCH 08/27] add explicit headings --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 7b2476d7c1..e3534d895b 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -86,8 +86,8 @@ def create_subpage( table_header_a = "".join([TH_TMPL_GLOBAL.format(c) for c in COLUMNS_GLOBAL] + [TH_TMPL_DIFFERENTIAL.format(c) for c in COLUMNS_DIFFERENTIAL]) table_header_b = list() for c in COLUMNS_DIFFERENTIAL: - table_header_b.append(TH_TMPL_SECOND_ROW.format(id_previous)) - table_header_b.append(TH_TMPL_SECOND_ROW.format(id_current)) + table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Previous Run
    ID: {id_previous}")) + table_header_b.append(TH_TMPL_SECOND_ROW.format(f"Current Run
    ID: {id_current}")) table_header_b = "".join(table_header_b) table_body = "\n".join( tr_from_row(row, id_current, id_previous) for row in merged_reports -- GitLab From d80c699c26109b9f7e035cc99da1051f0f5eb329 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 17:00:23 +0200 Subject: [PATCH 09/27] add explanation and better sort --- ci/basop-pages/create_report_pages.py | 54 ++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index e3534d895b..f635c04614 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -34,8 +34,37 @@ Comparing:
    -Table is sorted by Difference in MLD with ERRORs or missing values ("None", "") being on top additionally. +How is the table sorted? +
      +
    • Cases with result ERROR or invalid/missing values for the numerical measures are given first
      • +
    • Next are all cases with MLD(current) != MLD(previous), sorted in descending order (biggest MLD increase comes first)
      • +
    • All cases with no difference in MLD are at the bottom
      • +
    +
    +What do the colours indicate +
      +
    • + Red background: +
        +
      • This testcases either triggered an ERROR in the pytest run (probably a crash in the codec) or did not give valid values for the numerical measures (probably an error in the scripts).
        • +
      • For this testcase the MLD increased wrt the previous run - MLD(current) > MLD(previous)
        • +
      +
      • +
    • + Green background: +
        +
      • For this testcase the MLD decreased wrt the previous run - MLD(current) < MLD(previous)
        • +
      +
      • +

    +How to interpret the Result column? +
      +
    • ERROR: An error occured during test run. It should be checked if a crash in the codec occured.
      • +
    • FAIL: An MLD value > 0 was reported. This is to be expected, the test just reports a failure due to how the tests are currently implemented.
      • +
    • PASS: MLD value of 0 was reported. This should only be the case for some special operating points and could indicate that parts of the codec are not being converted yet.
      • +
    + @@ -59,8 +88,6 @@ TH_TMPL_SECOND_ROW = '' ARROW_UP = '⬀' ARROW_DOWN = '⬂' -# ARROW_UP = '⬀' -# ARROW_DOWN = '⬂' # expected columns. actual columns are filtered from the incoming data later, this # is mainly for controlling the order in the output table @@ -183,15 +210,24 @@ def merge_and_cleanup_mld_reports( mld_col_curr = f"MLD-{id_current}" mld_col_prev = f"MLD-{id_previous}" - # sort based on difference in MLD between current and previous run - # put cases with "None" at the top of the list def sort_func(x): - vals_missing = ["None", ""] - - if x[mld_col_curr] in vals_missing or x[mld_col_prev] in vals_missing: + """ + Sort function for the rows. Puts missing or invalid values on top as those usually + indicate crashes. Then sorts by MLD difference in descending order. MLD diffs of zero + are uninteresting and are put last. + """ + try: + float(x[mld_col_curr]) + float(x[mld_col_prev]) + except ValueError: + # Value is no valid floating point value return float("inf") - return float(x[mld_col_curr]) - float(x[mld_col_prev]) + diff = float(x[mld_col_curr]) - float(x[mld_col_prev]) + if diff == 0: + diff = float("-inf") + + return diff merged = sorted(merged, key=sort_func, reverse=True) -- GitLab From 924d6dc158a5ee25943f295fa055cc939a217f5e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 4 Jun 2024 17:14:26 +0200 Subject: [PATCH 10/27] sort so that any diffs are above no diffs --- ci/basop-pages/create_report_pages.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index f635c04614..f6f2aa0778 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -1,6 +1,7 @@ import csv import pathlib import argparse +from functools import partial CSV_DELIM = ";" @@ -94,7 +95,7 @@ ARROW_DOWN = '⬂' COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] COLUMNS_GLOBAL = COLUMNS[:1] COLUMNS_DIFFERENTIAL = COLUMNS[1:] - +COLUMNS_DIFFERENTIAL_NOT_MLD = COLUMNS_DIFFERENTIAL[2:] def create_subpage( html_out, @@ -210,7 +211,7 @@ def merge_and_cleanup_mld_reports( mld_col_curr = f"MLD-{id_current}" mld_col_prev = f"MLD-{id_previous}" - def sort_func(x): + def sort_func(x, other_col_pairs): """ Sort function for the rows. Puts missing or invalid values on top as those usually indicate crashes. Then sorts by MLD difference in descending order. MLD diffs of zero @@ -224,12 +225,24 @@ def merge_and_cleanup_mld_reports( return float("inf") diff = float(x[mld_col_curr]) - float(x[mld_col_prev]) + + # if no diff in mld col found, check if there is a diff in any other measure if diff == 0: diff = float("-inf") + diff_other = 0 + for col_pair in other_col_pairs: + col_prev = col_pair[0] + col_curr = col_pair[1] + diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + + if diff_other > 0: + diff = -1000000 + return diff - merged = sorted(merged, key=sort_func, reverse=True) + other_col_pairs = [(f"{col}-{id_previous}", f"{col}-{id_current}") for col in COLUMNS_DIFFERENTIAL_NOT_MLD] + merged = sorted(merged, key=partial(sort_func, other_col_pairs=other_col_pairs), reverse=True) # remove the unecessary whole path from the testcase names for row in merged: -- GitLab From d6d7ffc59eb1e2988b525de3a179a43eccefc11b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Jun 2024 08:33:02 +0200 Subject: [PATCH 11/27] change small part in description --- ci/basop-pages/create_report_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 6ad35c09f1..e3557e252b 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -63,7 +63,7 @@ Comparing:
    • ERROR: An error occured during test run. It should be checked if a crash in the codec occured.
    • FAIL: An MLD value > 0 was reported. This is to be expected, the test just reports a failure due to how the tests are currently implemented.
      • -
    • PASS: MLD value of 0 was reported. This should only be the case for some special operating points and could indicate that parts of the codec are not being converted yet.
      • +
    • PASS: MLD value of 0 was reported. This should only be the case for some special operating points and could indicate that parts of the codec are not (fully) converted yet.
    -- GitLab From 3140dc4b87265206a0182ef94ef93c2aedfe926a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Jun 2024 08:58:38 +0200 Subject: [PATCH 12/27] add scaled jobs for basop repo --- ci/setup_pages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 4754d09f5c..a28c12675e 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -23,6 +23,8 @@ JOBS_FLOAT_REPO = [ ] JOBS_BASOP_REPO = [ "ivas-pytest-mld-long-dec", + "ivas-pytest-mld-long-dec-lev+10", + "ivas-pytest-mld-long-dec-lev-10", ] JOBS_FOR_PROJECT_ID = { -- GitLab From 9efeed541d57c0abffc58256093299874b7c8ec3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Jun 2024 15:51:10 +0200 Subject: [PATCH 13/27] do not include errors in total number of testcases --- scripts/parse_mld_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/parse_mld_xml.py b/scripts/parse_mld_xml.py index 7370cfe7ff..7f8b243a98 100644 --- a/scripts/parse_mld_xml.py +++ b/scripts/parse_mld_xml.py @@ -76,5 +76,5 @@ if __name__ == "__main__": outfile.write(results_sorted[test]) print( - f"Parsed testsuite with {passes+failures+errors} tests: {passes} passes, {failures} failures and {errors} errors." + f"Parsed testsuite with {passes+failures} tests: {passes} passes, {failures} failures and {errors} errors." ) -- GitLab From 88307f0b8a379c8272b90505f891bfadbfbdf292 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Jun 2024 15:44:29 +0200 Subject: [PATCH 14/27] skip other unsupported aeid test --- tests/renderer/test_renderer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 911506c865..581f6323a8 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -153,6 +153,7 @@ def test_dynamic_acoustic_environment( aeid=aeid, ) +@pytest.mark.skip(reason="Not supported for BASOP code currently") @pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -- GitLab From 67899036aa9e58f59804c2affaf374b0da58e995 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Jun 2024 16:18:02 +0200 Subject: [PATCH 15/27] add new unsupported testcases --- ci/remove_unsupported_testcases.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/remove_unsupported_testcases.py b/ci/remove_unsupported_testcases.py index 09ddbc0a01..2a1e9ed755 100644 --- a/ci/remove_unsupported_testcases.py +++ b/ci/remove_unsupported_testcases.py @@ -54,6 +54,13 @@ TESTCASES = [ "Multi-channel 5_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 32kHz out, EXT out", "Multi-channel 7_1 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 16kHz out, EXT out", "Multi-channel 7_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, EXT out", + "SBA planar 2OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, EXT out", + "SBA planar FOA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out", + "SBA 2OA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out", + "SBA planar 3OA bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, random FER at 5%, EXT out", + "Multi-channel 5_1 at 512 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (CREND)", + "Multi-channel 5_1 at 64 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (FastConv)", + "Multi-channel 5_1 at 32 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (ParamBin)", ] -- GitLab From bf4d7ab2227503429ea499691e5ef9efc49c3f5f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Jun 2024 16:18:27 +0200 Subject: [PATCH 16/27] invert colouring logic for SSNR --- ci/basop-pages/create_report_pages.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index c849b160be..8edc502575 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -183,10 +183,12 @@ def tr_from_row(row, id_current, id_previous): try: if float(curr) > float(prev): curr += f" {ARROW_UP}" - td_tmpl_curr = TD_TMPL_INCREASE + # increase is bad -> mark in red, execpt for SNR for which it is good -> mark in green + td_tmpl_curr = TD_TMPL_REDUCE if c == "MIN_SSNR" else TD_TMPL_INCREASE elif float(curr) < float(prev): curr += f" {ARROW_DOWN}" - td_tmpl_curr = TD_TMPL_REDUCE + # reduce is good -> mark in green, execpt for SNR for which it is bad -> mark in red + td_tmpl_curr = TD_TMPL_INCREASE if c == "MIN_SSNR" else TD_TMPL_REDUCE except ValueError: # if we land here, one of the cells is not a number, this indicates a crash # or some error in the scripts, so mark with red as well -- GitLab From ef225b5ac2756d24febd2dfc593819173b537000 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Jun 2024 10:38:47 +0200 Subject: [PATCH 17/27] fix comparison function in presence of None values --- ci/basop-pages/create_report_pages.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 8edc502575..5c7be66428 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -244,7 +244,12 @@ def merge_and_cleanup_mld_reports( for col_pair in other_col_pairs: col_prev = col_pair[0] col_curr = col_pair[1] - diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + + try: + diff_other += abs(float(x[col_curr]) - float(x[col_prev])) + except ValueError: + # can't make float from the column contents, probably NONE -> put to top + diff_other += float("inf") if diff_other > 0: diff = -1000000 -- GitLab From 26ad499459aa7743c6a44442fdb810989366e28c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Jun 2024 12:13:56 +0200 Subject: [PATCH 18/27] add ssnr to renderer compare tests --- tests/renderer/test_renderer.py | 280 +++++++++++++++++++++++++------- tests/renderer/utils.py | 94 ++++++----- 2 files changed, 273 insertions(+), 101 deletions(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 581f6323a8..19f708b68f 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -1,36 +1,35 @@ #!/usr/bin/env python3 """ - (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ -from pathlib import Path import pytest from .constants import ( @@ -52,6 +51,7 @@ from .constants import ( METADATA_SCENES_TO_TEST, ) from .utils import run_renderer, compare_renderer_args, test_info +from ..conftest import props_to_record ############################################################################## # Bit-exactness tests @@ -67,10 +67,18 @@ from .utils import run_renderer, compare_renderer_args, test_info @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -86,10 +94,18 @@ def test_ambisonics( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics_binaural_static( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -107,6 +123,7 @@ def test_ambisonics_binaural_static( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ambisonics_binaural_headrotation( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -117,6 +134,7 @@ def test_ambisonics_binaural_headrotation( ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -133,15 +151,24 @@ def test_ambisonics_binaural_headrotation( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) -@pytest.mark.parametrize("aeid", ["1", "0"]) +@pytest.mark.parametrize("aeid", ["1", "0"]) def test_dynamic_acoustic_environment( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim, aeid + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, + aeid, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") - + run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -150,24 +177,33 @@ def test_dynamic_acoustic_environment( get_mld=get_mld, mld_lim=get_mld_lim, config_file=rend_config_path, - aeid=aeid, + aeid=aeid, ) + @pytest.mark.skip(reason="Not supported for BASOP code currently") @pytest.mark.create_ref @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_dynamic_acoustic_environment_file( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") aeid = TEST_VECTOR_DIR.joinpath(f"aeid1.txt") - + run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -176,7 +212,7 @@ def test_dynamic_acoustic_environment_file( get_mld=get_mld, mld_lim=get_mld_lim, config_file=rend_config_path, - aeid=aeid, + aeid=aeid, ) @@ -188,10 +224,18 @@ def test_dynamic_acoustic_environment_file( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_multichannel( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -207,13 +251,21 @@ def test_multichannel( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_multichannel_binaural_static( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -231,6 +283,7 @@ def test_multichannel_binaural_static( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_multichannel_binaural_headrotation( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -244,6 +297,7 @@ def test_multichannel_binaural_headrotation( run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -263,10 +317,18 @@ def test_multichannel_binaural_headrotation( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ism( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -283,7 +345,14 @@ def test_ism( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ism_binaural_static( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -292,6 +361,7 @@ def test_ism_binaural_static( run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -310,6 +380,7 @@ def test_ism_binaural_static( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_ism_binaural_headrotation( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -325,6 +396,7 @@ def test_ism_binaural_headrotation( run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -345,10 +417,18 @@ def test_ism_binaural_headrotation( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_masa( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -365,13 +445,21 @@ def test_masa( @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_masa_binaural_static( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -390,6 +478,7 @@ def test_masa_binaural_static( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_masa_binaural_headrotation( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -403,6 +492,7 @@ def test_masa_binaural_headrotation( run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -417,9 +507,12 @@ def test_masa_binaural_headrotation( @pytest.mark.create_ref @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) -def test_masa_prerend(record_property, test_info, in_fmt, get_mld, get_mld_lim): +def test_masa_prerend( + record_property, props_to_record, test_info, in_fmt, get_mld, get_mld_lim +): run_renderer( record_property, + props_to_record, test_info, "META", "MASA2", @@ -438,10 +531,18 @@ def test_masa_prerend(record_property, test_info, in_fmt, get_mld, get_mld_lim): @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input( - record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, @@ -456,10 +557,11 @@ def test_custom_ls_input( @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): run_renderer( record_property, + props_to_record, test_info, in_fmt, CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), @@ -473,10 +575,11 @@ def test_custom_ls_output( @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): run_renderer( record_property, + props_to_record, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_fmt}.txt"), CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), @@ -491,10 +594,18 @@ def test_custom_ls_input_output( @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input_binaural( - record_property, test_info, in_layout, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_layout, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, @@ -512,6 +623,7 @@ def test_custom_ls_input_binaural( @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_custom_ls_input_binaural_headrotation( record_property, + props_to_record, test_info, in_layout, out_fmt, @@ -522,6 +634,7 @@ def test_custom_ls_input_binaural_headrotation( ): run_renderer( record_property, + props_to_record, test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, @@ -541,10 +654,18 @@ def test_custom_ls_input_binaural_headrotation( @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) @pytest.mark.parametrize("frame_size", FRAMING_TO_TEST) def test_metadata( - record_property, test_info, in_fmt, out_fmt, frame_size, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + frame_size, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, "META", out_fmt, @@ -564,10 +685,18 @@ def test_metadata( @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_static( - record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -583,10 +712,18 @@ def test_non_diegetic_pan_static( @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_ism_static( - record_property, test_info, in_fmt, out_fmt, non_diegetic_pan, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + non_diegetic_pan, + get_mld, + get_mld_lim, ): run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -611,13 +748,21 @@ def test_non_diegetic_pan_ism_static( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotzero( - record_property, test_info, in_fmt, out_fmt, trj_file, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + get_mld, + get_mld_lim, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -640,13 +785,14 @@ def test_ambisonics_binaural_headrotation_refrotzero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refrotequal( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -674,13 +820,21 @@ def test_ambisonics_binaural_headrotation_refrotequal( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveczero( - record_property, test_info, in_fmt, out_fmt, trj_file, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + trj_file, + get_mld, + get_mld_lim, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -704,7 +858,7 @@ def test_ambisonics_binaural_headrotation_refveczero( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvecequal( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -715,6 +869,7 @@ def test_ambisonics_binaural_headrotation_refvecequal( else: compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -741,7 +896,7 @@ def test_ambisonics_binaural_headrotation_refvecequal( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -752,6 +907,7 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( else: compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -781,13 +937,14 @@ def test_ambisonics_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -816,13 +973,14 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -849,7 +1007,7 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) def test_multichannel_binaural_headrotation_refvec_rotating( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -859,6 +1017,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating( compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -886,7 +1045,7 @@ def test_multichannel_binaural_headrotation_refvec_rotating( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) def test_ism_binaural_headrotation_refvec_rotating( - record_property, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") @@ -898,6 +1057,7 @@ def test_ism_binaural_headrotation_refvec_rotating( compare_renderer_args( record_property, + props_to_record, test_info, in_fmt, out_fmt, diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 239e846ca0..7530ece5ef 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -1,33 +1,33 @@ #!/usr/bin/env python3 """ - (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import filecmp @@ -40,7 +40,6 @@ from typing import Dict, Optional, Union import numpy as np import pytest -import re from .compare_audio import compare_audio_arrays from .constants import ( @@ -56,7 +55,7 @@ from .constants import ( sys.path.append(SCRIPTS_DIR) from pyaudio3dtools.audiofile import readfile from ..cmp_pcm import cmp_pcm -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN +from ..conftest import parse_properties # fixture returns test information, enabling per-testcase SNR @@ -74,6 +73,7 @@ def run_cmd(cmd, env=None): f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" ) + def run_isar_ext_rend_cmd(cmd, env=None): logging.info(f"\nRunning ISAR EXT REND command\n{' '.join(cmd)}\n") try: @@ -83,6 +83,7 @@ def run_isar_ext_rend_cmd(cmd, env=None): f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" ) + def run_ivas_isar_enc_cmd(cmd, env=None): logging.info(f"\nRunning IVAS ISAR encoder command\n{' '.join(cmd)}\n") try: @@ -92,6 +93,7 @@ def run_ivas_isar_enc_cmd(cmd, env=None): f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" ) + def run_ivas_isar_dec_cmd(cmd, env=None): logging.info(f"\nDUT decoder command:\n\t{' '.join(cmd)}\n") try: @@ -100,7 +102,8 @@ def run_ivas_isar_dec_cmd(cmd, env=None): raise SystemError( f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" ) - + + def run_isar_post_rend_cmd(cmd, env=None): logging.info(f"\nRunning ISAR post renderer command\n{' '.join(cmd)}\n") try: @@ -108,7 +111,8 @@ def run_isar_post_rend_cmd(cmd, env=None): except sp.CalledProcessError as e: raise SystemError( f"Command returned non-zero exit status ({e.returncode}): {' '.join(e.cmd)}\n{e.stderr}\n{e.stdout}" - ) + ) + def check_BE( test_info, @@ -147,6 +151,7 @@ def check_BE( def run_renderer( record_property, + props_to_record, test_info, in_fmt: str, out_fmt: str, @@ -165,6 +170,8 @@ def run_renderer( get_mld=False, mld_lim=0, get_mld_lim=0, + abs_tol=0, + get_ssnr=False, aeid: Optional[Union[Path, int]] = None, ) -> str: # prepare arguments and filepaths @@ -307,17 +314,13 @@ def run_renderer( ref_fs, get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, ) - mld = 0 - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - max_diff = 0 - if output_differs: - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) if output_differs: pytest.fail(f"Output differs: ({reason})") @@ -331,6 +334,7 @@ def run_renderer( cut, cut_fs = readfile(out_file) + # TODO: this is duplicate effort with cmp_pcm being used above check_BE(test_info, ref, ref_fs, cut, cut_fs) # compare metadata files in case of MASA prerendering @@ -344,10 +348,17 @@ def run_renderer( def compare_renderer_args( - record_property, test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + ref_kwargs: Dict, + cut_kwargs: Dict, ): out_file_ref = run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, @@ -356,6 +367,7 @@ def compare_renderer_args( ref, ref_fs = readfile(out_file_ref) out_file_cut = run_renderer( record_property, + props_to_record, test_info, in_fmt, out_fmt, -- GitLab From 7116adc9ca580b52620cc0843cdf5e857341195a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 27 Jun 2024 12:38:38 +0200 Subject: [PATCH 19/27] fix property recording in renderer tests --- tests/renderer/test_renderer.py | 70 +++++++++++++++++++++++++++++++-- tests/renderer/utils.py | 55 +++++++++++++------------- 2 files changed, 94 insertions(+), 31 deletions(-) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 19f708b68f..7897af48ce 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -75,6 +75,7 @@ def test_ambisonics( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -86,6 +87,7 @@ def test_ambisonics( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -102,6 +104,7 @@ def test_ambisonics_binaural_static( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -113,6 +116,7 @@ def test_ambisonics_binaural_static( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -131,6 +135,7 @@ def test_ambisonics_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -143,6 +148,7 @@ def test_ambisonics_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -161,6 +167,7 @@ def test_dynamic_acoustic_environment( frame_size, get_mld, get_mld_lim, + get_ssnr, aeid, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") @@ -176,6 +183,7 @@ def test_dynamic_acoustic_environment( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, config_file=rend_config_path, aeid=aeid, ) @@ -195,6 +203,7 @@ def test_dynamic_acoustic_environment_file( frame_size, get_mld, get_mld_lim, + get_ssnr, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -211,6 +220,7 @@ def test_dynamic_acoustic_environment_file( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, config_file=rend_config_path, aeid=aeid, ) @@ -232,6 +242,7 @@ def test_multichannel( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -243,6 +254,7 @@ def test_multichannel( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -259,6 +271,7 @@ def test_multichannel_binaural_static( frame_size, get_mld, get_mld_lim, + get_ssnr, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -273,6 +286,7 @@ def test_multichannel_binaural_static( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -291,6 +305,7 @@ def test_multichannel_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -306,6 +321,7 @@ def test_multichannel_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -325,6 +341,7 @@ def test_ism( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -337,6 +354,7 @@ def test_ism( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -353,6 +371,7 @@ def test_ism_binaural_static( frame_size, get_mld, get_mld_lim, + get_ssnr, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -370,6 +389,7 @@ def test_ism_binaural_static( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -388,6 +408,7 @@ def test_ism_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -406,6 +427,7 @@ def test_ism_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -425,6 +447,7 @@ def test_masa( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -437,6 +460,7 @@ def test_masa( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -453,6 +477,7 @@ def test_masa_binaural_static( frame_size, get_mld, get_mld_lim, + get_ssnr, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -468,6 +493,7 @@ def test_masa_binaural_static( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -486,6 +512,7 @@ def test_masa_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -502,13 +529,20 @@ def test_masa_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @pytest.mark.create_ref @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST_MASA_PREREND) def test_masa_prerend( - record_property, props_to_record, test_info, in_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + get_mld, + get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -520,6 +554,7 @@ def test_masa_prerend( binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -539,6 +574,7 @@ def test_custom_ls_input( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -550,6 +586,7 @@ def test_custom_ls_input( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -557,7 +594,14 @@ def test_custom_ls_input( @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + get_mld, + get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -568,6 +612,7 @@ def test_custom_ls_output( binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -575,7 +620,14 @@ def test_custom_ls_output( @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @pytest.mark.parametrize("in_fmt", CUSTOM_LS_TO_TEST) def test_custom_ls_input_output( - record_property, props_to_record, test_info, in_fmt, out_fmt, get_mld, get_mld_lim + record_property, + props_to_record, + test_info, + in_fmt, + out_fmt, + get_mld, + get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -586,6 +638,7 @@ def test_custom_ls_input_output( binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -602,6 +655,7 @@ def test_custom_ls_input_binaural( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -613,6 +667,7 @@ def test_custom_ls_input_binaural( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -631,6 +686,7 @@ def test_custom_ls_input_binaural_headrotation( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -643,6 +699,7 @@ def test_custom_ls_input_binaural_headrotation( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -662,6 +719,7 @@ def test_metadata( frame_size, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -674,6 +732,7 @@ def test_metadata( frame_size=frame_size, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -693,6 +752,7 @@ def test_non_diegetic_pan_static( non_diegetic_pan, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -704,6 +764,7 @@ def test_non_diegetic_pan_static( binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -720,6 +781,7 @@ def test_non_diegetic_pan_ism_static( non_diegetic_pan, get_mld, get_mld_lim, + get_ssnr, ): run_renderer( record_property, @@ -731,6 +793,7 @@ def test_non_diegetic_pan_ism_static( binary_suffix=EXE_SUFFIX, get_mld=get_mld, mld_lim=get_mld_lim, + get_ssnr=get_ssnr, ) @@ -756,6 +819,7 @@ def test_ambisonics_binaural_headrotation_refrotzero( trj_file, get_mld, get_mld_lim, + get_ssnr, ): if test_info.config.option.create_ref or test_info.config.option.create_cut: pytest.skip("OTR tests only run for smoke test") diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 7530ece5ef..ede2dac169 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -304,38 +304,37 @@ def run_renderer( # CUT creation mode will run a comparison with REF out_file_ref = str(OUTPUT_PATH_REF.joinpath(out_file_stem)) - if get_mld: - # see constants.py - ref_fs = int(cmd[10]) * 1000 - output_differs, reason = cmp_pcm( - out_file, - out_file_ref, - out_fmt, - ref_fs, - get_mld=get_mld, - mld_lim=get_mld_lim, - abs_tol=abs_tol, - get_ssnr=get_ssnr, - ) + # see constants.py + ref_fs = int(cmd[10]) * 1000 + output_differs, reason = cmp_pcm( + out_file, + out_file_ref, + out_fmt, + ref_fs, + get_mld=get_mld, + mld_lim=get_mld_lim, + abs_tol=abs_tol, + get_ssnr=get_ssnr, + ) - props = parse_properties(reason, output_differs, props_to_record) - for k, v in props.items(): - record_property(k, v) + props = parse_properties(reason, output_differs, props_to_record) + for k, v in props.items(): + record_property(k, v) - if output_differs: - pytest.fail(f"Output differs: ({reason})") - else: - try: - ref, ref_fs = readfile(out_file_ref) - except FileNotFoundError: - pytest.fail( - f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" - ) + if output_differs: + pytest.fail(f"Output differs: ({reason})") + + try: + ref, ref_fs = readfile(out_file_ref) + except FileNotFoundError: + pytest.fail( + f"Reference vector not found! Ensure they were created with the --create_ref argument.\n{out_file_ref}" + ) - cut, cut_fs = readfile(out_file) + cut, cut_fs = readfile(out_file) - # TODO: this is duplicate effort with cmp_pcm being used above - check_BE(test_info, ref, ref_fs, cut, cut_fs) + # TODO: this is duplicate effort with cmp_pcm being used above + check_BE(test_info, ref, ref_fs, cut, cut_fs) # compare metadata files in case of MASA prerendering if "MASA" in str(out_fmt): -- GitLab From 5f4fc55e5b4c288f4ac25c83fd985fcf1dbc9a9e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 28 Jun 2024 08:38:52 +0200 Subject: [PATCH 20/27] put the per-channel SSNR printout into one line --- tests/cmp_pcm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 7bf1359f4e..1d225b90ec 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -102,10 +102,10 @@ def cmp_pcm( reason += f" > {mld_lim}" if get_ssnr: - reason += "\n" + reason += " - " for i, s in enumerate(cmp_result["SSNR"], start=1): msg = f"Channel {i} SSNR: {s}" - reason += msg + "\n" + reason += msg + " - " print(msg) return output_differs, reason -- GitLab From 1b1f108e9e20bae1729c194f13e902f81996c297 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Jul 2024 19:07:45 +0200 Subject: [PATCH 21/27] add max abs diff to printout --- tests/test_be_for_jbm_neutral_dly_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py index 64e7773a60..d87fa1f58f 100644 --- a/tests/test_be_for_jbm_neutral_dly_profile.py +++ b/tests/test_be_for_jbm_neutral_dly_profile.py @@ -197,4 +197,4 @@ def test_be_for_jbm_neutral_dly_profile( test_start_offset_ms=JBM_NEUTRAL_DELAY_MS, ) if not cmp_result["bitexact"]: - pytest.fail("Difference between no jbm and zero-delay jbm decoding!") + pytest.fail(f"Difference between no jbm and zero-delay jbm decoding found! Max abs diff: {cmp_result["max_abs_diff"]}") -- GitLab From dc369ffbb9ac3776dfe77373134e4ab87b0b78db Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Jul 2024 11:40:01 +0200 Subject: [PATCH 22/27] cleanup --- tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index c5d827569f..b75b3ba997 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -872,7 +872,6 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: props["MIN_SSNR_CHANNEL"] = min_ssnr_channel elif prop == "ODG": odgs = re.findall(ODG_PATTERN, text_to_parse) - print(odgs) min_odg = min(odgs) min_odg_channel = odgs.index(min_odg) props["MIN_ODG"] = min_odg -- GitLab From 14650f189adc821001006c067439ae030087fa7e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Jul 2024 15:29:37 +0200 Subject: [PATCH 23/27] fix parsing of negative SSNR values --- tests/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/constants.py b/tests/constants.py index e6ea1c1589..2f1746e76b 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -9,4 +9,4 @@ MLD_PATTERN = r"MLD: ([\d\.]*)" MAX_DIFF_PATTERN = r"MAXIMUM ABS DIFF: (\d*)" ODG_PATTERN_PQEVALAUDIO = r"Objective Difference Grade: (-*\d*\.\d*)" ODG_PATTERN = r"ODG: (-*\d*\.\d*)" -SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[\d\.]*)" +SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[-*\d\.]*)" -- GitLab From 382cef4a601fb301d3745069f4ae42666c1b5507 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Jul 2024 16:06:50 +0200 Subject: [PATCH 24/27] enable odg calculation for renderer tests --- tests/renderer/test_renderer.py | 45 +++++++++++++++++++++++++++++++++ tests/renderer/utils.py | 2 ++ 2 files changed, 47 insertions(+) diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index 7897af48ce..31a125f9a8 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -76,6 +76,7 @@ def test_ambisonics( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -88,6 +89,7 @@ def test_ambisonics( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -105,6 +107,7 @@ def test_ambisonics_binaural_static( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -117,6 +120,7 @@ def test_ambisonics_binaural_static( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -136,6 +140,7 @@ def test_ambisonics_binaural_headrotation( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -149,6 +154,7 @@ def test_ambisonics_binaural_headrotation( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -184,6 +190,7 @@ def test_dynamic_acoustic_environment( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, config_file=rend_config_path, aeid=aeid, ) @@ -204,6 +211,7 @@ def test_dynamic_acoustic_environment_file( get_mld, get_mld_lim, get_ssnr, + get_odg, ): rend_config_path = TEST_VECTOR_DIR.joinpath(f"rend_config_combined.cfg") rend_config_path.with_stem(f"rend_config") @@ -221,6 +229,7 @@ def test_dynamic_acoustic_environment_file( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, config_file=rend_config_path, aeid=aeid, ) @@ -243,6 +252,7 @@ def test_multichannel( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -255,6 +265,7 @@ def test_multichannel( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -272,6 +283,7 @@ def test_multichannel_binaural_static( get_mld, get_mld_lim, get_ssnr, + get_odg, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -287,6 +299,7 @@ def test_multichannel_binaural_static( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -306,6 +319,7 @@ def test_multichannel_binaural_headrotation( get_mld, get_mld_lim, get_ssnr, + get_odg, ): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") @@ -322,6 +336,7 @@ def test_multichannel_binaural_headrotation( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -342,6 +357,7 @@ def test_ism( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -355,6 +371,7 @@ def test_ism( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -372,6 +389,7 @@ def test_ism_binaural_static( get_mld, get_mld_lim, get_ssnr, + get_odg, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -390,6 +408,7 @@ def test_ism_binaural_static( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -409,6 +428,7 @@ def test_ism_binaural_headrotation( get_mld, get_mld_lim, get_ssnr, + get_odg, ): try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] @@ -428,6 +448,7 @@ def test_ism_binaural_headrotation( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -448,6 +469,7 @@ def test_masa( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -461,6 +483,7 @@ def test_masa( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -478,6 +501,7 @@ def test_masa_binaural_static( get_mld, get_mld_lim, get_ssnr, + get_odg, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -494,6 +518,7 @@ def test_masa_binaural_static( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -513,6 +538,7 @@ def test_masa_binaural_headrotation( get_mld, get_mld_lim, get_ssnr, + get_odg, ): if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") @@ -530,6 +556,7 @@ def test_masa_binaural_headrotation( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -543,6 +570,7 @@ def test_masa_prerend( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -555,6 +583,7 @@ def test_masa_prerend( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -575,6 +604,7 @@ def test_custom_ls_input( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -587,6 +617,7 @@ def test_custom_ls_input( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -602,6 +633,7 @@ def test_custom_ls_output( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -613,6 +645,7 @@ def test_custom_ls_output( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -628,6 +661,7 @@ def test_custom_ls_input_output( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -639,6 +673,7 @@ def test_custom_ls_input_output( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -656,6 +691,7 @@ def test_custom_ls_input_binaural( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -668,6 +704,7 @@ def test_custom_ls_input_binaural( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -687,6 +724,7 @@ def test_custom_ls_input_binaural_headrotation( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -700,6 +738,7 @@ def test_custom_ls_input_binaural_headrotation( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -720,6 +759,7 @@ def test_metadata( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -733,6 +773,7 @@ def test_metadata( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -753,6 +794,7 @@ def test_non_diegetic_pan_static( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -765,6 +807,7 @@ def test_non_diegetic_pan_static( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) @@ -782,6 +825,7 @@ def test_non_diegetic_pan_ism_static( get_mld, get_mld_lim, get_ssnr, + get_odg, ): run_renderer( record_property, @@ -794,6 +838,7 @@ def test_non_diegetic_pan_ism_static( get_mld=get_mld, mld_lim=get_mld_lim, get_ssnr=get_ssnr, + get_odg=get_odg, ) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index ede2dac169..9be3071e47 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -172,6 +172,7 @@ def run_renderer( get_mld_lim=0, abs_tol=0, get_ssnr=False, + get_odg=False, aeid: Optional[Union[Path, int]] = None, ) -> str: # prepare arguments and filepaths @@ -315,6 +316,7 @@ def run_renderer( mld_lim=get_mld_lim, abs_tol=abs_tol, get_ssnr=get_ssnr, + get_odg=get_ssnr, ) props = parse_properties(reason, output_differs, props_to_record) -- GitLab From a37ebf81f9abde5a3d1734fa48be3c6040065f8d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Jul 2024 16:24:41 +0200 Subject: [PATCH 25/27] add "" around input argument expansion --- ci/complexity_measurements/getWmops.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/complexity_measurements/getWmops.sh b/ci/complexity_measurements/getWmops.sh index c069e2d57b..a3d4710f2c 100755 --- a/ci/complexity_measurements/getWmops.sh +++ b/ci/complexity_measurements/getWmops.sh @@ -35,7 +35,7 @@ if [ $# -ne 2 ]; then fi ivas_format=$1 -output_format=$2 +output_format="$2" date=`date +%Y%m%d` # used for log-file file ending shortDate=`date "+%b %d" | sed -e "s/\ /_/g"` # stored in the log-file -- GitLab From 1cd2b22c356f779ce99e0793bf32ef916663ffcb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 16:49:49 +0200 Subject: [PATCH 26/27] fix calls to run_renderer with new interface --- tests/hrtf_binary_loading/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/hrtf_binary_loading/utils.py b/tests/hrtf_binary_loading/utils.py index 04b25640a7..2ba8b39e6b 100644 --- a/tests/hrtf_binary_loading/utils.py +++ b/tests/hrtf_binary_loading/utils.py @@ -369,6 +369,7 @@ def compare_renderer_vs_renderer_with_binary_hrir( hrtf_file_path = hrtf_file_dir.joinpath(hrtf_file.format(48)) ref_out = run_renderer( None, + list(), test_info, in_fmt, out_fmt, @@ -385,6 +386,7 @@ def compare_renderer_vs_renderer_with_binary_hrir( ) cut_out = run_renderer( None, + list(), test_info, in_fmt, out_fmt, @@ -458,6 +460,7 @@ def compare_renderer_with_binary_fix_vs_with_binary_float( ref_out = run_renderer( None, + list(), test_info, in_fmt, out_fmt, @@ -475,6 +478,7 @@ def compare_renderer_with_binary_fix_vs_with_binary_float( ) cut_out = run_renderer( None, + list(), test_info, in_fmt, out_fmt, -- GitLab From 21f96620d65eb3bedbe39ddd53c2cd5eb8334e62 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Jul 2024 16:56:03 +0200 Subject: [PATCH 27/27] fix qoutation marks --- tests/test_be_for_jbm_neutral_dly_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py index d87fa1f58f..61bb168a2f 100644 --- a/tests/test_be_for_jbm_neutral_dly_profile.py +++ b/tests/test_be_for_jbm_neutral_dly_profile.py @@ -197,4 +197,4 @@ def test_be_for_jbm_neutral_dly_profile( test_start_offset_ms=JBM_NEUTRAL_DELAY_MS, ) if not cmp_result["bitexact"]: - pytest.fail(f"Difference between no jbm and zero-delay jbm decoding found! Max abs diff: {cmp_result["max_abs_diff"]}") + pytest.fail(f"Difference between no jbm and zero-delay jbm decoding found! Max abs diff: {cmp_result['max_abs_diff']}") -- GitLab
    {}