From bf06c2e6c4a1ee848620e1e8cad3e0a79144bd1d Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Tue, 24 Sep 2024 09:22:34 +0200
Subject: [PATCH 01/11] Add option --compare_to_input

---
 tests/cmp_pcm.py                              | 16 +++++++---
 .../test_param_file.py                        |  6 ++++
 tests/codec_be_on_mr_nonselection/test_sba.py | 31 ++++++++++++++++---
 tests/conftest.py                             | 12 +++++++
 4 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py
index 5b335840e6..d3f9afc425 100755
--- a/tests/cmp_pcm.py
+++ b/tests/cmp_pcm.py
@@ -57,11 +57,17 @@ def cmp_pcm(
     # In case of wav input, override the nchannels with the one from the wav header
     nchannels = s1.shape[1]
 
+    # In case number of channels do not match, fail already now. Could happen in case of
+    # comparison to input with for a non-passthrough mode.
+    if s1.shape[1] != s2.shape[1]:
+        reason = "FAIL: Number of channels differ."
+        return 1, reason
+
     if allow_differing_lengths:
-        # to allow for MLD comparison, shorten longer file
-        min_len = min(s1.shape[0], s2.shape[0])
-        s1 = s1[:min_len, :]
-        s2 = s2[:min_len, :]
+        # to allow for MLD comparison, pad shorter file
+        max_len = max(s1.shape[0], s2.shape[0])
+        s1 = np.pad(s1,((0,max_len - s1.shape[0]),(0,0)),mode='constant',constant_values=0)
+        s2 = np.pad(s2,((0,max_len - s2.shape[0]),(0,0)),mode='constant',constant_values=0)
     elif s1.shape != s2.shape:
         print(
             f"file size in samples: file 1 = {s1.shape[0]},",
@@ -176,6 +182,8 @@ if __name__ == "__main__":
     parser.add_argument("--get_mld", action="store_true")
     parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim")
     parser.add_argument("--get_odg", action="store_true")
+    parser.add_argument("--get_ssnr", action="store_true")
+    parser.add_argument("--allow_differing_lengths", action="store_true")
     args = parser.parse_args()
 
     result, msg = cmp_pcm(**vars(args))
diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py
index 9a11b4d6d4..699ac8d170 100644
--- a/tests/codec_be_on_mr_nonselection/test_param_file.py
+++ b/tests/codec_be_on_mr_nonselection/test_param_file.py
@@ -161,6 +161,7 @@ def test_param_file_tests(
     get_ssnr,
     get_enc_stats,
     get_odg,
+    compare_to_input,
 ):
     enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
 
@@ -396,6 +397,11 @@ def test_param_file_tests(
         # shift differences between the two signals - cut longer signal to shorter size
         allow_differing_lengths = not tracefile_last_rtp_numbers_differ
 
+        if compare_to_input:
+            # If comparing to input, set input as reference and allow different length
+            ref_output_file = testv_file
+            allow_differing_lengths = True
+
         fs = int(sampling_rate) * 1000
         output_differs, reason = cmp_pcm(
             ref_output_file,
diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py
index 98f55b7fe5..cf8584f200 100644
--- a/tests/codec_be_on_mr_nonselection/test_sba.py
+++ b/tests/codec_be_on_mr_nonselection/test_sba.py
@@ -114,6 +114,7 @@ def test_pca_enc(
     get_ssnr,
     get_odg,
     get_enc_stats,
+    compare_to_input,
 ):
     pca = True
     bitrate = "256000"
@@ -165,6 +166,7 @@ def test_pca_enc(
         sba_dec(
             record_property,
             props_to_record,
+            test_vector_path,
             dut_decoder_frontend,
             ref_decoder_frontend,
             reference_path,
@@ -187,6 +189,7 @@ def test_pca_enc(
             abs_tol=abs_tol,
             get_ssnr=get_ssnr,
             get_odg=get_odg,
+            compare_to_input=compare_to_input,
         )
 
 
@@ -223,6 +226,7 @@ def test_sba_enc_system(
     get_ssnr,
     get_odg,
     get_enc_stats,
+    compare_to_input,
 ):
 
     plc_pattern = None
@@ -333,6 +337,7 @@ def test_sba_enc_system(
         sba_dec(
             record_property,
             props_to_record,
+            test_vector_path,
             dut_decoder_frontend,
             ref_decoder_frontend,
             reference_path,
@@ -355,6 +360,7 @@ def test_sba_enc_system(
             abs_tol=abs_tol,
             get_ssnr=get_ssnr,
             get_odg=get_odg,
+            compare_to_input=compare_to_input,
         )
 
 
@@ -383,6 +389,7 @@ def test_spar_hoa2_enc_system(
     get_ssnr,
     get_odg,
     get_enc_stats,
+    compare_to_input,
 ):
     sampling_rate = "48"
     pca = False
@@ -467,6 +474,7 @@ def test_spar_hoa2_enc_system(
         sba_dec(
             record_property,
             props_to_record,
+            test_vector_path,
             dut_decoder_frontend,
             ref_decoder_frontend,
             reference_path,
@@ -489,6 +497,7 @@ def test_spar_hoa2_enc_system(
             abs_tol=abs_tol,
             get_ssnr=get_ssnr,
             get_odg=get_odg,
+            compare_to_input=compare_to_input,
         )
 
 
@@ -517,6 +526,7 @@ def test_spar_hoa3_enc_system(
     get_ssnr,
     get_odg,
     get_enc_stats,
+    compare_to_input,
 ):
     sampling_rate = "48"
     pca = False
@@ -595,6 +605,7 @@ def test_spar_hoa3_enc_system(
         sba_dec(
             record_property,
             props_to_record,
+            test_vector_path,
             dut_decoder_frontend,
             ref_decoder_frontend,
             reference_path,
@@ -617,6 +628,7 @@ def test_spar_hoa3_enc_system(
             abs_tol=abs_tol,
             get_ssnr=get_ssnr,
             get_odg=get_odg,
+            compare_to_input=compare_to_input,
         )
 
 
@@ -649,6 +661,7 @@ def test_sba_enc_BWforce_system(
     get_ssnr,
     get_odg,
     get_enc_stats,
+    compare_to_input,
 ):
     sid = 0
     plc_pattern = None
@@ -743,6 +756,7 @@ def test_sba_enc_BWforce_system(
         sba_dec(
             record_property,
             props_to_record,
+            test_vector_path,
             dut_decoder_frontend,
             ref_decoder_frontend,
             reference_path,
@@ -765,6 +779,7 @@ def test_sba_enc_BWforce_system(
             abs_tol=abs_tol,
             get_ssnr=get_ssnr,
             get_odg=get_odg,
+            compare_to_input=compare_to_input,
         )
 
 
@@ -801,6 +816,7 @@ def test_sba_plc_system(
     get_ssnr,
     get_odg,
     get_enc_stats,
+    compare_to_input,
 ):
     sid = 0
     pca = False
@@ -872,6 +888,7 @@ def test_sba_plc_system(
         sba_dec(
             record_property,
             props_to_record,
+            test_vector_path,
             dut_decoder_frontend,
             ref_decoder_frontend,
             reference_path,
@@ -894,6 +911,7 @@ def test_sba_plc_system(
             abs_tol=abs_tol,
             get_ssnr=get_ssnr,
             get_odg=get_odg,
+            compare_to_input=compare_to_input,
         )
 
 
@@ -1026,6 +1044,7 @@ def sba_enc(
 def sba_dec(
     record_property,
     props_to_record,
+    test_vector_path,
     dut_decoder_frontend,
     ref_decoder_frontend,
     reference_path,
@@ -1048,6 +1067,7 @@ def sba_dec(
     abs_tol=0,
     get_ssnr=False,
     get_odg=False,
+    compare_to_input=False,
 ):
     dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt"
     ref_pkt_dir = f"{reference_path}/sba_bs/pkt"
@@ -1115,6 +1135,11 @@ def sba_dec(
             plc_file=plc_file,
         )
 
+        if compare_to_input:
+            # If comparing to input, set input as reference and allow different length
+            ref_out_file = test_vector_path
+            allow_differing_lengths = True
+
         sampling_rate_Hz = int(sampling_rate) * 1000
         cmp_result, reason = cmp_pcm(
             dut_out_file,
@@ -1124,6 +1149,7 @@ def sba_dec(
             get_mld=get_mld,
             mld_lim=get_mld_lim,
             abs_tol=abs_tol,
+            allow_differing_lengths=allow_differing_lengths,
             get_ssnr=get_ssnr,
             get_odg=get_odg,
         )
@@ -1136,8 +1162,3 @@ def sba_dec(
         # report compare result
         if cmp_result != 0:
             pytest.fail(text_to_parse)
-
-        # remove DUT output files when test result is OK (to save disk space)
-        # if not keep_files:
-        #     os.remove(dut_out_file)
-        #     os.remove(dut_pkt_file)
diff --git a/tests/conftest.py b/tests/conftest.py
index 636881ecef..58184f0d36 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -260,6 +260,12 @@ def pytest_addoption(parser):
         default=0,
     )
 
+    parser.addoption(
+            "--compare_to_input",
+            action="store_true",
+            help="Compare output to the input file instead of reference output",
+            default=False,
+        )
 
 @pytest.fixture(scope="session", autouse=True)
 def update_ref(request):
@@ -968,6 +974,12 @@ def encoder_only(request) -> bool:
     """
     return request.config.getoption("--encoder_only")
 
+@pytest.fixture(scope="session", autouse=True)
+def compare_to_input(request) -> bool:
+    """
+    Return value of cmdl param --compare_to_input
+    """
+    return request.config.getoption("--compare_to_input")
 
 def pytest_configure(config):
     config.addinivalue_line("markers", "serial: mark test to run only in serial")
-- 
GitLab


From 2730753e1c81c22d933c3845fa9479c22e190260 Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Tue, 24 Sep 2024 09:59:40 +0200
Subject: [PATCH 02/11] Fix for when compare_to_input is not set

---
 tests/codec_be_on_mr_nonselection/test_sba.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py
index cf8584f200..ba1fff6477 100644
--- a/tests/codec_be_on_mr_nonselection/test_sba.py
+++ b/tests/codec_be_on_mr_nonselection/test_sba.py
@@ -1135,6 +1135,7 @@ def sba_dec(
             plc_file=plc_file,
         )
 
+        allow_differing_lengths = False
         if compare_to_input:
             # If comparing to input, set input as reference and allow different length
             ref_out_file = test_vector_path
-- 
GitLab


From 4a382cd9b10495a3a832fc8a3683314c4d433c4d Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Tue, 24 Sep 2024 14:17:27 +0200
Subject: [PATCH 03/11] Add option to parse junit reports which do not follow
 the IVAS test suite naming

---
 scripts/parse_xml_report.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py
index 3be7a9ca0b..e0f02238e4 100644
--- a/scripts/parse_xml_report.py
+++ b/scripts/parse_xml_report.py
@@ -31,6 +31,7 @@ EVS_FORMATS = {
     "EVS_JBM_dec": r"Readme_JBM_dec",
 }
 
+NO_FORMATS = {"Default": r".*"}
 
 IVAS_CATEGORIES = {
     "Normal operation": r".*",
@@ -48,6 +49,8 @@ EVS_CATEGORIES = {
     "JBM": r"JBM",
 }
 
+NO_CATEGORIES = {"N/A": r".*"}
+
 # Main routine
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
@@ -64,15 +67,23 @@ if __name__ == "__main__":
         action="store_true",
         help="Parse using EVS 26.444 formats",
     )
+    parser.add_argument(
+        "--skip_formats",
+        action="store_true",
+        help="Parse without formats and categories. Suitable for general tests which do not match the IVAS categories.",
+    )
     args = parser.parse_args()
     xml_report = args.xml_report
     csv_file = args.csv_file
+    FORMATS = IVAS_FORMATS
+    CATEGORIES = IVAS_CATEGORIES
     if args.evs:
         FORMATS = EVS_FORMATS
         CATEGORIES = EVS_CATEGORIES
-    else:
-        FORMATS = IVAS_FORMATS
-        CATEGORIES = IVAS_CATEGORIES
+    if args.skip_formats:
+        FORMATS = NO_FORMATS
+        CATEGORIES = NO_CATEGORIES
+
     tree = ElementTree.parse(xml_report)
 
     testsuite = tree.find(".//testsuite")
-- 
GitLab


From ff792ab03c45fe3df4d18c7dcd69dee57d552c21 Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Mon, 30 Sep 2024 11:27:20 +0200
Subject: [PATCH 04/11] Ignore JBM tracefiles when comparing to input -- not
 present for input files

---
 tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +-
 tests/conftest.py                                    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py
index 699ac8d170..11699b45f1 100644
--- a/tests/codec_be_on_mr_nonselection/test_param_file.py
+++ b/tests/codec_be_on_mr_nonselection/test_param_file.py
@@ -379,7 +379,7 @@ def test_param_file_tests(
         # set to false per default even if this is no JBM case - makes later check for failure easier
         tracefile_last_rtp_numbers_differ = False
 
-        if len(tracefile_dec) > 0:
+        if len(tracefile_dec) > 0 and not compare_to_input:
             dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}"
             ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}"
 
diff --git a/tests/conftest.py b/tests/conftest.py
index 58184f0d36..d8b40313d1 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -933,7 +933,7 @@ def reference_path(request) -> str:
 
     path = str(path.resolve())
 
-    if request.config.option.update_ref == "0":
+    if request.config.option.update_ref == "0" and not request.config.option.compare_to_input:
         if not os.path.isdir(path):
             raise FileNotFoundError(
                 f"REF path {path} not found!\nPlease generate the references, first!\n!"
-- 
GitLab


From bde591466275e5567d46284f5ed2f96d2882c311 Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Tue, 1 Oct 2024 13:09:32 +0200
Subject: [PATCH 05/11] Add metadata support to compare_to_input option

---
 .../codec_be_on_mr_nonselection/test_param_file.py  | 13 +++++++++----
 tests/conftest.py                                   |  2 +-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py
index 11699b45f1..29b5108dc6 100644
--- a/tests/codec_be_on_mr_nonselection/test_param_file.py
+++ b/tests/codec_be_on_mr_nonselection/test_param_file.py
@@ -397,10 +397,13 @@ def test_param_file_tests(
         # shift differences between the two signals - cut longer signal to shorter size
         allow_differing_lengths = not tracefile_last_rtp_numbers_differ
 
+        md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)
+
         if compare_to_input:
             # If comparing to input, set input as reference and allow different length
             ref_output_file = testv_file
             allow_differing_lengths = True
+            input_md = [x for x in enc_split if "csv" in x]
 
         fs = int(sampling_rate) * 1000
         output_differs, reason = cmp_pcm(
@@ -415,7 +418,6 @@ def test_param_file_tests(
             get_ssnr=get_ssnr,
             get_odg=get_odg,
         )
-        md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)
 
         cmp_result_msg += reason
 
@@ -424,9 +426,12 @@ def test_param_file_tests(
             record_property(k, v)
 
         metadata_differs = False
-        for md_file in md_out_files:
-            dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}")
-            ref_metadata_file = Path(f"{reference_path}/param_file/dec/{md_file}")
+        if compare_to_input:
+            md_file_pairs = [(Path(f"{dut_base_path}/param_file/dec/{md_file}"), md_in_file) for (md_file,md_in_file) in zip(md_out_files,input_md)]
+        else:
+            md_file_pairs = [(Path(f"{dut_base_path}/param_file/dec/{md_file}"), Path(f"{reference_path}/param_file/dec/{md_file}")) for md_file in md_out_files]
+        for (dut_metadata_file, ref_metadata_file) in md_file_pairs:
+            md_file = os.path.basename(dut_metadata_file)
             try:
                 if not filecmp.cmp(dut_metadata_file, ref_metadata_file):
                     print("Output metadata differs for file: " + md_file)
diff --git a/tests/conftest.py b/tests/conftest.py
index d8b40313d1..8f3e4e961b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -263,7 +263,7 @@ def pytest_addoption(parser):
     parser.addoption(
             "--compare_to_input",
             action="store_true",
-            help="Compare output to the input file instead of reference output",
+            help="Compare output to the input file instead of reference output. N.B. Only applicable to pass-through tests.",
             default=False,
         )
 
-- 
GitLab


From 2c639832c3728b0e01a58369baba3206ec73c1b4 Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Wed, 2 Oct 2024 15:27:08 +0200
Subject: [PATCH 06/11] Skip non-passthrough tests with compare_to_input, add
 manual test job

---
 .gitlab-ci.yml                                | 41 ++++++++++++++++++-
 .../test_param_file.py                        | 18 ++++++++
 tests/codec_be_on_mr_nonselection/test_sba.py |  2 +-
 3 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6e5c88431c..e490d6d49d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,6 +24,7 @@ variables:
       - 'ivas-conformance'
       - 'ivas-conformance-linux'
       - 'check-float-reference'
+      - 'test-branch-vs-input-passthrough'
   GIT_CLEAN_FLAGS: -ffdxq
   TESTCASE_TIMEOUT_STV_SANITIZERS: 180
   TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200
@@ -69,7 +70,9 @@ workflow:
     - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-float-reference'
       variables:
         IVAS_PIPELINE_NAME: 'check-float-reference: $CI_COMMIT_BRANCH'
-    
+    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'
+      variables:
+        IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH'
 
 stages:
   - .pre
@@ -265,6 +268,8 @@ stages:
       when: never      
     - if: $MANUAL_PIPELINE_TYPE == 'check-float-reference'
       when: never
+    - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'
+      when: never
     - when: on_success
 
 .rules-merge-request:
@@ -1639,6 +1644,40 @@ test-long-self-test:
       junit:
         - report-junit-ltv.xml
 
+test-branch-vs-input-passthrough:
+  tags:
+    - ivas-linux
+  stage: compare
+  timeout: "30 minutes" # TBD
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'
+  allow_failure:
+    exit_codes:
+      - 123
+  script:
+    - *print-common-info
+    - make -j
+
+    - exit_code=0
+    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$?
+    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true
+    - python3 scripts/parse_xml_report.py report-junit.xml report.csv
+    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi
+    - if [ $exit_code -eq 1 ]; then echo "Differences encountered."; exit $EXIT_CODE_NON_BE; fi
+    - exit 0
+
+  artifacts:
+    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"
+    when: always
+    expire_in: 4 mos
+    paths:
+      - report-junit.xml
+      - report.html
+      - report.csv
+    expose_as: "$CI_JOB_NAME results"
+    reports:
+      junit:
+        - report-junit-ltv.xml
 
 # ---------------------------------------------------------------
 # Scheduled jobs on main
diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py
index 29b5108dc6..f8d0804c73 100644
--- a/tests/codec_be_on_mr_nonselection/test_param_file.py
+++ b/tests/codec_be_on_mr_nonselection/test_param_file.py
@@ -39,6 +39,7 @@ import platform
 from pathlib import Path
 from subprocess import run
 import pytest
+import re
 import numpy as np
 
 from tests.cmp_pcm import cmp_pcm
@@ -51,6 +52,17 @@ from tests.constants import (
     SCRIPTS_DIR,
 )
 
+PASSTHROUGH_CONF = [
+    (r'[\s\S]*',"EXT"), # Any input with EXT output is pass-through
+    (r'-stereo',"STEREO"),
+    (r'-sba\s?[-+]?1',"FOA"),
+    (r'-sba\s?[-+]?2',"HOA2"),
+    (r'-sba\s?[-+]?3',"HOA3"),
+    (r'-mc\s5_1',"5_1"),
+    (r'-mc\s7_1',"7_1"),
+    (r'-mc\s5_1_4',"5_1_4"),
+    (r'-mc\s7_1_4',"7_1_4"),
+]
 
 VALID_DEC_OUTPUT_CONF = [
     "MONO",
@@ -165,6 +177,12 @@ def test_param_file_tests(
 ):
     enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
 
+    # If compare_to_input is set, only run pass-through test cases
+    if compare_to_input:
+        passthrough = [(a, b) for (a, b) in PASSTHROUGH_CONF if re.search(a, enc_opts) and b in dec_opts]
+        if len(passthrough) != 1:
+            pytest.skip()
+
     tag_str = convert_test_string_to_tag(test_tag)
 
     # evaluate encoder options
diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py
index ba1fff6477..885ac488ad 100644
--- a/tests/codec_be_on_mr_nonselection/test_sba.py
+++ b/tests/codec_be_on_mr_nonselection/test_sba.py
@@ -1138,7 +1138,7 @@ def sba_dec(
         allow_differing_lengths = False
         if compare_to_input:
             # If comparing to input, set input as reference and allow different length
-            ref_out_file = test_vector_path
+            ref_out_file = f"{test_vector_path}/{tag}.wav"
             allow_differing_lengths = True
 
         sampling_rate_Hz = int(sampling_rate) * 1000
-- 
GitLab


From d1a7419b2fd5283b32501f6ebf597c0596e4eff7 Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Wed, 2 Oct 2024 15:30:15 +0200
Subject: [PATCH 07/11] Fix expose as artifacts name

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e490d6d49d..8b8ff10635 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1667,14 +1667,14 @@ test-branch-vs-input-passthrough:
     - exit 0
 
   artifacts:
-    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"
+    name: "test-branch-vs-input-passthrough--sha-$CI_COMMIT_SHORT_SHA--results"
     when: always
     expire_in: 4 mos
     paths:
       - report-junit.xml
       - report.html
       - report.csv
-    expose_as: "$CI_JOB_NAME results"
+    expose_as: "test-branch-vs-input-passthrough results"
     reports:
       junit:
         - report-junit-ltv.xml
-- 
GitLab


From ba024ea86ed02e7bbd95da541cadeaa7212a4a69 Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Wed, 2 Oct 2024 15:42:49 +0200
Subject: [PATCH 08/11] Fix pass-through filtering for param file, syntax error
 in junit report

---
 .gitlab-ci.yml                                       | 2 +-
 tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8b8ff10635..625a3cc8bc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1677,7 +1677,7 @@ test-branch-vs-input-passthrough:
     expose_as: "test-branch-vs-input-passthrough results"
     reports:
       junit:
-        - report-junit-ltv.xml
+        - report-junit.xml
 
 # ---------------------------------------------------------------
 # Scheduled jobs on main
diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py
index f8d0804c73..b071b5e5aa 100644
--- a/tests/codec_be_on_mr_nonselection/test_param_file.py
+++ b/tests/codec_be_on_mr_nonselection/test_param_file.py
@@ -179,7 +179,7 @@ def test_param_file_tests(
 
     # If compare_to_input is set, only run pass-through test cases
     if compare_to_input:
-        passthrough = [(a, b) for (a, b) in PASSTHROUGH_CONF if re.search(a, enc_opts) and b in dec_opts]
+        passthrough = [(a, b) for (a, b) in PASSTHROUGH_CONF if re.search(a, enc_opts) and b in dec_opts.split()[0]]
         if len(passthrough) != 1:
             pytest.skip()
 
-- 
GitLab


From 2803b946ae65453ac70f365ed77a8a0ab0eb7b55 Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Wed, 2 Oct 2024 16:04:49 +0200
Subject: [PATCH 09/11] Correct pass-through filtering

---
 tests/codec_be_on_mr_nonselection/test_param_file.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py
index 7d08fb8fa4..a8e24ca393 100644
--- a/tests/codec_be_on_mr_nonselection/test_param_file.py
+++ b/tests/codec_be_on_mr_nonselection/test_param_file.py
@@ -179,7 +179,7 @@ def test_param_file_tests(
 
     # If compare_to_input is set, only run pass-through test cases
     if compare_to_input:
-        passthrough = [(a, b) for (a, b) in PASSTHROUGH_CONF if re.search(a, enc_opts) and b in dec_opts.split()[0]]
+        passthrough = [(a, b) for (a, b) in PASSTHROUGH_CONF if re.search(a, enc_opts) and b in dec_opts.split()]
         if len(passthrough) != 1:
             pytest.skip()
 
-- 
GitLab


From 65b558c911bb166c10b6430a839252352fb37cb8 Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Wed, 2 Oct 2024 16:47:24 +0200
Subject: [PATCH 10/11] Another fix in the pass-through filter

---
 tests/codec_be_on_mr_nonselection/test_param_file.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py
index a8e24ca393..9e856140f5 100644
--- a/tests/codec_be_on_mr_nonselection/test_param_file.py
+++ b/tests/codec_be_on_mr_nonselection/test_param_file.py
@@ -58,8 +58,8 @@ PASSTHROUGH_CONF = [
     (r'-sba\s?[-+]?1',"FOA"),
     (r'-sba\s?[-+]?2',"HOA2"),
     (r'-sba\s?[-+]?3',"HOA3"),
-    (r'-mc\s5_1',"5_1"),
-    (r'-mc\s7_1',"7_1"),
+    (r'-mc\s5_1\s',"5_1"),
+    (r'-mc\s7_1\s',"7_1"),
     (r'-mc\s5_1_4',"5_1_4"),
     (r'-mc\s7_1_4',"7_1_4"),
 ]
-- 
GitLab


From 95a6bd2406f39b3403f1bd2678a994a9c32f9cb3 Mon Sep 17 00:00:00 2001
From: Erik Norvell <erik.norvell@ericsson.com>
Date: Mon, 7 Oct 2024 19:14:30 +0200
Subject: [PATCH 11/11] Formal improvements to test_param_file.py

---
 .../test_param_file.py                        | 54 ++++++++++++-------
 1 file changed, 36 insertions(+), 18 deletions(-)

diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py
index 9e856140f5..5d6b5cbaa1 100644
--- a/tests/codec_be_on_mr_nonselection/test_param_file.py
+++ b/tests/codec_be_on_mr_nonselection/test_param_file.py
@@ -53,15 +53,15 @@ from tests.constants import (
 )
 
 PASSTHROUGH_CONF = [
-    (r'[\s\S]*',"EXT"), # Any input with EXT output is pass-through
-    (r'-stereo',"STEREO"),
-    (r'-sba\s?[-+]?1',"FOA"),
-    (r'-sba\s?[-+]?2',"HOA2"),
-    (r'-sba\s?[-+]?3',"HOA3"),
-    (r'-mc\s5_1\s',"5_1"),
-    (r'-mc\s7_1\s',"7_1"),
-    (r'-mc\s5_1_4',"5_1_4"),
-    (r'-mc\s7_1_4',"7_1_4"),
+    (r"[\s\S]*", "EXT"),  # Any input with EXT output is pass-through
+    (r"-stereo", "STEREO"),
+    (r"-sba\s?[-+]?1", "FOA"),
+    (r"-sba\s?[-+]?2", "HOA2"),
+    (r"-sba\s?[-+]?3", "HOA3"),
+    (r"-mc\s5_1\s", "5_1"),
+    (r"-mc\s7_1\s", "7_1"),
+    (r"-mc\s5_1_4", "5_1_4"),
+    (r"-mc\s7_1_4", "7_1_4"),
 ]
 
 VALID_DEC_OUTPUT_CONF = [
@@ -179,9 +179,15 @@ def test_param_file_tests(
 
     # If compare_to_input is set, only run pass-through test cases
     if compare_to_input:
-        passthrough = [(a, b) for (a, b) in PASSTHROUGH_CONF if re.search(a, enc_opts) and b in dec_opts.split()]
+        passthrough = [
+            (a, b)
+            for (a, b) in PASSTHROUGH_CONF
+            if re.search(a, enc_opts) and b in dec_opts.split()
+        ]
         if len(passthrough) != 1:
-            pytest.skip()
+            pytest.skip(
+                "All non-passthrough modes are skipped when --compare-to-input is set"
+            )
 
     tag_str = convert_test_string_to_tag(test_tag)
 
@@ -418,17 +424,17 @@ def test_param_file_tests(
         # shift differences between the two signals - cut longer signal to shorter size
         allow_differing_lengths = not tracefile_last_rtp_numbers_differ
 
-        md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)
-
         if compare_to_input:
             # If comparing to input, set input as reference and allow different length
-            ref_output_file = testv_file
+            ref_file = testv_file
             allow_differing_lengths = True
             input_md = [x for x in enc_split if "csv" in x]
+        else:
+            ref_file = ref_output_file
 
         fs = int(sampling_rate) * 1000
         output_differs, reason = cmp_pcm(
-            ref_output_file,
+            ref_file,
             dut_output_file,
             output_config,
             fs,
@@ -449,11 +455,23 @@ def test_param_file_tests(
             record_property(k, v)
 
         metadata_differs = False
+
+        md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)
+
         if compare_to_input:
-            md_file_pairs = [(Path(f"{dut_base_path}/param_file/dec/{md_file}"), md_in_file) for (md_file,md_in_file) in zip(md_out_files,input_md)]
+            md_file_pairs = [
+                (Path(f"{dut_base_path}/param_file/dec/{md_file}"), md_in_file)
+                for (md_file, md_in_file) in zip(md_out_files, input_md)
+            ]
         else:
-            md_file_pairs = [(Path(f"{dut_base_path}/param_file/dec/{md_file}"), Path(f"{reference_path}/param_file/dec/{md_file}")) for md_file in md_out_files]
-        for (dut_metadata_file, ref_metadata_file) in md_file_pairs:
+            md_file_pairs = [
+                (
+                    Path(f"{dut_base_path}/param_file/dec/{md_file}"),
+                    Path(f"{reference_path}/param_file/dec/{md_file}"),
+                )
+                for md_file in md_out_files
+            ]
+        for dut_metadata_file, ref_metadata_file in md_file_pairs:
             md_file = os.path.basename(dut_metadata_file)
             try:
                 if not filecmp.cmp(dut_metadata_file, ref_metadata_file):
-- 
GitLab