diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e5c88431ce306aa3f4e8267a42f5f42330b2e52..625a3cc8bcd812e91926c2e909a1188ae8016e3e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,7 @@ variables: - 'ivas-conformance' - 'ivas-conformance-linux' - 'check-float-reference' + - 'test-branch-vs-input-passthrough' GIT_CLEAN_FLAGS: -ffdxq TESTCASE_TIMEOUT_STV_SANITIZERS: 180 TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200 @@ -69,7 +70,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-float-reference' variables: IVAS_PIPELINE_NAME: 'check-float-reference: $CI_COMMIT_BRANCH' - + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + variables: + IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH' stages: - .pre @@ -265,6 +268,8 @@ stages: when: never - if: $MANUAL_PIPELINE_TYPE == 'check-float-reference' when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + when: never - when: on_success .rules-merge-request: @@ -1639,6 +1644,40 @@ test-long-self-test: junit: - report-junit-ltv.xml +test-branch-vs-input-passthrough: + tags: + - ivas-linux + stage: compare + timeout: "30 minutes" # TBD + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + allow_failure: + exit_codes: + - 123 + script: + - *print-common-info + - make -j + + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py report-junit.xml report.csv + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered."; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + artifacts: + name: "test-branch-vs-input-passthrough--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 4 mos + paths: + - report-junit.xml + - report.html + - report.csv + expose_as: "test-branch-vs-input-passthrough results" + reports: + junit: + - report-junit.xml # --------------------------------------------------------------- # Scheduled jobs on main diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 3be7a9ca0b62e62d2469a07c9142a8fdd3879a84..e0f02238e4fddd970567023b2637975e4ead4213 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -31,6 +31,7 @@ EVS_FORMATS = { "EVS_JBM_dec": r"Readme_JBM_dec", } +NO_FORMATS = {"Default": r".*"} IVAS_CATEGORIES = { "Normal operation": r".*", @@ -48,6 +49,8 @@ EVS_CATEGORIES = { "JBM": r"JBM", } +NO_CATEGORIES = {"N/A": r".*"} + # Main routine if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -64,15 +67,23 @@ if __name__ == "__main__": action="store_true", help="Parse using EVS 26.444 formats", ) + parser.add_argument( + "--skip_formats", + action="store_true", + help="Parse without formats and categories. Suitable for general tests which do not match the IVAS categories.", + ) args = parser.parse_args() xml_report = args.xml_report csv_file = args.csv_file + FORMATS = IVAS_FORMATS + CATEGORIES = IVAS_CATEGORIES if args.evs: FORMATS = EVS_FORMATS CATEGORIES = EVS_CATEGORIES - else: - FORMATS = IVAS_FORMATS - CATEGORIES = IVAS_CATEGORIES + if args.skip_formats: + FORMATS = NO_FORMATS + CATEGORIES = NO_CATEGORIES + tree = ElementTree.parse(xml_report) testsuite = tree.find(".//testsuite") diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 67aeff0dd96ac91c2a2241972a37e9db1a0e2b6a..1593814136e5bd233aacab596cb5845e8afa7497 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -59,7 +59,18 @@ def cmp_pcm( # In case of wav input, override the nchannels with the one from the wav header nchannels = s1.shape[1] - if s1.shape != s2.shape and not allow_differing_lengths: + # In case number of channels do not match, fail already now. Could happen in case of + # comparison to input with for a non-passthrough mode. + if s1.shape[1] != s2.shape[1]: + reason = "FAIL: Number of channels differ." + return 1, reason + + if allow_differing_lengths: + # to allow for MLD comparison, pad shorter file + max_len = max(s1.shape[0], s2.shape[0]) + s1 = np.pad(s1,((0,max_len - s1.shape[0]),(0,0)),mode='constant',constant_values=0) + s2 = np.pad(s2,((0,max_len - s2.shape[0]),(0,0)),mode='constant',constant_values=0) + elif s1.shape != s2.shape: print( f"file size in samples: file 1 = {s1.shape[0]},", f"file 2 = {s2.shape[0]}", @@ -175,6 +186,8 @@ if __name__ == "__main__": parser.add_argument("--get_mld", action="store_true") parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim") parser.add_argument("--get_odg", action="store_true") + parser.add_argument("--get_ssnr", action="store_true") + parser.add_argument("--allow_differing_lengths", action="store_true") args = parser.parse_args() result, msg = cmp_pcm(**vars(args)) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 34bffea7b60bbffee745bf81c21993ff92032dc6..5d6b5cbaa1fa7b3bb126d6da34f7c7e643d401bb 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -39,6 +39,7 @@ import platform from pathlib import Path from subprocess import run import pytest +import re import numpy as np from tests.cmp_pcm import cmp_pcm @@ -51,6 +52,17 @@ from tests.constants import ( SCRIPTS_DIR, ) +PASSTHROUGH_CONF = [ + (r"[\s\S]*", "EXT"), # Any input with EXT output is pass-through + (r"-stereo", "STEREO"), + (r"-sba\s?[-+]?1", "FOA"), + (r"-sba\s?[-+]?2", "HOA2"), + (r"-sba\s?[-+]?3", "HOA3"), + (r"-mc\s5_1\s", "5_1"), + (r"-mc\s7_1\s", "7_1"), + (r"-mc\s5_1_4", "5_1_4"), + (r"-mc\s7_1_4", "7_1_4"), +] VALID_DEC_OUTPUT_CONF = [ "MONO", @@ -161,9 +173,22 @@ def test_param_file_tests( get_ssnr, get_enc_stats, get_odg, + compare_to_input, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] + # If compare_to_input is set, only run pass-through test cases + if compare_to_input: + passthrough = [ + (a, b) + for (a, b) in PASSTHROUGH_CONF + if re.search(a, enc_opts) and b in dec_opts.split() + ] + if len(passthrough) != 1: + pytest.skip( + "All non-passthrough modes are skipped when --compare-to-input is set" + ) + tag_str = convert_test_string_to_tag(test_tag) # evaluate encoder options @@ -378,7 +403,7 @@ def test_param_file_tests( # set to false per default even if this is no JBM case - makes later check for failure easier tracefile_last_rtp_numbers_differ = False - if len(tracefile_dec) > 0: + if len(tracefile_dec) > 0 and not compare_to_input: dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}" ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}" @@ -399,9 +424,17 @@ def test_param_file_tests( # shift differences between the two signals - cut longer signal to shorter size allow_differing_lengths = not tracefile_last_rtp_numbers_differ + if compare_to_input: + # If comparing to input, set input as reference and allow different length + ref_file = testv_file + allow_differing_lengths = True + input_md = [x for x in enc_split if "csv" in x] + else: + ref_file = ref_output_file + fs = int(sampling_rate) * 1000 output_differs, reason = cmp_pcm( - ref_output_file, + ref_file, dut_output_file, output_config, fs, @@ -414,7 +447,6 @@ def test_param_file_tests( ref_jbm_tf=ref_tracefile_dec, cut_jbm_tf=dut_tracefile_dec, ) - md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) cmp_result_msg += reason @@ -423,9 +455,24 @@ def test_param_file_tests( record_property(k, v) metadata_differs = False - for md_file in md_out_files: - dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}") - ref_metadata_file = Path(f"{reference_path}/param_file/dec/{md_file}") + + md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) + + if compare_to_input: + md_file_pairs = [ + (Path(f"{dut_base_path}/param_file/dec/{md_file}"), md_in_file) + for (md_file, md_in_file) in zip(md_out_files, input_md) + ] + else: + md_file_pairs = [ + ( + Path(f"{dut_base_path}/param_file/dec/{md_file}"), + Path(f"{reference_path}/param_file/dec/{md_file}"), + ) + for md_file in md_out_files + ] + for dut_metadata_file, ref_metadata_file in md_file_pairs: + md_file = os.path.basename(dut_metadata_file) try: if not filecmp.cmp(dut_metadata_file, ref_metadata_file): print("Output metadata differs for file: " + md_file) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 98f55b7fe57f89c8ca30609e968ddef31045459e..885ac488ad36b2ec2cad0b1ae762c4536fa0cecb 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -114,6 +114,7 @@ def test_pca_enc( get_ssnr, get_odg, get_enc_stats, + compare_to_input, ): pca = True bitrate = "256000" @@ -165,6 +166,7 @@ def test_pca_enc( sba_dec( record_property, props_to_record, + test_vector_path, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -187,6 +189,7 @@ def test_pca_enc( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + compare_to_input=compare_to_input, ) @@ -223,6 +226,7 @@ def test_sba_enc_system( get_ssnr, get_odg, get_enc_stats, + compare_to_input, ): plc_pattern = None @@ -333,6 +337,7 @@ def test_sba_enc_system( sba_dec( record_property, props_to_record, + test_vector_path, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -355,6 +360,7 @@ def test_sba_enc_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + compare_to_input=compare_to_input, ) @@ -383,6 +389,7 @@ def test_spar_hoa2_enc_system( get_ssnr, get_odg, get_enc_stats, + compare_to_input, ): sampling_rate = "48" pca = False @@ -467,6 +474,7 @@ def test_spar_hoa2_enc_system( sba_dec( record_property, props_to_record, + test_vector_path, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -489,6 +497,7 @@ def test_spar_hoa2_enc_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + compare_to_input=compare_to_input, ) @@ -517,6 +526,7 @@ def test_spar_hoa3_enc_system( get_ssnr, get_odg, get_enc_stats, + compare_to_input, ): sampling_rate = "48" pca = False @@ -595,6 +605,7 @@ def test_spar_hoa3_enc_system( sba_dec( record_property, props_to_record, + test_vector_path, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -617,6 +628,7 @@ def test_spar_hoa3_enc_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + compare_to_input=compare_to_input, ) @@ -649,6 +661,7 @@ def test_sba_enc_BWforce_system( get_ssnr, get_odg, get_enc_stats, + compare_to_input, ): sid = 0 plc_pattern = None @@ -743,6 +756,7 @@ def test_sba_enc_BWforce_system( sba_dec( record_property, props_to_record, + test_vector_path, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -765,6 +779,7 @@ def test_sba_enc_BWforce_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + compare_to_input=compare_to_input, ) @@ -801,6 +816,7 @@ def test_sba_plc_system( get_ssnr, get_odg, get_enc_stats, + compare_to_input, ): sid = 0 pca = False @@ -872,6 +888,7 @@ def test_sba_plc_system( sba_dec( record_property, props_to_record, + test_vector_path, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -894,6 +911,7 @@ def test_sba_plc_system( abs_tol=abs_tol, get_ssnr=get_ssnr, get_odg=get_odg, + compare_to_input=compare_to_input, ) @@ -1026,6 +1044,7 @@ def sba_enc( def sba_dec( record_property, props_to_record, + test_vector_path, dut_decoder_frontend, ref_decoder_frontend, reference_path, @@ -1048,6 +1067,7 @@ def sba_dec( abs_tol=0, get_ssnr=False, get_odg=False, + compare_to_input=False, ): dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt" ref_pkt_dir = f"{reference_path}/sba_bs/pkt" @@ -1115,6 +1135,12 @@ def sba_dec( plc_file=plc_file, ) + allow_differing_lengths = False + if compare_to_input: + # If comparing to input, set input as reference and allow different length + ref_out_file = f"{test_vector_path}/{tag}.wav" + allow_differing_lengths = True + sampling_rate_Hz = int(sampling_rate) * 1000 cmp_result, reason = cmp_pcm( dut_out_file, @@ -1124,6 +1150,7 @@ def sba_dec( get_mld=get_mld, mld_lim=get_mld_lim, abs_tol=abs_tol, + allow_differing_lengths=allow_differing_lengths, get_ssnr=get_ssnr, get_odg=get_odg, ) @@ -1136,8 +1163,3 @@ def sba_dec( # report compare result if cmp_result != 0: pytest.fail(text_to_parse) - - # remove DUT output files when test result is OK (to save disk space) - # if not keep_files: - # os.remove(dut_out_file) - # os.remove(dut_pkt_file) diff --git a/tests/conftest.py b/tests/conftest.py index 636881ecefa58e5394eb00afb7024e76b9347ebe..8f3e4e961bc7e87499d0c9af4642fe96d84f22ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -260,6 +260,12 @@ def pytest_addoption(parser): default=0, ) + parser.addoption( + "--compare_to_input", + action="store_true", + help="Compare output to the input file instead of reference output. N.B. Only applicable to pass-through tests.", + default=False, + ) @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -927,7 +933,7 @@ def reference_path(request) -> str: path = str(path.resolve()) - if request.config.option.update_ref == "0": + if request.config.option.update_ref == "0" and not request.config.option.compare_to_input: if not os.path.isdir(path): raise FileNotFoundError( f"REF path {path} not found!\nPlease generate the references, first!\n!" @@ -968,6 +974,12 @@ def encoder_only(request) -> bool: """ return request.config.getoption("--encoder_only") +@pytest.fixture(scope="session", autouse=True) +def compare_to_input(request) -> bool: + """ + Return value of cmdl param --compare_to_input + """ + return request.config.getoption("--compare_to_input") def pytest_configure(config): config.addinivalue_line("markers", "serial: mark test to run only in serial")