diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 42753e8e98e3a11105366ac9ca5f5a2a7e570f9f..f6648a5274afd72340ddcfafef1cf829c41926c6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -201,7 +201,23 @@ stages: - make clean - make -j CLANG=$CLANG_NUM - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 + + - exit_code20=0 + - exit_code10=0 + - exit_code5=0 + + - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode + - python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi # --------------------------------------------------------------- @@ -283,12 +299,18 @@ stages: expire_in: 1 week when: always paths: - - report-junit.xml - - report.html + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + - report-20ms.html + - report-10ms.html + - report-5ms.html expose_as: "Sanitizer selftest results" reports: junit: - - report-junit.xml + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml .sanitizer-selftest-ltv: stage: test @@ -299,12 +321,18 @@ stages: expire_in: 2 week when: always paths: - - report-junit.xml - - report.html + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + - report-20ms.html + - report-10ms.html + - report-5ms.html expose_as: "Sanitizer selftest results" reports: junit: - - report-junit.xml + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml # --------------------------------------------------------------- # .pre jobs for setting up things diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index c7df4b800d8f47db19131ef43b216c57adb43180..709c8ddb8dfc5ac4307e82452122f6b96d3183b2 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -12,7 +12,7 @@ import pyaudio3dtools import pyivastest -def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str): +def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0, abs_tol=0) -> (int, str): """ Compare 2 PCM files for bitexactness """ @@ -49,10 +49,13 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str s1, s2, fs, per_frame=False, get_mld=get_mld ) + output_differs = 0 reason = "SUCCESS: Files are bitexact" - if not cmp_result["bitexact"]: + if not cmp_result["bitexact"] and cmp_result["max_abs_diff"] <= abs_tol: + reason = "SUCCESS: Maximum absolute diff below threshold" + elif not cmp_result["bitexact"]: diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" print(diff_msg) diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 9dc8ce9e6dbcae08effdb32ddf9f92a1fe4ef514..78bf427fe9c746497e1778f7ed598b78f3e2c25b 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -113,6 +113,7 @@ def test_masa_enc_dec( get_mld, get_mld_lim, decoder_only, + abs_tol, ): # Input parameters in_fs = 48 @@ -217,6 +218,7 @@ def test_masa_enc_dec( int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) if get_mld: mld = re.search(MLD_PATTERN, reason).groups(1)[0] @@ -253,6 +255,7 @@ def test_masa_enc_dec( int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) if get_mld: mld = re.search(MLD_PATTERN, reason).groups(1)[0] diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 890d530f825561e6a9e89e16ee8bdcff427b2ee1..12e9208da89f6d57edf4e4cc486ffcdadea4ae5b 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -143,6 +143,7 @@ def test_param_file_tests( test_tag, get_mld, get_mld_lim, + abs_tol, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -340,6 +341,7 @@ def test_param_file_tests( fs, get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 5b52b9bd2f1c4f42bfd96b49a90b6a415f8ce784..a11d1d30bf840b5487672f5c5a7a597803092bc6 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -92,6 +92,7 @@ def test_sba_plc_system( gain_flag, get_mld, get_mld_lim, + abs_tol, ): SID = 0 if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: @@ -130,6 +131,7 @@ def test_sba_plc_system( keep_files, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -153,6 +155,7 @@ def sba_dec_plc( keep_files, get_mld=False, get_mld_lim=0, + abs_tol=0, ): # ------------ run cmd ------------ @@ -209,6 +212,7 @@ def sba_dec_plc( fs, get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) if get_mld: mld = re.search(MLD_PATTERN, reason).groups(1)[0] diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 0a07a0d1d7a6348b898111815c8293e3a1303195..b7effa986c272f4799583589e2d12262f026c745 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -107,6 +107,7 @@ def test_pca_enc( get_mld, get_mld_lim, decoder_only, + abs_tol, ): pca = True tag = tag + fs + "c" @@ -161,6 +162,7 @@ def test_pca_enc( get_mld=get_mld, get_mld_lim=get_mld_lim, pca=pca, + abs_tol=abs_tol, ) @@ -192,6 +194,7 @@ def test_sba_enc_system( get_mld, get_mld_lim, decoder_only, + abs_tol, ): if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -266,6 +269,7 @@ def test_sba_enc_system( decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -288,6 +292,7 @@ def test_spar_hoa2_enc_system( get_mld, get_mld_lim, decoder_only, + abs_tol, ): fs = "48" dtx = "0" @@ -339,6 +344,7 @@ def test_spar_hoa2_enc_system( decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -361,6 +367,7 @@ def test_spar_hoa3_enc_system( get_mld, get_mld_lim, decoder_only, + abs_tol, ): fs = "48" dtx = "0" @@ -412,6 +419,7 @@ def test_spar_hoa3_enc_system( decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -438,6 +446,7 @@ def test_sba_enc_BWforce_system( get_mld, get_mld_lim, decoder_only, + abs_tol, ): if dtx == "1" and ivas_br not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -495,6 +504,7 @@ def test_sba_enc_BWforce_system( decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -664,6 +674,7 @@ def sba_dec( get_mld=False, get_mld_lim=0, pca=False, + abs_tol=0, ): # -------- run cmd ------------ # sampling rate to BW mapping @@ -727,6 +738,7 @@ def sba_dec( fs, get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) if get_mld: mld = re.search(MLD_PATTERN, reason).groups(1)[0] diff --git a/tests/conftest.py b/tests/conftest.py index 07004cad091aac0851115c9854bc99a64c1e1f80..c975e409c01ed4a2401e146d6946cdaba37bb7ba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -197,6 +197,13 @@ def pytest_addoption(parser): default=False, ) + parser.addoption( + "--abs_tol", + help="Tolerance for absolute difference in PCM output. Differences below this threshold are ignored when deciding about PASS or FAIL.", + type=int, + default=0, + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -234,6 +241,14 @@ def get_mld_lim(request): return float(request.config.getoption("--mld-lim")) +@pytest.fixture(scope="session") +def abs_tol(request) -> int: + """ + Return tolerance value for absolute diff in PCM output. + """ + return request.config.option.abs_tol + + @pytest.fixture(scope="session") def keep_files(request) -> bool: """ @@ -331,7 +346,9 @@ class EncoderFrontend: log_dbg_msg(f"{self._type} encoder command:\n{cmd_str}") try: - result = run(command, capture_output=True, check=False, timeout=self.timeout) + result = run( + command, capture_output=True, check=False, timeout=self.timeout + ) except TimeoutExpired: pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.")