From 3d689af690ba336f70edec8e09c7053086127a6b Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 7 Mar 2024 10:50:28 +0100 Subject: [PATCH 1/4] add 5ms and 10ms runs to sanitizer tests --- .gitlab-ci.yml | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 157c565d0f..51a4bf0406 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -201,7 +201,16 @@ stages: - make clean - make -j CLANG=$CLANG_NUM - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 + - exit_code20=0 + - exit_code10=0 + - exit_code5=0 + + - python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi # --------------------------------------------------------------- @@ -283,12 +292,18 @@ stages: expire_in: 1 week when: always paths: - - report-junit.xml - - report.html + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + - report-20ms.html + - report-10ms.html + - report-5ms.html expose_as: "Sanitizer selftest results" reports: junit: - - report-junit.xml + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml .sanitizer-selftest-ltv: stage: test @@ -299,12 +314,18 @@ stages: expire_in: 2 week when: always paths: - - report-junit.xml - - report.html + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + - report-20ms.html + - report-10ms.html + - report-5ms.html expose_as: "Sanitizer selftest results" reports: junit: - - report-junit.xml + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml # --------------------------------------------------------------- # .pre jobs for setting up things -- GitLab From 548762f4fb1cf61e29838f18773d681f7c15575e Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 7 Mar 2024 13:02:44 +0100 Subject: [PATCH 2/4] check for sanitizer errors in the reports instead --- .gitlab-ci.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 946af7aa44..4c2b943464 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -202,15 +202,30 @@ stages: - make -j CLANG=$CLANG_NUM - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 + - exit_code20=0 - exit_code10=0 - exit_code5=0 + - errors20=0 + - errors10=0 + - errors5=0 + + - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + # ignore exit codes, instead search in the report for errors - python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - errors20=$(cat report-20ms.html | grep -c "$sanitizer_type") || true + - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - errors10=$(cat report-10ms.html | grep -c "$sanitizer_type") || true + - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - errors5=$(cat report-5ms.html | grep -c "sanitizer_type") || true - - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi + - if [ $errors20 -ne 0 ]; then echo "CLANG $sanitizer_type found errors in 20ms framesize decoding"; fi + - if [ $errors10 -ne 0 ]; then echo "CLANG $sanitizer_type found errors in 10ms framesize decoding"; fi + - if [ $errors5 -ne 0 ]; then echo "CLANG $sanitizer_type found errors in 5ms framesize decoding"; fi + - if [ $errors20 -ne 0 ] || [ $errors10 -ne 0 ] || [ $errors5 -ne 0 ]; then exit 1; fi # --------------------------------------------------------------- -- GitLab From baeef46b4b50947441001f6b5997c9ce01686c9a Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 18 Mar 2024 16:44:03 +0100 Subject: [PATCH 3/4] add cmdl arg for tolerance to have better error reporting --- .gitlab-ci.yml | 18 ++++-------------- tests/cmp_pcm.py | 4 +++- .../test_masa_enc_dec.py | 3 +++ .../test_param_file.py | 2 ++ .../test_sba_bs_dec_plc.py | 4 ++++ .../test_sba_bs_enc.py | 12 ++++++++++++ tests/conftest.py | 19 ++++++++++++++++++- 7 files changed, 46 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4c2b943464..b9cff4b364 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -206,26 +206,16 @@ stages: - exit_code20=0 - exit_code10=0 - exit_code5=0 - - errors20=0 - - errors10=0 - - errors5=0 - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi # ignore exit codes, instead search in the report for errors - python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? - - errors20=$(cat report-20ms.html | grep -c "$sanitizer_type") || true + # run with high enough tolerance to have no failures due to non-BE -> the only detected failures will be crashes from the sanitizers + - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? - - errors10=$(cat report-10ms.html | grep -c "$sanitizer_type") || true - - - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? - - errors5=$(cat report-5ms.html | grep -c "sanitizer_type") || true - - - if [ $errors20 -ne 0 ]; then echo "CLANG $sanitizer_type found errors in 20ms framesize decoding"; fi - - if [ $errors10 -ne 0 ]; then echo "CLANG $sanitizer_type found errors in 10ms framesize decoding"; fi - - if [ $errors5 -ne 0 ]; then echo "CLANG $sanitizer_type found errors in 5ms framesize decoding"; fi - - if [ $errors20 -ne 0 ] || [ $errors10 -ne 0 ] || [ $errors5 -ne 0 ]; then exit 1; fi + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi # --------------------------------------------------------------- diff --git a/tests/cmp_pcm.py b/tests/cmp_pcm.py index 13a1c1ba5a..649eb66c54 100755 --- a/tests/cmp_pcm.py +++ b/tests/cmp_pcm.py @@ -12,7 +12,7 @@ import pyaudio3dtools import pyivastest -def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str): +def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0, abs_tol=0) -> (int, str): """ Compare 2 PCM files for bitexactness """ @@ -51,6 +51,8 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str if cmp_result["bitexact"]: return 0, "SUCCESS: Files are bitexact" + elif cmp_result["max_abs_diff"] <= abs_tol: + return 0, "SUCCESS: Maximum absolute diff below threshold" else: diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)" first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)" diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py index 75b01db382..541aec589c 100644 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py @@ -110,6 +110,7 @@ def test_masa_enc_dec( get_mld, get_mld_lim, decoder_only, + abs_tol, ): # Input parameters in_fs = 48 @@ -214,6 +215,7 @@ def test_masa_enc_dec( int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) if get_mld: mld = 0 @@ -246,6 +248,7 @@ def test_masa_enc_dec( int(out_fs * 1000), get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) if get_mld: mld = 0 diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 459964a186..ab78a1120d 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -141,6 +141,7 @@ def test_param_file_tests( test_tag, get_mld, get_mld_lim, + abs_tol, ): enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag] @@ -338,6 +339,7 @@ def test_param_file_tests( fs, get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config) diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py index 11a8aeca04..eb6a45b2ec 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py @@ -90,6 +90,7 @@ def test_sba_plc_system( gain_flag, get_mld, get_mld_lim, + abs_tol, ): SID = 0 if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: @@ -128,6 +129,7 @@ def test_sba_plc_system( keep_files, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -151,6 +153,7 @@ def sba_dec_plc( keep_files, get_mld=False, get_mld_lim=0, + abs_tol=0, ): # ------------ run cmd ------------ @@ -207,6 +210,7 @@ def sba_dec_plc( fs, get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) if get_mld: mld = 0 diff --git a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py index 4f4c1a3271..03cb9d29de 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py +++ b/tests/codec_be_on_mr_nonselection/test_sba_bs_enc.py @@ -105,6 +105,7 @@ def test_pca_enc( get_mld, get_mld_lim, decoder_only, + abs_tol, ): pca = True tag = tag + fs + "c" @@ -159,6 +160,7 @@ def test_pca_enc( get_mld=get_mld, get_mld_lim=get_mld_lim, pca=pca, + abs_tol=abs_tol, ) @@ -190,6 +192,7 @@ def test_sba_enc_system( get_mld, get_mld_lim, decoder_only, + abs_tol, ): if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -264,6 +267,7 @@ def test_sba_enc_system( decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -286,6 +290,7 @@ def test_spar_hoa2_enc_system( get_mld, get_mld_lim, decoder_only, + abs_tol, ): fs = "48" dtx = "0" @@ -337,6 +342,7 @@ def test_spar_hoa2_enc_system( decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -359,6 +365,7 @@ def test_spar_hoa3_enc_system( get_mld, get_mld_lim, decoder_only, + abs_tol, ): fs = "48" dtx = "0" @@ -410,6 +417,7 @@ def test_spar_hoa3_enc_system( decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -436,6 +444,7 @@ def test_sba_enc_BWforce_system( get_mld, get_mld_lim, decoder_only, + abs_tol, ): if dtx == "1" and ivas_br not in ["32000", "64000"]: # skip high bitrates for DTX until DTX issue is resolved @@ -493,6 +502,7 @@ def test_sba_enc_BWforce_system( decoder_only, get_mld=get_mld, get_mld_lim=get_mld_lim, + abs_tol=abs_tol, ) @@ -662,6 +672,7 @@ def sba_dec( get_mld=False, get_mld_lim=0, pca=False, + abs_tol=0, ): # -------- run cmd ------------ # sampling rate to BW mapping @@ -725,6 +736,7 @@ def sba_dec( fs, get_mld=get_mld, mld_lim=get_mld_lim, + abs_tol=abs_tol, ) if get_mld: mld = 0 diff --git a/tests/conftest.py b/tests/conftest.py index 37b5281403..c7b053dcb3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -198,6 +198,13 @@ def pytest_addoption(parser): default=False, ) + parser.addoption( + "--abs_tol", + help="Tolerance for absolute difference in PCM output. Differences below this threshold are ignored when deciding about PASS or FAIL.", + type=int, + default=0, + ) + @pytest.fixture(scope="session", autouse=True) def update_ref(request): @@ -235,6 +242,14 @@ def get_mld_lim(request): return float(request.config.getoption("--mld-lim")) +@pytest.fixture(scope="session") +def abs_tol(request) -> int: + """ + Return tolerance value for absolute diff in PCM output. + """ + return request.config.option.abs_tol + + @pytest.fixture(scope="session") def keep_files(request) -> bool: """ @@ -332,7 +347,9 @@ class EncoderFrontend: log_dbg_msg(f"{self._type} encoder command:\n{cmd_str}") try: - result = run(command, capture_output=True, check=False, timeout=self.timeout) + result = run( + command, capture_output=True, check=False, timeout=self.timeout + ) except TimeoutExpired: pytest.fail(f"{self._type} encoder run timed out after {self.timeout}s.") -- GitLab From efc94d9c232e8fabb696012400181c662890014a Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 19 Mar 2024 12:33:23 +0100 Subject: [PATCH 4/4] add/modify swome comments --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9cff4b364..f6648a5274 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -209,9 +209,11 @@ stages: - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi - # ignore exit codes, instead search in the report for errors + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? - # run with high enough tolerance to have no failures due to non-BE -> the only detected failures will be crashes from the sanitizers + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? -- GitLab