From 8bd15c8bb644d8abb90af6eaae76b66f0983beb7 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 31 Jan 2024 09:48:18 +0100 Subject: [PATCH 01/11] convert msan and asan MR jobs to use pytest instead of self_test.py --- .gitlab-ci.yml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b01a02bf5e..154d79c252 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -449,17 +449,22 @@ codec-msan: - *print-common-info - make clean - make -j CLANG=1 - - python3 scripts/self_test.py -z console --create | tee test_output.txt - - run_errors=$(cat test_output.txt | grep -ic "run errors") || true - - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang memory-sanitizer"; exit 1; fi + - testcase_timeout=180 + - exit_code=0 + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - if [ $zero_errors != 0 || exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week when: always paths: - - scripts/ref/logs/ - - test_output.txt + - report-junit.xml + - report.html expose_as: "msan selftest results" + reports: + junit: + - report-junit.xml # code selftest testvectors with address-sanitizer binaries codec-asan: @@ -472,17 +477,20 @@ codec-asan: - *print-common-info - make clean - make -j CLANG=2 - - python3 scripts/self_test.py -z console --create | tee test_output.txt - - run_errors=$(cat test_output.txt | grep -ic "run errors") || true - - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang address-sanitizer"; exit 1; fi + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - if [ $zero_errors != 0 || exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week when: always paths: - - scripts/ref/logs/ - - test_output.txt + - report-junit.xml + - report.html expose_as: "asan selftest results" + reports: + junit: + - report-junit.xml # code selftest testvectors with address-sanitizer binaries codec-usan: -- GitLab From cd015db74d0d153d6f9b946823eef568dee1de9e Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 12:14:45 +0100 Subject: [PATCH 02/11] convert usan job as well --- .gitlab-ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 154d79c252..a30b1610f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -477,6 +477,8 @@ codec-asan: - *print-common-info - make clean - make -j CLANG=2 + - testcase_timeout=180 + - exit_code=0 - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 0 || exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi @@ -503,17 +505,23 @@ codec-usan: - *print-common-info - make clean - make -j CLANG=3 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 scripts/self_test.py -z console --create + - testcase_timeout=180 + - exit_code=0 + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - grep_exit_code=0 - - grep UndefinedBehaviorSanitizer scripts/ref/logs/* || grep_exit_code=$? - - if [ $grep_exit_code != 1 ] ; then echo "Run errors in self_test.py with Clang undefined-behavior-sanitizer"; exit 1; fi + - grep UndefinedBehaviorSanitizer report.html || grep_exit_code=$? + - if [ $grep_exit_code != 1 ] ; then echo "Run errors with Clang undefined-behavior-sanitizer"; exit 1; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week when: always paths: - - scripts/ref/logs/ + - report-junit.xml + - report.html expose_as: "usan selftest results" + reports: + junit: + - report-junit.xml # test renderer executable renderer-smoke-test: -- GitLab From d7048bb75df05c8b01d05dfc29e4087092c384fe Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 12:19:33 +0100 Subject: [PATCH 03/11] point to codec binaries in pytest call --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a30b1610f2..9a8ebdd5f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -451,7 +451,7 @@ codec-msan: - make -j CLANG=1 - testcase_timeout=180 - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 0 || exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi artifacts: @@ -479,7 +479,7 @@ codec-asan: - make -j CLANG=2 - testcase_timeout=180 - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 0 || exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi artifacts: @@ -507,7 +507,7 @@ codec-usan: - make -j CLANG=3 - testcase_timeout=180 - exit_code=0 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - grep_exit_code=0 - grep UndefinedBehaviorSanitizer report.html || grep_exit_code=$? - if [ $grep_exit_code != 1 ] ; then echo "Run errors with Clang undefined-behavior-sanitizer"; exit 1; fi -- GitLab From 859ad67e40a1a5441b307ab157c395cc89b9f1c5 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 12:27:13 +0100 Subject: [PATCH 04/11] intentionally insert an asan error to test CI job --- lib_enc/ivas_mdct_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/ivas_mdct_core_enc.c b/lib_enc/ivas_mdct_core_enc.c index f37eeb4351..2b87df3f42 100644 --- a/lib_enc/ivas_mdct_core_enc.c +++ b/lib_enc/ivas_mdct_core_enc.c @@ -631,7 +631,7 @@ void ivas_mdct_core_whitening_enc( } } - for ( ch = 0; ch < CPE_CHANNELS; ch++ ) + for ( ch = 0; ch < CPE_CHANNELS + 1; ch++ ) { orig_spectrum[ch][0] = orig_spectrum_long[ch]; orig_spectrum[ch][1] = orig_spectrum_long[ch] + N_TCX10_MAX; -- GitLab From 676d819dbd223c46c1235172f7c52f50ebb46430 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 12:31:02 +0100 Subject: [PATCH 05/11] remove suppression file to test error deteection --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a8ebdd5f5..9d78f26756 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -507,7 +507,7 @@ codec-usan: - make -j CLANG=3 - testcase_timeout=180 - exit_code=0 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - grep_exit_code=0 - grep UndefinedBehaviorSanitizer report.html || grep_exit_code=$? - if [ $grep_exit_code != 1 ] ; then echo "Run errors with Clang undefined-behavior-sanitizer"; exit 1; fi -- GitLab From 69990f5938b3b9d37bb52ac1855fc1fc77289580 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 12:43:29 +0100 Subject: [PATCH 06/11] fix possible whitespace error --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d78f26756..66f23e1e53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -451,7 +451,7 @@ codec-msan: - make -j CLANG=1 - testcase_timeout=180 - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 0 || exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi artifacts: @@ -479,7 +479,7 @@ codec-asan: - make -j CLANG=2 - testcase_timeout=180 - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 0 || exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi artifacts: @@ -507,7 +507,7 @@ codec-usan: - make -j CLANG=3 - testcase_timeout=180 - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - grep_exit_code=0 - grep UndefinedBehaviorSanitizer report.html || grep_exit_code=$? - if [ $grep_exit_code != 1 ] ; then echo "Run errors with Clang undefined-behavior-sanitizer"; exit 1; fi -- GitLab From c9f72227400ca2bd3249f09f799709e16ed634fb Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 12:58:48 +0100 Subject: [PATCH 07/11] fix check for failure --- .gitlab-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66f23e1e53..1c81c0629b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -452,8 +452,7 @@ codec-msan: - testcase_timeout=180 - exit_code=0 - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 0 || exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi + - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week @@ -480,8 +479,7 @@ codec-asan: - testcase_timeout=180 - exit_code=0 - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 0 || exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi + - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week -- GitLab From 73a93c37d789fa2cc78516ff78ab9e9690f2a720 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 14:28:45 +0100 Subject: [PATCH 08/11] make pytest cases explicitly fail when there is undef behaviour found --- tests/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index cccae2e906..4250c2fe87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -292,6 +292,8 @@ class EncoderFrontend: pytest.fail( f"{self._type} encoder terminated with a non-0 return code: {self.returncode}" ) + if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr: + pytest.fail("Undefined Behaviour runtime error encountered") def _check_run(self): if self.returncode is not None: @@ -485,6 +487,8 @@ class DecoderFrontend: pytest.fail( f"{self._type} decoder terminated with a non-0 return code: {self.returncode}" ) + if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr: + pytest.fail("Undefined Behaviour runtime error encountered") def _check_run(self): if self.returncode is not None: -- GitLab From 5f66382b2879400f5cec596a07fc5fa30f5f2723 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 14:30:02 +0100 Subject: [PATCH 09/11] adapt yml file --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c81c0629b..9456b0f9fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -506,9 +506,7 @@ codec-usan: - testcase_timeout=180 - exit_code=0 - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - - grep_exit_code=0 - - grep UndefinedBehaviorSanitizer report.html || grep_exit_code=$? - - if [ $grep_exit_code != 1 ] ; then echo "Run errors with Clang undefined-behavior-sanitizer"; exit 1; fi + - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang undefined-behavior-sanitizer"; exit 1; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week -- GitLab From a705034209038690fe1c7f75f5c21d5dff5091f1 Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 14:39:46 +0100 Subject: [PATCH 10/11] Revert "remove suppression file to test error deteection" This reverts commit 676d819dbd223c46c1235172f7c52f50ebb46430. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9456b0f9fa..e2230ceafc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -505,7 +505,7 @@ codec-usan: - make -j CLANG=3 - testcase_timeout=180 - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang undefined-behavior-sanitizer"; exit 1; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" -- GitLab From 5813c4a0a8700ace13e874c98767e6e266232cde Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 1 Feb 2024 14:39:55 +0100 Subject: [PATCH 11/11] Revert "intentionally insert an asan error to test CI job" This reverts commit 859ad67e40a1a5441b307ab157c395cc89b9f1c5. --- lib_enc/ivas_mdct_core_enc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_enc/ivas_mdct_core_enc.c b/lib_enc/ivas_mdct_core_enc.c index 2b87df3f42..f37eeb4351 100644 --- a/lib_enc/ivas_mdct_core_enc.c +++ b/lib_enc/ivas_mdct_core_enc.c @@ -631,7 +631,7 @@ void ivas_mdct_core_whitening_enc( } } - for ( ch = 0; ch < CPE_CHANNELS + 1; ch++ ) + for ( ch = 0; ch < CPE_CHANNELS; ch++ ) { orig_spectrum[ch][0] = orig_spectrum_long[ch]; orig_spectrum[ch][1] = orig_spectrum_long[ch] + N_TCX10_MAX; -- GitLab