diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4795924c086de16496c2c39f03427f97d61b2203..9344bda9138d7cb73ae959e7f323fe462412729e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,6 +22,8 @@ variables: - 'test-long-self-test' - 'ivas-conformance' GIT_CLEAN_FLAGS: -ffdxq + TESTCASE_TIMEOUT_STV_SANITIZERS: 180 + TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200 default: @@ -186,6 +188,20 @@ stages: - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } +# to be reused in MR and LTV-scheduled sanitizer test jobs +# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section +.sanitizer-selftest-anchor: &sanitizer-selftest-anchor + stage: test + needs: ["build-codec-sanitizers-linux"] + script: + - *print-common-info + - *copy-ltv-files-to-testv-dir + - make clean + - make -j CLANG=$CLANG_NUM + - testcase_timeout=$SELFTEST_SANITY_TIMEOUT + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + + # --------------------------------------------------------------- # Job templates # --------------------------------------------------------------- @@ -255,6 +271,37 @@ stages: - python3 scripts/prepare_combined_format_inputs.py - cp -r scripts/testv/* $TESTV_DIR/ +.sanitizer-selftest-on-mr: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "Sanitizer selftest results" + reports: + junit: + - report-junit.xml + +.sanitizer-selftest-ltv: + extends: + - .test-job-linux-needs-testv-dir + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 2 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "Sanitizer selftest results" + reports: + junit: + - report-junit.xml + # --------------------------------------------------------------- # .pre jobs for setting up things # --------------------------------------------------------------- @@ -449,83 +496,32 @@ codec-smoke-test: # code selftest testvectors with memory-sanitizer binaries codec-msan: extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request - stage: test - needs: ["build-codec-sanitizers-linux"] - script: - - *print-common-info - - make clean - - make -j CLANG=1 - - testcase_timeout=180 - - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "msan selftest results" - reports: - junit: - - report-junit.xml + - .sanitizer-selftest-on-mr + before_script: + - CLANG_NUM=1 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS + - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py" + <<: *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries codec-asan: extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request - stage: test - needs: ["build-codec-sanitizers-linux"] - script: - - *print-common-info - - make clean - - make -j CLANG=2 - - testcase_timeout=180 - - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "asan selftest results" - reports: - junit: - - report-junit.xml + - .sanitizer-selftest-on-mr + before_script: + - CLANG_NUM=2 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS + - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py" + <<: *sanitizer-selftest-anchor -# code selftest testvectors with address-sanitizer binaries +# code selftest testvectors with undefined-behaviour-sanitizer binaries codec-usan: extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request - stage: test - needs: ["build-codec-sanitizers-linux"] - script: - - *print-common-info - - make clean - - make -j CLANG=3 - - testcase_timeout=180 - - exit_code=0 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$? - - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang undefined-behavior-sanitizer"; exit 1; fi - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "usan selftest results" - reports: - junit: - - report-junit.xml + - .sanitizer-selftest-on-mr + before_script: + - CLANG_NUM=3 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS + - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py" + <<: *sanitizer-selftest-anchor # test renderer executable renderer-smoke-test: @@ -1374,6 +1370,56 @@ test-long-self-test: # --------------------------------------------------------------- # Scheduled jobs on main # --------------------------------------------------------------- + +# code selftest long testvectors with memory-sanitizer binaries +ltv-msan: + extends: + - .sanitizer-selftest-ltv + rules: + - if: $SANITIZER_SCHEDULE_E + timeout: 1 hour + tags: + - ivas-linux-fast + before_script: + - CLANG_NUM=1 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py" + <<: *sanitizer-selftest-anchor + +# code selftest long testvectors with address-sanitizer binaries +ltv-asan: + extends: + - .sanitizer-selftest-ltv + rules: + - if: $SANITIZER_SCHEDULE_E + when: delayed + start_in: 1 hours + tags: + - ivas-linux-fast + timeout: 1 hour + before_script: + - CLANG_NUM=2 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py" + <<: *sanitizer-selftest-anchor + +# code selftest long testvectors with undefined-behaviour-sanitizer binaries +ltv-usan: + extends: + - .sanitizer-selftest-ltv + rules: + - if: $SANITIZER_SCHEDULE_E + when: delayed + start_in: 2 hours + tags: + - ivas-linux-fast + timeout: 1 hour + before_script: + - CLANG_NUM=3 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py" + <<: *sanitizer-selftest-anchor + .sanitizer-test-template: extends: - .test-job-linux