diff --git a/main-basop.yml b/main-basop.yml index 142c30b6c4b4fb519afd9d361bd33ddadf4af7c8..0f590d31bc7a99337d7d96e37bf23d2c33447232 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -275,6 +275,15 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' when: never +.rules-pytest-sanitizers-on-schedule: + rules: + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_SANITIZERS + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" + # --------------------------------------------------------------- # Job templates # --------------------------------------------------------------- @@ -1492,95 +1501,78 @@ renderer-smoke-test: junit: - report-junit.xml -# from float -# TODO: turn into scheduled job -.renderer-asan: +ivas-pytest-sanitizers: extends: - .test-job-linux - needs: ["build-codec-linux-clang-cmake"] + - .rules-pytest-sanitizers-on-schedule + needs: ["build-codec-sanitizers-clang-linux"] stage: test - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - make clean - - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - - testcase_timeout=180 - - python3 -m pytest -q -n auto --tb=no --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout - + timeout: "32 hours" + resource_group: basop-long-sanitizers + parallel: + matrix: + - SANITIZER: + - MSAN + - ASAN + - USAN + WHAT: + - CODEC + - RENDERER + TESTSIGNALS: + - SHORT + - LONG artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 2 week when: always paths: - report-junit.xml + - report.html + - errors.csv - $MAKE_BUILD_LOG_FILE_DEFAULT - expose_as: "renderer asan pytest results" reports: junit: - report-junit.xml - -# from float -# TODO: turn into scheduled job -.renderer-msan: - extends: - - .test-job-linux - needs: ["build-codec-linux-clang-cmake"] - stage: test - tags: - - ivas-linux script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - make clean - - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - - testcase_timeout=180 - - python3 -m pytest -q -n auto --tb=no --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout - - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - $MAKE_BUILD_LOG_FILE_DEFAULT - expose_as: "renderer msan pytest results" - reports: - junit: - - report-junit.xml + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh -# from float -# TODO: turn into scheduled job -# NOTE: disabled for now until existing bug is fixed -.renderer-usan: - extends: - - .test-job-linux - needs: ["build-codec-linux-clang-cmake"] - stage: test - tags: - - ivas-linux - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - python3 scripts/basop_create_ignorelist_for_ubsan.py - - make -j CLANG=3 IGNORELIST=1 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - - testcase_timeout=180 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest -q -n auto --tb=no --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout - - grep_exit_code=0 - - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest - - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? - - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer.py with Clang undefined-behavior-sanitizer"; exit 1; fi - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - $MAKE_BUILD_LOG_FILE_DEFAULT - expose_as: "renderer usan pytest results" - reports: - junit: - - report-junit.xml + - set -euxo pipefail + - declare -A sanitizer_to_clangnum + - sanitizer_to_clangnum["MSAN"]=1 + - sanitizer_to_clangnum["ASAN"]=2 + - sanitizer_to_clangnum["USAN"]=3 + + # build everything with the right sanitizer + - make clean + - make_args="-j CLANG=${sanitizer_to_clangnum["$SANITIZER"]}" + - | + if [ $SANITIZER == "USAN" ]; then + make_args="$make_args IGNORELIST=1" + fi + - make -j $make_args 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT + + # assemble arguments and run pytest + - pytest_args="-q -n auto --tb=no --log-level=INFO --html=report.html --self-contained-html --junit-xml=report-junit.xml" + - | + if [ "$TESTSIGNALS" == "LONG" ]; then + pytest_args="$pytest_args --param_file scripts/config/self_test_ltv.prm --use_ltv" + fi + - | + if [ "$WHAT" == "CODEC" ]; then + pytest_args="tests/codec_be_on_mr_nonselection $pytest_args --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec" + else # RENDERER + pytest_args="tests/renderer/test_renderer.py $pytest_args --create_ref" + # renderer tests have this naming convention and no way to point to specific binaries + mv IVAS_rend IVAS_rend_ref + fi + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest $pytest_args + + after_script: + - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml errors.csv .sanitizer-selftest-on-mr: stage: test @@ -2019,148 +2011,6 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: - SPLIT_COMPARISON="true" <<: *compare-to-ref-anchor -### These next 3 jobs run three different sanitizers for fx -> fx codec chain usin - -.ivas-pytest-sanitizers-anchor: - extends: - - .job-linux - stage: test - needs: ["build-codec-linux-clang-make"] - timeout: "6 hours" - parallel: - matrix: - - TESTSIGNALS: - - long - - short - before_script: - - case $TESTSIGNALS in - "long") - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER - ;; - "short") - TEST_SUITE=$SHORT_TEST_SUITE - ;; - esac - script: - - !reference [ .test-job-linux, before_script ] - - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - - set -euxo pipefail - - make_args="CLANG=$CLANG_NUM" - - if [[ $CLANG_NUM == 3 ]]; then - - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" - - python3 scripts/basop_create_ignorelist_for_ubsan.py - - make_args="$make_args IGNORELIST=1" - - fi - - make clean - - make -j $make_args 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - - testcase_timeout_arg="--testcase_timeout $TESTCASE_TIMEOUT_LTV_SANITIZERS" - # disable per-testcase timeout for msan to evaluate what is going on that it takes so long - - if [[ $CLANG_NUM = 1 ]]; then - - testcase_timeout_arg="" - - fi - - # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh) - - python3 -m pytest $TEST_SUITE --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec - -ivas-pytest-msan: - extends: - - .ivas-pytest-sanitizers-anchor - tags: - - ivas-linux-fast - resource_group: basop-long-sanitizers - rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" - timeout: "24 hours" - before_script: - - !reference [ .ivas-pytest-sanitizers-anchor, before_script ] - - CLANG_NUM=1 - after_script: - - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml msan-errors.csv - artifacts: - name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" - expire_in: 2 weeks - when: always - paths: - - report-junit.xml - - report.html - - msan-errors.csv - - $MAKE_BUILD_LOG_FILE_DEFAULT - expose_as: "Sanitizer selftest results" - reports: - junit: - - report-junit.xml - -ivas-pytest-asan: - extends: - - .ivas-pytest-sanitizers-anchor - tags: - - ivas-linux-fast - resource_group: basop-long-sanitizers - rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" - before_script: - - !reference [ .ivas-pytest-sanitizers-anchor, before_script ] - - CLANG_NUM=2 - after_script: - - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml asan-errors.csv - artifacts: - name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" - expire_in: 2 weeks - when: always - paths: - - report-junit.xml - - report.html - - asan-errors.csv - - $MAKE_BUILD_LOG_FILE_DEFAULT - expose_as: "Sanitizer selftest results" - reports: - junit: - - report-junit.xml - -ivas-pytest-usan: - extends: - - .ivas-pytest-sanitizers-anchor - tags: - - ivas-linux-fast - resource_group: basop-long-sanitizers - rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" - before_script: - - !reference [ .ivas-pytest-sanitizers-anchor, before_script ] - - CLANG_NUM=3 - after_script: - - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml usan-errors.csv - artifacts: - name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" - expire_in: 2 weeks - when: always - paths: - - report-junit.xml - - report.html - - usan-errors.csv - - $MAKE_BUILD_LOG_FILE_DEFAULT - expose_as: "Sanitizer selftest results" - reports: - junit: - - report-junit.xml # Manual long self-test (not including tests_sba.py) against BASOP main test-long-self-test: