Loading includes/default-variables.yml +1 −1 Original line number Diff line number Diff line Loading @@ -11,7 +11,7 @@ variables: TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" BASOP_CI_BRANCH_PC_REPO: "ci/add-inplace-option-to-create-short-testvectory-script" PYTEST_ADDOPTS: "" # overwrite this via .gitlab/variables.yml in the respective other repo to disable "(model from file)" testcases DISABLE_HRTF: "false" main-basop.yml +70 −84 Original line number Diff line number Diff line Loading @@ -1718,8 +1718,10 @@ renderer-usan: .sanitizer-selftest-on-mr: stage: test needs: ["build-codec-sanitizers-linux"] extends: - .rules-merge-request-to-float-ref-no-draft - .rules-merge-request-no-draft - .test-job-linux artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week Loading @@ -1742,73 +1744,91 @@ renderer-usan: # to be reused in MR and LTV-scheduled sanitizer test jobs # set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section .sanitizer-selftest-anchor: &sanitizer-selftest-anchor script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - make clean - make -j CLANG=$CLANG_NUM - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 - python3 ci/remove_unsupported_testcases.py $PRM_FILES - make_args="CLANG=$CLANG_NUM" # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) # shorten files to 1s for that case - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then - python3 tests/create_short_testvectors.py --cut_len 1.0 --inplace --all # also need different setup for ubsan - if [[ $CLANG_NUM == 3 ]]; then - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" - python3 scripts/basop_create_ignorelist_for_ubsan.py - make_args="$make_args IGNORELIST=1" - fi - fi - exit_code20=0 - exit_code10=0 - exit_code5=0 - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi - make clean - make -j $make_args # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? # check different renderer fremsizes only for float - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - fi - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi # from float repo # NOTE: (per-testcase) timeout was adjusted to work for float only # code selftest testvectors with memory-sanitizer binaries .codec-msan: codec-msan: extends: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux before_script: script: - CLANG_NUM=1 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" <<: *sanitizer-selftest-anchor - *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries .codec-asan: codec-asan: extends: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux before_script: script: - CLANG_NUM=2 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" <<: *sanitizer-selftest-anchor - *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries .codec-usan: codec-usan: extends: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux before_script: script: - CLANG_NUM=3 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" <<: *sanitizer-selftest-anchor - *sanitizer-selftest-anchor # --------------------------------------------------------------- # Short test jobs for running from web interface or schedule Loading Loading @@ -1891,39 +1911,6 @@ ivas-pytest-compare_to_ref-dmx-short-enc-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor ivas-pytest-enc-msan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=1 - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_ENCODER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-enc-asan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=2 - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_ENCODER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-enc-usan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=3 - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_ENCODER <<: *ivas-pytest-sanitizers-anchor ### jobs that test flt encoder -> fx decoder ivas-pytest-compare_to_ref-short-dec: extends: Loading Loading @@ -1981,39 +1968,6 @@ ivas-pytest-compare-to-input-short-enc: - LEVEL_SCALING=1.0 <<: *ivas-pytest-compare-to-input-anchor ivas-pytest-dec-msan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=1 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-asan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=2 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-usan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=3 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor # --------------------------------------------------------------- # Long test job # --------------------------------------------------------------- Loading Loading @@ -2141,6 +2095,38 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ### These next 3 jobs run three different sanitizers for fx -> fx codec chain usin ivas-pytest-msan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=1 - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-asan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=2 - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-usan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=3 - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor # Manual long self-test (not including tests_sba.py) against BASOP main test-long-self-test: extends: Loading Loading
includes/default-variables.yml +1 −1 Original line number Diff line number Diff line Loading @@ -11,7 +11,7 @@ variables: TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" BASOP_CI_BRANCH_PC_REPO: "ci/add-inplace-option-to-create-short-testvectory-script" PYTEST_ADDOPTS: "" # overwrite this via .gitlab/variables.yml in the respective other repo to disable "(model from file)" testcases DISABLE_HRTF: "false"
main-basop.yml +70 −84 Original line number Diff line number Diff line Loading @@ -1718,8 +1718,10 @@ renderer-usan: .sanitizer-selftest-on-mr: stage: test needs: ["build-codec-sanitizers-linux"] extends: - .rules-merge-request-to-float-ref-no-draft - .rules-merge-request-no-draft - .test-job-linux artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week Loading @@ -1742,73 +1744,91 @@ renderer-usan: # to be reused in MR and LTV-scheduled sanitizer test jobs # set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section .sanitizer-selftest-anchor: &sanitizer-selftest-anchor script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - make clean - make -j CLANG=$CLANG_NUM - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 - python3 ci/remove_unsupported_testcases.py $PRM_FILES - make_args="CLANG=$CLANG_NUM" # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) # shorten files to 1s for that case - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then - python3 tests/create_short_testvectors.py --cut_len 1.0 --inplace --all # also need different setup for ubsan - if [[ $CLANG_NUM == 3 ]]; then - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" - python3 scripts/basop_create_ignorelist_for_ubsan.py - make_args="$make_args IGNORELIST=1" - fi - fi - exit_code20=0 - exit_code10=0 - exit_code5=0 - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi - make clean - make -j $make_args # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? # check different renderer fremsizes only for float - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - fi - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi # from float repo # NOTE: (per-testcase) timeout was adjusted to work for float only # code selftest testvectors with memory-sanitizer binaries .codec-msan: codec-msan: extends: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux before_script: script: - CLANG_NUM=1 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" <<: *sanitizer-selftest-anchor - *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries .codec-asan: codec-asan: extends: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux before_script: script: - CLANG_NUM=2 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" <<: *sanitizer-selftest-anchor - *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries .codec-usan: codec-usan: extends: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux before_script: script: - CLANG_NUM=3 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" <<: *sanitizer-selftest-anchor - *sanitizer-selftest-anchor # --------------------------------------------------------------- # Short test jobs for running from web interface or schedule Loading Loading @@ -1891,39 +1911,6 @@ ivas-pytest-compare_to_ref-dmx-short-enc-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor ivas-pytest-enc-msan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=1 - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_ENCODER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-enc-asan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=2 - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_ENCODER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-enc-usan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=3 - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_ENCODER <<: *ivas-pytest-sanitizers-anchor ### jobs that test flt encoder -> fx decoder ivas-pytest-compare_to_ref-short-dec: extends: Loading Loading @@ -1981,39 +1968,6 @@ ivas-pytest-compare-to-input-short-enc: - LEVEL_SCALING=1.0 <<: *ivas-pytest-compare-to-input-anchor ivas-pytest-dec-msan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=1 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-asan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=2 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-usan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=3 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor # --------------------------------------------------------------- # Long test job # --------------------------------------------------------------- Loading Loading @@ -2141,6 +2095,38 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ### These next 3 jobs run three different sanitizers for fx -> fx codec chain usin ivas-pytest-msan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=1 - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-asan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=2 - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-usan: extends: - .test-job-linux tags: - ivas-basop-linux-fast before_script: - CLANG_NUM=3 - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor # Manual long self-test (not including tests_sba.py) against BASOP main test-long-self-test: extends: Loading
