From aa99c334fe8ed9598eba1987352a14013894f699 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 12:56:31 +0200 Subject: [PATCH 01/12] add sanitizer tests to MR pipelines in BASOP repo - float ref MRs use the stv version from float repo - main MRs shorten files to 1s --- main-basop.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/main-basop.yml b/main-basop.yml index cc48076..034044f 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1709,7 +1709,7 @@ renderer-usan: .sanitizer-selftest-on-mr: stage: test extends: - - .rules-merge-request-to-float-ref-no-draft + - .rules-merge-request-no-draft artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week @@ -1743,6 +1743,12 @@ renderer-usan: - python3 ci/remove_unsupported_testcases.py $PRM_FILES + # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) + # shorten files to 1s for that case + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ] then + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - fi + - exit_code20=0 - exit_code10=0 - exit_code5=0 -- GitLab From acc3b70ceea86623a99317ca2e58bcd0a5e4cdd5 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 13:04:19 +0200 Subject: [PATCH 02/12] activate sanitizer jobs again --- main-basop.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 034044f..3f2b691 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1768,7 +1768,7 @@ renderer-usan: # from float repo # NOTE: (per-testcase) timeout was adjusted to work for float only # code selftest testvectors with memory-sanitizer binaries -.codec-msan: +codec-msan: extends: - .sanitizer-selftest-on-mr tags: @@ -1781,7 +1781,7 @@ renderer-usan: <<: *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries -.codec-asan: +codec-asan: extends: - .sanitizer-selftest-on-mr tags: @@ -1794,7 +1794,7 @@ renderer-usan: <<: *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries -.codec-usan: +codec-usan: extends: - .sanitizer-selftest-on-mr tags: -- GitLab From 8415f062ffe0eb7005e74e67c3df3be9d66641b9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 13:06:59 +0200 Subject: [PATCH 03/12] add correct needs for sanitizer jobs --- main-basop.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/main-basop.yml b/main-basop.yml index 3f2b691..c693e41 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1708,6 +1708,7 @@ renderer-usan: .sanitizer-selftest-on-mr: stage: test + needs: ["build-codec-sanitizers-linux"] extends: - .rules-merge-request-no-draft artifacts: -- GitLab From e951d186d634e2d6a85cab061f1b4ff2daa98746 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 13:16:48 +0200 Subject: [PATCH 04/12] add missing dependency to get scripts repo --- main-basop.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/main-basop.yml b/main-basop.yml index c693e41..0f76a59 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1711,6 +1711,7 @@ renderer-usan: needs: ["build-codec-sanitizers-linux"] extends: - .rules-merge-request-no-draft + - .test-job-linux artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week -- GitLab From 3aba0bb96dc891919bc2952c4732e0c5cddc00fc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 13:28:45 +0200 Subject: [PATCH 05/12] change use of anchor to avoid before_script before_script section in sanitizer job template overwrites the one from test-job-linux and then the script repo does not get cloned --- main-basop.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 0f76a59..0393cf7 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1734,7 +1734,6 @@ renderer-usan: # to be reused in MR and LTV-scheduled sanitizer test jobs # set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section .sanitizer-selftest-anchor: &sanitizer-selftest-anchor - script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh @@ -1775,12 +1774,12 @@ codec-msan: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux - before_script: + script: - CLANG_NUM=1 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" - <<: *sanitizer-selftest-anchor + - *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries codec-asan: @@ -1788,12 +1787,12 @@ codec-asan: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux - before_script: + script: - CLANG_NUM=2 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" - <<: *sanitizer-selftest-anchor + - *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries codec-usan: @@ -1801,12 +1800,12 @@ codec-usan: - .sanitizer-selftest-on-mr tags: - ivas-basop-linux - before_script: + script: - CLANG_NUM=3 - SELFTEST_SANITY_TIMEOUT=180 - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - USE_LTV="" - <<: *sanitizer-selftest-anchor + - *sanitizer-selftest-anchor # --------------------------------------------------------------- # Short test jobs for running from web interface or schedule -- GitLab From 6e8f33139b0e0b4017ac415cf93c6df5787c19a1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 13:49:38 +0200 Subject: [PATCH 06/12] fix typo --- main-basop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main-basop.yml b/main-basop.yml index 0393cf7..8e107d2 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1746,7 +1746,7 @@ renderer-usan: # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) # shorten files to 1s for that case - - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ] then + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then - python3 tests/create_short_testvectors.py --cut_len 1.0 - fi -- GitLab From 99972ea508f4e3ade5bf9efff99377cacc2ef304 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 14:48:15 +0200 Subject: [PATCH 07/12] only run rend framesize !=20 ms for MR to float ref --- main-basop.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 8e107d2..059705e 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1758,11 +1758,15 @@ renderer-usan: # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? - # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option - # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active - # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + + # check different renderer fremsizes only for float + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + - fi - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi -- GitLab From d6a7022303f3722f229acd1fce10df6f83cab9c9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 16:22:19 +0200 Subject: [PATCH 08/12] correct ignorelist for BASOP USAN in the MR sanity pipelines --- main-basop.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 059705e..5c145b4 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1737,8 +1737,6 @@ renderer-usan: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - make clean - - make -j CLANG=$CLANG_NUM - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 @@ -1748,6 +1746,14 @@ renderer-usan: # shorten files to 1s for that case - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then - python3 tests/create_short_testvectors.py --cut_len 1.0 + + - make_args="CLANG=$CLANG_NUM" + # also need different setup for ubsan + - if [[ $CLANG_NUM == 3 ]]; then + - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" + - python3 scripts/basop_create_ignorelist_for_ubsan.py + - make_args="$make_args IGNORELIST=1" + - fi - fi - exit_code20=0 @@ -1756,6 +1762,9 @@ renderer-usan: - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + - make clean + - make -j $make_args + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? -- GitLab From aa84c5cf3646e98b69528fa7f9544f486f1d384a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 16:31:48 +0200 Subject: [PATCH 09/12] move variable init out of if --- main-basop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main-basop.yml b/main-basop.yml index 5c145b4..7858a11 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1742,12 +1742,12 @@ renderer-usan: - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - make_args="CLANG=$CLANG_NUM" # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) # shorten files to 1s for that case - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then - python3 tests/create_short_testvectors.py --cut_len 1.0 - - make_args="CLANG=$CLANG_NUM" # also need different setup for ubsan - if [[ $CLANG_NUM == 3 ]]; then - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" -- GitLab From c93f17fb1557f693cc10ef51e3d10c053817724b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 17:57:09 +0200 Subject: [PATCH 10/12] use --inplace flag for shortening testvectors --- main-basop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main-basop.yml b/main-basop.yml index 7858a11..8e38dd9 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1746,7 +1746,7 @@ renderer-usan: # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) # shorten files to 1s for that case - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then - - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 tests/create_short_testvectors.py --cut_len 1.0 --inplace # also need different setup for ubsan - if [[ $CLANG_NUM == 3 ]]; then -- GitLab From f9ab70916430d7f943936dd771e76f6bf72e066b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 1 Sep 2025 17:57:55 +0200 Subject: [PATCH 11/12] [revert-me] use temporary branch for scripts --- includes/default-variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/default-variables.yml b/includes/default-variables.yml index 3209ec5..f21d2e0 100644 --- a/includes/default-variables.yml +++ b/includes/default-variables.yml @@ -11,4 +11,4 @@ variables: TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" - BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" + BASOP_CI_BRANCH_PC_REPO: "ci/add-inplace-option-to-create-short-testvectory-script" -- GitLab From 79fb02d41dc1fc28b13faf3befdd1d522d07f8a4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 2 Sep 2025 15:26:26 +0200 Subject: [PATCH 12/12] run LTV sanitizer jobs for fx -> fx only (this was actually the case before already, but the naming was not correct) --- main-basop.yml | 97 ++++++++++++++++---------------------------------- 1 file changed, 31 insertions(+), 66 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 8e38dd9..bec9ee5 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1901,39 +1901,6 @@ ivas-pytest-compare_to_ref-dmx-short-enc-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor -ivas-pytest-enc-msan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=1 - - DUT_DECODER_PATH=./$REF_DECODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_ENCODER - <<: *ivas-pytest-sanitizers-anchor - -ivas-pytest-enc-asan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=2 - - DUT_DECODER_PATH=./$REF_DECODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_ENCODER - <<: *ivas-pytest-sanitizers-anchor - -ivas-pytest-enc-usan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=3 - - DUT_DECODER_PATH=./$REF_DECODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_ENCODER - <<: *ivas-pytest-sanitizers-anchor - ### jobs that test flt encoder -> fx decoder ivas-pytest-compare_to_ref-short-dec: extends: @@ -1991,39 +1958,6 @@ ivas-pytest-compare-to-input-short-enc: - LEVEL_SCALING=1.0 <<: *ivas-pytest-compare-to-input-anchor -ivas-pytest-dec-msan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=1 - - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER - <<: *ivas-pytest-sanitizers-anchor - -ivas-pytest-dec-asan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=2 - - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER - <<: *ivas-pytest-sanitizers-anchor - -ivas-pytest-dec-usan: - extends: - - .test-job-linux - tags: - - ivas-basop-linux-fast - before_script: - - CLANG_NUM=3 - - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER - <<: *ivas-pytest-sanitizers-anchor - # --------------------------------------------------------------- # Long test job # --------------------------------------------------------------- @@ -2151,6 +2085,37 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor +### These next 3 jobs run three different sanitizers for fx -> fx codec chain usin + +ivas-pytest-msan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=1 + - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-asan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=2 + - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-usan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=3 + - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + <<: *ivas-pytest-sanitizers-anchor ivas-smoke-test-saturation: extends: -- GitLab