From 3ac21be59624e84dc9c01657847f839c2a3aa724 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 10 Sep 2024 08:26:13 +0200 Subject: [PATCH 1/2] Added SBA LTV to test-long-self-test and ltv sanitizer tests --- .gitlab-ci.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a57b67497a..8bc0f5c163 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -228,12 +228,12 @@ stages: - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi @@ -570,6 +570,7 @@ codec-msan: - CLANG_NUM=1 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - $USE_LTV="" <<: *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries @@ -580,6 +581,7 @@ codec-asan: - CLANG_NUM=2 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - $USE_LTV="" <<: *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries @@ -592,6 +594,7 @@ codec-usan: - CLANG_NUM=3 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - $USE_LTV="" <<: *sanitizer-selftest-anchor # compare bit-exactness between 5ms and 20 on the branch @@ -1612,11 +1615,11 @@ test-long-self-test: # create references - exit_code_ref=0 - testcase_timeout=300 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? ### run pytest self-test using long test vectors - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi @@ -1654,6 +1657,7 @@ ltv-msan: - CLANG_NUM=1 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" + - $USE_LTV="--use_ltv" <<: *sanitizer-selftest-anchor # code selftest long testvectors with address-sanitizer binaries @@ -1671,6 +1675,7 @@ ltv-asan: - CLANG_NUM=2 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" + - $USE_LTV="--use_ltv" <<: *sanitizer-selftest-anchor # code selftest long testvectors with undefined-behaviour-sanitizer binaries @@ -1688,6 +1693,7 @@ ltv-usan: - CLANG_NUM=3 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" + - $USE_LTV="--use_ltv" <<: *sanitizer-selftest-anchor .sanitizer-test-template: -- GitLab From d22bcb1522d826497bba553020a6fcfe5f44d1e3 Mon Sep 17 00:00:00 2001 From: norvell Date: Tue, 10 Sep 2024 11:22:50 +0000 Subject: [PATCH 2/2] Correct syntax error for $USE_LTV="--use_ltv" --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8bc0f5c163..6e5c88431c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -570,7 +570,7 @@ codec-msan: - CLANG_NUM=1 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - - $USE_LTV="" + - USE_LTV="" <<: *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries @@ -581,7 +581,7 @@ codec-asan: - CLANG_NUM=2 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - - $USE_LTV="" + - USE_LTV="" <<: *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries @@ -594,7 +594,7 @@ codec-usan: - CLANG_NUM=3 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - - $USE_LTV="" + - USE_LTV="" <<: *sanitizer-selftest-anchor # compare bit-exactness between 5ms and 20 on the branch @@ -1657,7 +1657,7 @@ ltv-msan: - CLANG_NUM=1 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" - - $USE_LTV="--use_ltv" + - USE_LTV="--use_ltv" <<: *sanitizer-selftest-anchor # code selftest long testvectors with address-sanitizer binaries @@ -1675,7 +1675,7 @@ ltv-asan: - CLANG_NUM=2 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" - - $USE_LTV="--use_ltv" + - USE_LTV="--use_ltv" <<: *sanitizer-selftest-anchor # code selftest long testvectors with undefined-behaviour-sanitizer binaries @@ -1693,7 +1693,7 @@ ltv-usan: - CLANG_NUM=3 - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" - - $USE_LTV="--use_ltv" + - USE_LTV="--use_ltv" <<: *sanitizer-selftest-anchor .sanitizer-test-template: -- GitLab