diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4d954bfc386f4376d73e0dbaefde286e12f0d32..b6d1f6de4e5aea20725b6d1727d265fb2f044e5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ variables: TESTV_DIR: "/usr/local/testv" LTV_DIR: "/usr/local/ltv" + EVS_BE_TEST_DIR_FLOAT: "/usr/local/be_2_evs_test" EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" REFERENCE_BRANCH: "ivas-float-update" BUILD_OUTPUT: "build_output.txt" @@ -146,6 +147,10 @@ stages: .mr-get-target-commit: &mr-get-target-commit # compare to last target branch commit before pipeline was created - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) +# from float +.check-for-testvectors: &check-for-testvectors # check if the testvector files specified in scripts/config/ci_linux*.json are present + - python3 -m pytest ci/test_vectors_available.py + # From float CI .merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons @@ -241,6 +246,10 @@ stages: - git pull - cd - +.disable-limiter: &disable-limiter +# automatically enable #define DISABLE_LIMITER in options.h, handling both /**/-comment and //-comment + - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*DISABLE_LIMITER\)[[:space:]]*\*\//\1/g" ./lib_com/options.h + .get-commits-behind-count: &get-commits-behind-count - echo $CI_COMMIT_SHA - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME @@ -266,6 +275,37 @@ stages: - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } +# From float repo +# to be reused in MR and LTV-scheduled sanitizer test jobs +# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section +.sanitizer-selftest-anchor: &sanitizer-selftest-anchor + script: + - *print-common-info + - *update-scripts-repo + - *copy-ltv-files-to-testv-dir + - make clean + - make -j CLANG=$CLANG_NUM + - testcase_timeout=$SELFTEST_SANITY_TIMEOUT + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + + - exit_code20=0 + - exit_code10=0 + - exit_code5=0 + + - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi + .rules-pytest-to-ref-short: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" @@ -364,6 +404,13 @@ stages: - if: $CI_PIPELINE_SOURCE == 'push' when: never +.rules-merge-request-to-float-pc: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "float-pc" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + # templates to define stages and platforms .test-job-linux: tags: @@ -392,6 +439,28 @@ stages: - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ +.sanitizer-selftest-on-mr: + stage: test + extends: + - .rules-merge-request-to-float-pc + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + - report-20ms.html + - report-10ms.html + - report-5ms.html + expose_as: "Sanitizer selftest results" + reports: + junit: + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + .ivas-pytest-anchor: &ivas-pytest-anchor stage: test needs: ["build-codec-linux-make"] @@ -810,6 +879,76 @@ clang-format-check: name: "$ARTIFACT_BASE_NAME" expose_as: "formatting patch" +# from float +# check for crashes if first received frame on decoder side is an SID +check-first-frame-is-sid: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-float-pc + tags: + - ivas-linux + stage: test + needs: ["build-codec-linux-make"] + script: + - *print-common-info + - *update-scripts-repo + - *update-ltv-repo + # this rm makes check-for-testvectors only check for the signals we actually need in this test + - rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json + - *check-for-testvectors + - bash ci/run-first-frame-is-sid-test.sh + +# from float +check-bitexactness-hrtf-rom-and-file: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + stage: test + needs: ["build-codec-linux-make"] + timeout: "5 minutes" + script: + - *print-common-info + - *update-scripts-repo + - make clean + - make -j + - python3 tests/create_short_testvectors.py --cut_len 1.0 + # TODO: run full test again once the custom binary files are supported + - python3 -m pytest -k "not diff_from_rom and not test_binary_file" tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html + artifacts: + paths: + - report.html + - report-junit.xml + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading" + expose_as: "logs-hrtf-loading" + expire_in: "5 days" + +# from float +# TODO: this needs setup/addition of tag on runners +.be-2-evs-linux: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + tags: + - be-2-evs-float + stage: test + needs: ["build-codec-linux-make"] + timeout: "20 minutes" # To be revisited + script: + - *print-common-info + - *update-scripts-repo + + - make clean + - make -j + + # copy over to never change the testvector dir + - cp -r $EVS_BE_TEST_DIR_FLOAT ./evs_be_test + - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod + - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec + + - cd evs_be_test + - python3 ../ci/run_evs_be_test.py + # --------------------------------------------------------------- # Build jobs # --------------------------------------------------------------- @@ -847,7 +986,6 @@ build-codec-linux-instrumented-make: tags: - ivas-linux script: - - *update-scripts-repo - *print-common-info - *update-scripts-repo - bash scripts/prepare_instrumentation.sh -m MEM_ONLY -p BASOP @@ -1273,7 +1411,7 @@ be-2-evs-26444: - .test-job-linux rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "evs-26444" - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' (&& $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main-pc" ) tags: - be-2-evs-basop stage: test @@ -1281,7 +1419,9 @@ be-2-evs-26444: script: - *print-common-info - *update-scripts-repo + # needed on BASOP branches only. On ivas-flaot-update + derivates this does nothing as the replaced string does not exist in options.h - sed -i".bak" "s/\(#define EVS_FLOAT\)/\/\/\1/" lib_com/options.h + - make clean - make -j # copy over to never change the testvector dir @@ -1326,8 +1466,10 @@ voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir rules: - # - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "voip-be-test" + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "float-pc" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never stage: test needs: ["build-codec-linux-make"] timeout: "10 minutes" @@ -1375,6 +1517,221 @@ voip-be-on-merge-request: - smoke_test_output_hrtf.txt expose_as: "Smoke test results" +# from float repo +# NOTE: timeout was adjusted to work for float only +# code selftest testvectors with memory-sanitizer binaries +.codec-msan: + extends: + - .sanitizer-selftest-on-mr + tags: + - ivas-linux + before_script: + - CLANG_NUM=1 + - SELFTEST_SANITY_TIMEOUT=180 + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + <<: *sanitizer-selftest-anchor + +# code selftest testvectors with address-sanitizer binaries +.codec-asan: + extends: + - .sanitizer-selftest-on-mr + tags: + - ivas-linux + before_script: + - CLANG_NUM=2 + - SELFTEST_SANITY_TIMEOUT=180 + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + <<: *sanitizer-selftest-anchor + +# code selftest testvectors with undefined-behaviour-sanitizer binaries +.codec-usan: + extends: + - .sanitizer-selftest-on-mr + tags: + - ivas-linux + before_script: + - CLANG_NUM=3 + - SELFTEST_SANITY_TIMEOUT=180 + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + <<: *sanitizer-selftest-anchor + +# from float +# compare bit-exactness between 5ms and 20 on the branch +# TODO: activate again once all the renderer framesize BE fixes are ported +.pytest-compare-20ms-and-5ms-rendering: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + stage: test + needs: ["build-codec-linux-make", "build-codec-linux-instrumented-make", "build-codec-sanitizers-linux"] + script: + - *print-common-info + - *update-scripts-repo + - *disable-limiter + - make clean + - make -j + ### prepare pytest + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + # create references + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + ### run pytest + - exit_code=0 + - exit_code5=0 + - exit_code10=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true + - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true + - zero_errors=1 + - if [ $zero_errors5 != 1 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors10 != 1 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $exit_code5 -ne 0 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi + - if [ $exit_code10 -ne 0 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi + - if [ $exit_code -ne 0 ]; then exit $EXIT_CODE_FAIL; fi + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + expose_as: "pytest 5ms and 10ms vs 20ms results" + paths: + - report-junit-5ms.xml + - report-5ms.html + - report-junit-10ms.xml + - report-10ms.html + reports: + junit: + - report-junit-5ms.xml + - report-junit-10ms.xml + +# from float +# test renderer executable +renderer-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + needs: ["build-codec-linux-make"] + stage: test + script: + - *print-common-info + - *update-scripts-repo + - make -j IVAS_rend + - testcase_timeout=60 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer make pytest results" + reports: + junit: + - report-junit.xml + +# from float +# test renderer executable with cmake + asan +renderer-asan: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + needs: ["build-codec-linux-make"] + tags: + - ivas-linux + stage: test + script: + # was there from float repo. commented out until cmake is there + # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + # - cmake --build cmake-build -- -j + - *update-scripts-repo + - make clean + - make -j + - testcase_timeout=180 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer asan pytest results" + reports: + junit: + - report-junit.xml + +# from float +# test renderer executable with cmake + msan +renderer-msan: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + needs: ["build-codec-linux-make"] + stage: test + tags: + - ivas-linux + script: + # was there from float repo. commented out until cmake is there + # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + # - cmake --build cmake-build -- -j + - *update-scripts-repo + - make clean + - make -j + - testcase_timeout=180 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer msan pytest results" + reports: + junit: + - report-junit.xml + +# from float +# test renderer executable with cmake + usan +renderer-usan: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + needs: ["build-codec-linux-make"] + stage: test + tags: + - ivas-linux + script: + # was there from float repo. commented out until cmake is there + # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + # - cmake --build cmake-build -- -j + - *update-scripts-repo + - make clean + - make -j + - testcase_timeout=180 + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - grep_exit_code=0 + - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest + - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? + - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer.py with Clang undefined-behavior-sanitizer"; exit 1; fi + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer usan pytest results" + reports: + junit: + - report-junit.xml + # compare renderer bitexactness between target and source branch renderer-pytest-on-merge-request: extends: @@ -1478,6 +1835,59 @@ ivas-pytest-on-merge-request: junit: - report-junit.xml +### From Float +# Check interop IVAS_cod_test -> IVAS_dec_ref +ivas-interop-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-float-pc + stage: test + needs: ["build-codec-linux-make"] + timeout: "10 minutes" + script: + - *print-common-info + - *get-commits-behind-count + - *check-commits-behind-count-in-compare-jobs + - *merge-request-comparison-setup-codec + # the next line is dependent on ref-using-main flag in the other tests, but here the flag does not make sense + - git checkout $source_branch_commit_sha + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_interop_flag=$(grep -c --ignore-case "\[non[ -]*io\]" tmp.txt) || true + + ### prepare pytest + + # Run reference creation, using source branch encoder and main decoder (see .merge-request-comparison-setup-codec) + - exit_code=0 + - exit_code2=0 + # set timeout for individual testcase runs to 60 seconds + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true + + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "interop test results" + reports: + junit: + - report*-junit.xml + # --------------------------------------------------------------- # Complexity measurement jobs # --------------------------------------------------------------- diff --git a/lib_com/options.h b/lib_com/options.h index ee78db378b7511b2b592122744273619549e2457..1691f9deae7921509c60fea07f57c91e32ed627e 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -139,6 +139,8 @@ #endif /* DEBUGGING */ +/*#define DISABLE_LIMITER*/ /* disable the limiter - needed for testing Bitexactness between different renderer framings */ + /* #################### End DEBUGGING switches ############################ */ /* ################### Start FIXES switches ########################### */ diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 31c8e326c86b92c8787f4d166c7db3a97016b5ec..1253423d4d8acec208453c524fd8d3e3afb0a6d9 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -336,6 +336,7 @@ static void accumulate2dArrayToBuffer( * In-place saturation control for multichannel buffers with adaptive release time *-------------------------------------------------------------------*/ +#ifndef DISABLE_LIMITER /*! r: number of clipped output samples */ static int32_t limitRendererOutput( IVAS_LIMITER_HANDLE hLimiter, /* i/o: limiter struct handle */ @@ -380,6 +381,7 @@ static int32_t limitRendererOutput( return numClipping; } +#endif /*-------------------------------------------------------------------*