From 01d9063e342fbd7a64567771f10b5b3a3f1b1cc1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 17 Jan 2025 16:42:06 +0100 Subject: [PATCH 01/23] port codec-*san jobs --- .gitlab-ci.yml | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4d954bfc..eba7f078c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -266,6 +266,34 @@ stages: - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } +# From float repo +# to be reused in MR and LTV-scheduled sanitizer test jobs +# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section +.sanitizer-selftest-anchor: &sanitizer-selftest-anchor + script: + - *print-common-info + - *copy-ltv-files-to-testv-dir + - make clean + - make -j CLANG=$CLANG_NUM + - testcase_timeout=$SELFTEST_SANITY_TIMEOUT + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 + + - exit_code20=0 + - exit_code10=0 + - exit_code5=0 + + - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi + .rules-pytest-to-ref-short: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" @@ -364,6 +392,13 @@ stages: - if: $CI_PIPELINE_SOURCE == 'push' when: never +.rules-merge-request-to-float-pc: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "float-pc" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + # templates to define stages and platforms .test-job-linux: tags: @@ -392,6 +427,29 @@ stages: - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ +.sanitizer-selftest-on-mr: + stage: test + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-float-pc + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + - report-20ms.html + - report-10ms.html + - report-5ms.html + expose_as: "Sanitizer selftest results" + reports: + junit: + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + .ivas-pytest-anchor: &ivas-pytest-anchor stage: test needs: ["build-codec-linux-make"] @@ -1375,6 +1433,47 @@ voip-be-on-merge-request: - smoke_test_output_hrtf.txt expose_as: "Smoke test results" +# from float repo +# NOTE: timeout was adjusted to work for float only +# code selftest testvectors with memory-sanitizer binaries +codec-msan: + extends: + - .sanitizer-selftest-on-mr + tags: + - ivas-linux + before_script: + - CLANG_NUM=1 + - SELFTEST_SANITY_TIMEOUT=180 + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + <<: *sanitizer-selftest-anchor + +# code selftest testvectors with address-sanitizer binaries +codec-asan: + extends: + - .sanitizer-selftest-on-mr + tags: + - ivas-linux + before_script: + - CLANG_NUM=2 + - SELFTEST_SANITY_TIMEOUT=180 + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + <<: *sanitizer-selftest-anchor + +# code selftest testvectors with undefined-behaviour-sanitizer binaries +codec-usan: + extends: + - .sanitizer-selftest-on-mr + tags: + - ivas-linux + before_script: + - CLANG_NUM=3 + - SELFTEST_SANITY_TIMEOUT=180 + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + <<: *sanitizer-selftest-anchor + # compare renderer bitexactness between target and source branch renderer-pytest-on-merge-request: extends: -- GitLab From a348723088049232455197bad66e9b69c8b98822 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 17 Jan 2025 16:44:25 +0100 Subject: [PATCH 02/23] add remove_unsupported_testcases.py call --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eba7f078c..9345f6fa3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -278,6 +278,8 @@ stages: - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - exit_code20=0 - exit_code10=0 - exit_code5=0 -- GitLab From 04884579b90e3c8772a3d0420930ea46f025a53d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 17 Jan 2025 16:48:08 +0100 Subject: [PATCH 03/23] fix missing testv pull from basop-ci-branch --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9345f6fa3..abf6faa59 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -272,6 +272,7 @@ stages: .sanitizer-selftest-anchor: &sanitizer-selftest-anchor script: - *print-common-info + - *update-scripts-repo - *copy-ltv-files-to-testv-dir - make clean - make -j CLANG=$CLANG_NUM @@ -432,7 +433,6 @@ stages: .sanitizer-selftest-on-mr: stage: test extends: - - .test-job-linux-needs-testv-dir - .rules-merge-request-to-float-pc artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" -- GitLab From ea9aa541719f0397ff521f64f89e366b251d3753 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 08:35:07 +0100 Subject: [PATCH 04/23] disable codec-*san jobs until update work has caught up --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index abf6faa59..04c4ac579 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1438,7 +1438,7 @@ voip-be-on-merge-request: # from float repo # NOTE: timeout was adjusted to work for float only # code selftest testvectors with memory-sanitizer binaries -codec-msan: +.codec-msan: extends: - .sanitizer-selftest-on-mr tags: @@ -1451,7 +1451,7 @@ codec-msan: <<: *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries -codec-asan: +.codec-asan: extends: - .sanitizer-selftest-on-mr tags: @@ -1464,7 +1464,7 @@ codec-asan: <<: *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries -codec-usan: +.codec-usan: extends: - .sanitizer-selftest-on-mr tags: -- GitLab From add126edb7df0aac9d00060e5349bcf497706ed1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 08:39:40 +0100 Subject: [PATCH 05/23] enable again, but only 20ms --- .gitlab-ci.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 04c4ac579..b64bb2d5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -292,8 +292,10 @@ stages: # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + + # TODO: enable once the update process has caught up + # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi @@ -440,17 +442,17 @@ stages: when: always paths: - report-junit-20ms.xml - - report-junit-10ms.xml - - report-junit-5ms.xml + # - report-junit-10ms.xml + # - report-junit-5ms.xml - report-20ms.html - - report-10ms.html - - report-5ms.html + # - report-10ms.html + # - report-5ms.html expose_as: "Sanitizer selftest results" reports: junit: - report-junit-20ms.xml - - report-junit-10ms.xml - - report-junit-5ms.xml + # - report-junit-10ms.xml + # - report-junit-5ms.xml .ivas-pytest-anchor: &ivas-pytest-anchor stage: test @@ -1438,7 +1440,7 @@ voip-be-on-merge-request: # from float repo # NOTE: timeout was adjusted to work for float only # code selftest testvectors with memory-sanitizer binaries -.codec-msan: +codec-msan: extends: - .sanitizer-selftest-on-mr tags: @@ -1451,7 +1453,7 @@ voip-be-on-merge-request: <<: *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries -.codec-asan: +codec-asan: extends: - .sanitizer-selftest-on-mr tags: @@ -1464,7 +1466,7 @@ voip-be-on-merge-request: <<: *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries -.codec-usan: +codec-usan: extends: - .sanitizer-selftest-on-mr tags: -- GitLab From e9d78b46cbc935bbdbd0ff94cd62f0317e170cce Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 09:00:39 +0100 Subject: [PATCH 06/23] Revert "enable again, but only 20ms" This reverts commit add126edb7df0aac9d00060e5349bcf497706ed1. --- .gitlab-ci.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b64bb2d5b..04c4ac579 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -292,10 +292,8 @@ stages: # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs - - # TODO: enable once the update process has caught up - # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi @@ -442,17 +440,17 @@ stages: when: always paths: - report-junit-20ms.xml - # - report-junit-10ms.xml - # - report-junit-5ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml - report-20ms.html - # - report-10ms.html - # - report-5ms.html + - report-10ms.html + - report-5ms.html expose_as: "Sanitizer selftest results" reports: junit: - report-junit-20ms.xml - # - report-junit-10ms.xml - # - report-junit-5ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml .ivas-pytest-anchor: &ivas-pytest-anchor stage: test @@ -1440,7 +1438,7 @@ voip-be-on-merge-request: # from float repo # NOTE: timeout was adjusted to work for float only # code selftest testvectors with memory-sanitizer binaries -codec-msan: +.codec-msan: extends: - .sanitizer-selftest-on-mr tags: @@ -1453,7 +1451,7 @@ codec-msan: <<: *sanitizer-selftest-anchor # code selftest testvectors with address-sanitizer binaries -codec-asan: +.codec-asan: extends: - .sanitizer-selftest-on-mr tags: @@ -1466,7 +1464,7 @@ codec-asan: <<: *sanitizer-selftest-anchor # code selftest testvectors with undefined-behaviour-sanitizer binaries -codec-usan: +.codec-usan: extends: - .sanitizer-selftest-on-mr tags: -- GitLab From efa1f3783562839a9bd913b8f18afc8f2b9bedbb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 09:13:30 +0100 Subject: [PATCH 07/23] add ivas-interop-on-merge-request job to float-pc pipeline --- .gitlab-ci.yml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 04c4ac579..def52201e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1579,6 +1579,59 @@ ivas-pytest-on-merge-request: junit: - report-junit.xml +### From Float +# Check interop IVAS_cod_test -> IVAS_dec_ref +ivas-interop-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-float-pc + stage: test + needs: ["build-codec-linux-make"] + timeout: "10 minutes" + script: + - *print-common-info + - *get-commits-behind-count + - *check-commits-behind-count-in-compare-jobs + - *merge-request-comparison-setup-codec + # the next line is dependent on ref-using-main flag in the other tests, but here the flag does not make sense + - git checkout $source_branch_commit_sha + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_interop_flag=$(grep -c --ignore-case "\[non[ -]*io\]" tmp.txt) || true + + ### prepare pytest + + # Run reference creation, using source branch encoder and main decoder (see .merge-request-comparison-setup-codec) + - exit_code=0 + - exit_code2=0 + # set timeout for individual testcase runs to 60 seconds + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true + + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "interop test results" + reports: + junit: + - report*-junit.xml + # --------------------------------------------------------------- # Complexity measurement jobs # --------------------------------------------------------------- -- GitLab From 163964669ce00b17cf4d499d04f5ee5457c8f8e1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 09:27:07 +0100 Subject: [PATCH 08/23] make voip-be-on-mr job run for float-pc MRs --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index def52201e..febab88da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1386,8 +1386,10 @@ voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir rules: - # - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "voip-be-test" + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "float-pc" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never stage: test needs: ["build-codec-linux-make"] timeout: "10 minutes" -- GitLab From 5730e088916519f2604544bc62fedd2756b71200 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 09:30:38 +0100 Subject: [PATCH 09/23] add hrtf loading BE test --- .gitlab-ci.yml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index febab88da..8910d4921 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -870,6 +870,30 @@ clang-format-check: name: "$ARTIFACT_BASE_NAME" expose_as: "formatting patch" +# from float +check-bitexactness-hrtf-rom-and-file: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + stage: test + needs: ["build-codec-linux-make"] + timeout: "5 minutes" + script: + - *print-common-info + - *update-scripts-repo + - make clean + - make -j + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html + artifacts: + paths: + - report.html + - report-junit.xml + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading" + expose_as: "logs-hrtf-loading" + expire_in: "5 days" + # --------------------------------------------------------------- # Build jobs # --------------------------------------------------------------- @@ -907,7 +931,6 @@ build-codec-linux-instrumented-make: tags: - ivas-linux script: - - *update-scripts-repo - *print-common-info - *update-scripts-repo - bash scripts/prepare_instrumentation.sh -m MEM_ONLY -p BASOP -- GitLab From 714170ea43fcbf1fddfda4d49fab5c2f137dc115 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 09:38:19 +0100 Subject: [PATCH 10/23] add evs be test job for float needs runner setup b4 being usable --- .gitlab-ci.yml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8910d4921..0b0c91d58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ variables: TESTV_DIR: "/usr/local/testv" LTV_DIR: "/usr/local/ltv" + EVS_BE_TEST_DIR_FLOAT: "/usr/local/be_2_evs_test" EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" REFERENCE_BRANCH: "ivas-float-update" BUILD_OUTPUT: "build_output.txt" @@ -894,6 +895,32 @@ check-bitexactness-hrtf-rom-and-file: expose_as: "logs-hrtf-loading" expire_in: "5 days" +# from float +# TODO: this needs setup/addition of tag on runners +.be-2-evs-linux: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + tags: + - be-2-evs-float + stage: test + needs: ["build-codec-linux-make"] + timeout: "20 minutes" # To be revisited + script: + - *print-common-info + - *update-scripts-repo + + - make clean + - make -j + + # copy over to never change the testvector dir + - cp -r $EVS_BE_TEST_DIR_FLOAT ./evs_be_test + - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod + - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec + + - cd evs_be_test + - python3 ../ci/run_evs_be_test.py + # --------------------------------------------------------------- # Build jobs # --------------------------------------------------------------- @@ -1356,7 +1383,7 @@ be-2-evs-26444: - .test-job-linux rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "evs-26444" - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' tags: - be-2-evs-basop stage: test @@ -1364,7 +1391,9 @@ be-2-evs-26444: script: - *print-common-info - *update-scripts-repo + # needed on BASOP branches only. On ivas-flaot-update + derivates this does nothing as the replaced string does not exist in options.h - sed -i".bak" "s/\(#define EVS_FLOAT\)/\/\/\1/" lib_com/options.h + - make clean - make -j # copy over to never change the testvector dir -- GitLab From 919684b1098c2e9370173bbbce34be4f56ffb96f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 09:40:44 +0100 Subject: [PATCH 11/23] add first-frame-is-sid job --- .gitlab-ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b0c91d58..0d93836e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -871,6 +871,25 @@ clang-format-check: name: "$ARTIFACT_BASE_NAME" expose_as: "formatting patch" +# from float +# check for crashes if first received frame on decoder side is an SID +check-first-frame-is-sid: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-float-pc + tags: + - ivas-linux + stage: test + needs: ["build-codec-linux-make"] + script: + - *print-common-info + - *update-scripts-repo + - *update-ltv-repo + # this rm makes check-for-testvectors only check for the signals we actually need in this test + - rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json + - *check-for-testvectors + - bash ci/run-first-frame-is-sid-test.sh + # from float check-bitexactness-hrtf-rom-and-file: extends: -- GitLab From 83de534fa7488a31c7c196e093b606c361b1df99 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 09:44:38 +0100 Subject: [PATCH 12/23] add renderer framesizes BE job --- .gitlab-ci.yml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d93836e6..e0015e9e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -242,6 +242,10 @@ stages: - git pull - cd - +.disable-limiter: &disable-limiter +# automatically enable #define DISABLE_LIMITER in options.h, handling both /**/-comment and //-comment + - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*DISABLE_LIMITER\)[[:space:]]*\*\//\1/g" ./lib_com/options.h + .get-commits-behind-count: &get-commits-behind-count - echo $CI_COMMIT_SHA - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME @@ -1549,6 +1553,55 @@ voip-be-on-merge-request: - USE_LTV="" <<: *sanitizer-selftest-anchor +# from float +# compare bit-exactness between 5ms and 20 on the branch +pytest-compare-20ms-and-5ms-rendering: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + stage: test + needs: ["build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + script: + - *print-common-info + - *update-scripts-repo + - *disable-limiter + - make clean + - make -j + ### prepare pytest + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + # create references + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + ### run pytest + - exit_code=0 + - exit_code5=0 + - exit_code10=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true + - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true + - zero_errors=1 + - if [ $zero_errors5 != 1 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors10 != 1 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $exit_code5 -ne 0 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi + - if [ $exit_code10 -ne 0 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi + - if [ $exit_code -ne 0 ]; then exit $EXIT_CODE_FAIL; fi + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + expose_as: "pytest 5ms and 10ms vs 20ms results" + paths: + - report-junit-5ms.xml + - report-5ms.html + - report-junit-10ms.xml + - report-10ms.html + reports: + junit: + - report-junit-5ms.xml + - report-junit-10ms.xml + # compare renderer bitexactness between target and source branch renderer-pytest-on-merge-request: extends: -- GitLab From e43f83f4bb0c2da35bd3443fa02d5ff073f6c189 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 09:50:08 +0100 Subject: [PATCH 13/23] port DISABLE_LIMITER switch --- lib_com/options.h | 2 ++ lib_rend/lib_rend.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib_com/options.h b/lib_com/options.h index ee78db378..1691f9dea 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -139,6 +139,8 @@ #endif /* DEBUGGING */ +/*#define DISABLE_LIMITER*/ /* disable the limiter - needed for testing Bitexactness between different renderer framings */ + /* #################### End DEBUGGING switches ############################ */ /* ################### Start FIXES switches ########################### */ diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 31c8e326c..1253423d4 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -336,6 +336,7 @@ static void accumulate2dArrayToBuffer( * In-place saturation control for multichannel buffers with adaptive release time *-------------------------------------------------------------------*/ +#ifndef DISABLE_LIMITER /*! r: number of clipped output samples */ static int32_t limitRendererOutput( IVAS_LIMITER_HANDLE hLimiter, /* i/o: limiter struct handle */ @@ -380,6 +381,7 @@ static int32_t limitRendererOutput( return numClipping; } +#endif /*-------------------------------------------------------------------* -- GitLab From e5e60338c0f1e5b38f386ee9d889ab870c35c25c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 10:31:21 +0100 Subject: [PATCH 14/23] port missing anchor --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e0015e9e5..544d3042f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -147,6 +147,10 @@ stages: .mr-get-target-commit: &mr-get-target-commit # compare to last target branch commit before pipeline was created - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) +# from float +.check-for-testvectors: &check-for-testvectors # check if the testvector files specified in scripts/config/ci_linux*.json are present + - python3 -m pytest ci/test_vectors_available.py + # From float CI .merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons -- GitLab From 4c86d1e574205f00900900a3aaad7e32d2197c3b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 10:33:07 +0100 Subject: [PATCH 15/23] use correct build job name --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 544d3042f..a02bef363 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1564,7 +1564,7 @@ pytest-compare-20ms-and-5ms-rendering: - .test-job-linux - .rules-merge-request-to-float-pc stage: test - needs: ["build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + needs: ["build-codec-linux-make", "build-codec-linux-instrumented-make", "build-codec-sanitizers-linux"] script: - *print-common-info - *update-scripts-repo -- GitLab From b648eced8dabba64a7ae434a5d744e9d0bed31df Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 10:35:56 +0100 Subject: [PATCH 16/23] add renderer smoke test --- .gitlab-ci.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a02bef363..c00acecc6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1606,6 +1606,31 @@ pytest-compare-20ms-and-5ms-rendering: - report-junit-5ms.xml - report-junit-10ms.xml +# from float +# test renderer executable +renderer-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + needs: ["build-codec-linux-make"] + stage: test + script: + - *print-common-info + - *update-scripts-repo + - make -j IVAS_rend + - testcase_timeout=60 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer make pytest results" + reports: + junit: + - report-junit.xml + # compare renderer bitexactness between target and source branch renderer-pytest-on-merge-request: extends: -- GitLab From 4d8ff8d21587596f37932c0ae6be43dfbfd8c3bd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 10:42:20 +0100 Subject: [PATCH 17/23] add renderer-*san jobs --- .gitlab-ci.yml | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c00acecc6..d71c32574 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1631,6 +1631,100 @@ renderer-smoke-test: junit: - report-junit.xml +# from float +# test renderer executable with cmake + asan +renderer-asan: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + needs: ["build-codec-linux-make"] + tags: + - ivas-linux + stage: test + script: + # was there from float repo. commented out until cmake is there + # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + # - cmake --build cmake-build -- -j + - make clean + - make -j + - testcase_timeout=180 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer asan pytest results" + reports: + junit: + - report-junit.xml + +# from float +# test renderer executable with cmake + msan +renderer-msan: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + needs: ["build-codec-linux-make"] + stage: test + tags: + - ivas-linux + script: + # was there from float repo. commented out until cmake is there + # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + # - cmake --build cmake-build -- -j + - make clean + - make -j + - testcase_timeout=180 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer msan pytest results" + reports: + junit: + - report-junit.xml + +# from float +# test renderer executable with cmake + usan +renderer-usan: + extends: + - .test-job-linux + - .rules-merge-request-to-float-pc + needs: ["build-codec-linux-make"] + stage: test + tags: + - ivas-linux + script: + # was there from float repo. commented out until cmake is there + # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + # - cmake --build cmake-build -- -j + - make clean + - make -j + - testcase_timeout=180 + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - grep_exit_code=0 + - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest + - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? + - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer.py with Clang undefined-behavior-sanitizer"; exit 1; fi + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer usan pytest results" + reports: + junit: + - report-junit.xml + # compare renderer bitexactness between target and source branch renderer-pytest-on-merge-request: extends: -- GitLab From 1912cb4758a7267f47f86415c533fb4227e186e5 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 10:45:21 +0100 Subject: [PATCH 18/23] remove unsupported testcases in framing compare job --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d71c32574..f6407a347 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1574,6 +1574,8 @@ pytest-compare-20ms-and-5ms-rendering: ### prepare pytest - cp IVAS_cod IVAS_cod_ref - cp IVAS_dec IVAS_dec_ref + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES # create references - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 ### run pytest -- GitLab From 2d9b17b9af9d34cbe630852f7fa7bc124fe19916 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 10:51:18 +0100 Subject: [PATCH 19/23] exclude testcases whcih use the custom binary file format feature --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f6407a347..b727e6eac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -912,7 +912,8 @@ check-bitexactness-hrtf-rom-and-file: - make clean - make -j - python3 tests/create_short_testvectors.py --cut_len 1.0 - - python3 -m pytest tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html + # TODO: run full test again once the custom binary files are supported + - python3 -m pytest -k "not diff_from_rom" tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html artifacts: paths: - report.html -- GitLab From cc1e49a80f32a13a5199c229c06e261b8be406eb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 10:55:21 +0100 Subject: [PATCH 20/23] correct rule for be-2-evs-2644 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b727e6eac..124f3f9b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1411,7 +1411,7 @@ be-2-evs-26444: - .test-job-linux rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "evs-26444" - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' (&& $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main-pc" ) tags: - be-2-evs-basop stage: test -- GitLab From eea29e934763a196ac9698976027298f2565ec3d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 11:22:02 +0100 Subject: [PATCH 21/23] correct pytest filter for hrtf job --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 124f3f9b8..c85dad5d2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -913,7 +913,7 @@ check-bitexactness-hrtf-rom-and-file: - make -j - python3 tests/create_short_testvectors.py --cut_len 1.0 # TODO: run full test again once the custom binary files are supported - - python3 -m pytest -k "not diff_from_rom" tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html + - python3 -m pytest -k "not diff_from_rom and not test_binary_file" tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html artifacts: paths: - report.html -- GitLab From e92c302d6fa605768b7a009b946e2d23abaa6a69 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 11:24:58 +0100 Subject: [PATCH 22/23] disable job until necessary fixes are ported --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c85dad5d2..f22e64677 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1560,7 +1560,8 @@ voip-be-on-merge-request: # from float # compare bit-exactness between 5ms and 20 on the branch -pytest-compare-20ms-and-5ms-rendering: +# TODO: activate again once all the renderer framesize BE fixes are ported +.pytest-compare-20ms-and-5ms-rendering: extends: - .test-job-linux - .rules-merge-request-to-float-pc -- GitLab From 242dcb954f6bf85db79b686007dd17f27dae33f5 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 20 Jan 2025 11:26:20 +0100 Subject: [PATCH 23/23] add forgotten anchor where it is needed --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f22e64677..b6d1f6de4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1649,6 +1649,7 @@ renderer-asan: # was there from float repo. commented out until cmake is there # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true # - cmake --build cmake-build -- -j + - *update-scripts-repo - make clean - make -j - testcase_timeout=180 @@ -1679,6 +1680,7 @@ renderer-msan: # was there from float repo. commented out until cmake is there # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true # - cmake --build cmake-build -- -j + - *update-scripts-repo - make clean - make -j - testcase_timeout=180 @@ -1709,6 +1711,7 @@ renderer-usan: # was there from float repo. commented out until cmake is there # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true # - cmake --build cmake-build -- -j + - *update-scripts-repo - make clean - make -j - testcase_timeout=180 -- GitLab