diff --git a/includes/default-variables-float.yml b/includes/default-variables-float.yml new file mode 100644 index 0000000000000000000000000000000000000000..296f068f69a5019991d4e359ac138a46fb357b6a --- /dev/null +++ b/includes/default-variables-float.yml @@ -0,0 +1,33 @@ +variables: + EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test" + EVS_BE_WIN_TEST_DIR: "C:/Users/gitlab-runner/testvec" + PROCESSING_SCRIPTS_BIN_DIR: "/test-bin" + SANITIZER_TESTS: "CLANG1 CLANG2 CLANG3" + OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4" + OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3" + OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB" + OUT_FORMATS_ALL: "$OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT" + MANUAL_PIPELINE_TYPE: + description: "Type for the manual pipeline run. Use 'test-be-release' to run BE test against release codec." + value: 'default' + options: + - 'default' + - 'test-be-release' + - 'test-long-self-test' + - 'ivas-conformance' + - 'ivas-conformance-linux' + - 'check-clipping' + - 'test-branch-vs-input-passthrough' + - 'coverage' + + PYTEST_ARGS: "" + LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" + LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" + LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" + TEST_SUITE: "" + COVERAGE_OUTPUT_FILE_STV: "coverage-stv.info" + COVERAGE_OUTPUT_FILE_LTV: "coverage-ltv.info" + COVERAGE_OUTPUT_FILE_CONFORMANCE: "coverage-conformance.info" + COVERAGE_OUTPUT_FILE_MERGED: "coverage-merged.info" diff --git a/includes/rules.yml b/includes/rules.yml index 486257f83103a667f79b67838c46d134485f0a09..0a588999475e9494ec0bf56b81930a205a9f071f 100644 --- a/includes/rules.yml +++ b/includes/rules.yml @@ -28,20 +28,6 @@ - if: $CI_PIPELINE_SOURCE == 'push' when: never -.rules-merge-request-to-basop-ci-branch: - extends: .rules-basis - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - -.rules-merge-request-to-basop-ci-branch-or-main: - extends: .rules-basis - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main") - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - .rules-main-push: extends: .rules-basis rules: diff --git a/main-float.yml b/main-float.yml new file mode 100644 index 0000000000000000000000000000000000000000..f17b861ae0bc8e792ae6ea1b3d21801db73172d4 --- /dev/null +++ b/main-float.yml @@ -0,0 +1,2762 @@ + +include: + - local: main.yml + - local: includes/default-variables-float.yml + +stages: + - .pre + - maintenance + - prevalidate + - build + - test + - basop-compat + - compare + - postvalidate + - deploy + +# This sets when pipelines are created. Jobs have more specific rules to restrict them. +workflow: + name: "$IVAS_PIPELINE_NAME" + rules: + # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + variables: + IVAS_PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main + variables: + IVAS_PIPELINE_NAME: "Push pipeline: $CI_COMMIT_BRANCH" + - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main + variables: + IVAS_PIPELINE_NAME: "Scheduled pipeline: $CI_COMMIT_BRANCH" + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing + variables: + IVAS_PIPELINE_NAME: "Web run pipeline: $CI_COMMIT_BRANCH" + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' + variables: + IVAS_PIPELINE_NAME: "Test BE to release pipeline: $CI_COMMIT_BRANCH" + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' + variables: + IVAS_PIPELINE_NAME: "Test long self-test against main pipeline: $CI_COMMIT_BRANCH" + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' + variables: + IVAS_PIPELINE_NAME: "Draft IVAS Conformance test: $CI_COMMIT_BRANCH" + - if: $CI_PIPELINE_SOURCE == 'trigger' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + variables: + IVAS_PIPELINE_NAME: "Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH" + - if: $CI_PIPELINE_SOURCE == 'trigger' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' + variables: + IVAS_PIPELINE_NAME: "Check core input clipping: $CI_COMMIT_BRANCH" + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + variables: + IVAS_PIPELINE_NAME: "Pass-through comparison vs input: $CI_COMMIT_BRANCH" + - if: $MANUAL_PIPELINE_TYPE == 'coverage' + variables: + IVAS_PIPELINE_NAME: "Coverage measurement" + +# --------------------------------------------------------------- +# Generic script anchors +# --------------------------------------------------------------- + +# These can be used later on to do common tasks + +.print-common-info-windows: &print-common-info-windows + - | + echo "Printing common information for build job." + echo "Current job is run on commit $CI_COMMIT_SHA" + echo "Commit time was $CI_COMMIT_TIMESTAMP" + ("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") | Invoke-Expression + +.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec + ### build test binaries, initial clean for paranoia reasons + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh + - make clean + - mkdir build + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_test + - mv IVAS_dec ../IVAS_dec_test + - mv IVAS_rend ../IVAS_rend_test + - cd .. + - rm -rf build/* + - git restore . + + ### store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### checkout version to compare against + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-fetch-target-branch.sh + - target_commit="$("${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-get-target-commit.sh)" + - git checkout $target_commit + - echo "Building reference codec at commit $target_commit" + + ### build reference binaries + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_ref + - mv IVAS_dec ../IVAS_dec_ref + - mv IVAS_rend ../IVAS_rend_ref + - cd .. + - git restore . + + # rename test binaries back + - mv IVAS_cod_test IVAS_cod + - mv IVAS_dec_test IVAS_dec + - mv IVAS_rend_test IVAS_rend + +.merge-request-comparison-check: &merge-request-comparison-check + - echo "--------------- Running merge-request-comparison-check anchor ---------------" + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + +.update-ltv-repo-win: &update-ltv-repo-win + - Push-Location + - cd $LTV_DIR_WIN + - git pull + - Pop-Location + +.copy-ltv-files-to-testv-dir-win: ©-ltv-files-to-testv-dir-win + - cp $LTV_DIR_WIN\*.wav scripts\testv + - cp $LTV_DIR_WIN\*.met scripts\testv + - cp $LTV_DIR_WIN\*.csv scripts\testv + +.activate-WX-windows: &activate-WX-windows + - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" + - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } + +# to be reused in MR and LTV-scheduled sanitizer test jobs +# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section +.sanitizer-selftest-anchor: + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - make clean + - make -j CLANG=$CLANG_NUM + - testcase_timeout=$SELFTEST_SANITY_TIMEOUT + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1 + + - exit_code20=0 + - exit_code10=0 + - exit_code5=0 + + - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi + +# --------------------------------------------------------------- +# .pre jobs for setting up things +# --------------------------------------------------------------- + +# See: https://gitlab.com/gitlab-org/gitlab/-/issues/194023 +# Solution to make main branch pipelines uninterruptible while all other +# pipelines can be interrupted by default. This works because all jobs +# after uninterruptible jobs will be uninterruptible. Resource group +# setting avoids rare case where two fast merges could still interrupt +# pipeline. This should be revisited if there are updates to Gitlab. +uninterruptible: + stage: .pre + interruptible: false + resource_group: uninterruptible + script: + - echo "$CI_COMMIT_BRANCH is uninterruptible" + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: always + tags: + - ivas-linux + +# --------------------------------------------------------------- +# Validation jobs +# --------------------------------------------------------------- + +branch-is-up-to-date-with-main-pre: + extends: + - .rules-merge-request-to-main + - .job-linux + stage: prevalidate + needs: [] + script: + - | + commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + echo "Commits behind: $commits_behind_count" + if [ "$commits_behind_count" -ne 0 ]; then + echo "Your branch is behind main, run 'git merge origin/main' to update." + exit 1 + fi + +check-self-test-names-pre: + extends: + - .rules-merge-request-to-main + - .job-linux + stage: prevalidate + needs: [] + script: + - python3 ci/check_self_test_names.py scripts/config/self_test.prm 135 + +check-no-duplicates-in-self-tests: + extends: + - .rules-merge-request-to-main + - .job-linux + stage: prevalidate + needs: [] + script: + - python3 ci/find_duplicates_in_prm_files.py + +branch-is-up-to-date-with-main-post: + extends: + - .rules-merge-request-to-main + - .job-linux + stage: postvalidate + script: + - | + commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + echo "Commits behind: $commits_behind_count" + if [ "$commits_behind_count" -ne 0 ]; then + echo "Your branch is behind main, possibly main changed during your pipeline run, run 'git merge origin/main' to update." + exit 1 + fi + +### BASOP compatibility test jobs +### these jobs run the tests with the BASOP executables to check if added testcases work fine both with float and BASOP codec +### run on manual trigger only to not prolong MR pipelines too much +.basop-ci-branch-compat-template: + extends: + - .test-job-linux + stage: basop-compat + # allows for pipelines to pass when these were not started + allow_failure: true + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + when: manual + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + tags: + - ivas-linux-fast + before_script: + - !reference [.job-linux, before_script] + - python3 scripts/prepare_combined_format_inputs.py + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/build-binaries-from-basop-repo.sh main + + # long timeout for BASOP executables + - testcase_timeout=$TESTCASE_TIMEOUT_LTV_SANITIZERS + + # hack for renderer tests to match the expected naming + - mv IVAS_rend IVAS_rend_ref + + # BASOP renderer test uses cut input signals + - | + if [ "$TEST_SUITE" == "tests/renderer_short/test_renderer.py" ]; then + python3 tests/create_short_testvectors --inplace --all --cut_len 5.0 + fi + - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --create_ref --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout + + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 1 month + paths: + - report-junit.xml + - report.html + + expose_as: "basop-compat results" + reports: + junit: + - report-junit.xml + +### The stv jobs are run for both main and ivas-float-update +basop-compat-stv: + extends: + - .basop-ci-branch-compat-template + variables: + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS + parallel: + matrix: + - TEST_SUITE: + - tests/codec_be_on_mr_nonselection/test_param_file.py + - tests/renderer_short/test_renderer.py + + script: + - exit 0 + +basop-compat-encoder-dmx-comp: + extends: + - .basop-ci-branch-compat-template + variables: + TEST_SUITE: $SHORT_TEST_SUITE_ENCODER + # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option + PYTEST_ARGS: "--compare_enc_dmx" + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS + script: + - exit 0 + +basop-compat-ltv: + extends: + - .basop-ci-branch-compat-template + variables: + TEST_SUITE: $LONG_TEST_SUITE + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS + script: + - exit 0 + +basop-compat-encoder-ltv: + extends: + - .basop-ci-branch-compat-template + variables: + TEST_SUITE: $LONG_TEST_SUITE_ENCODER + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS + script: + - exit 0 + + +# --------------------------------------------------------------- +# Build jobs +# --------------------------------------------------------------- + +build-codec-linux-make: + extends: + - .build-job-linux + - .rules-basis + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh + - make -j + +build-codec-linux-cmake: + extends: + - .build-job-linux + - .rules-basis + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh + - mkdir build + - cd build + - cmake .. + - cd .. + - make -C build -j + +build-codec-instrumented-linux: + extends: + - .build-job-linux + - .rules-basis + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - ./scripts/prepare_instrumentation.sh + - make -j -C scripts/c-code_instrument + artifacts: + when: always + access: all + expire_in: "30 days" + paths: + - wmc_tool_output.txt + +# make sure that the codec builds with msan, asan and usan +build-codec-sanitizers-linux: + extends: + - .build-job-linux + - .rules-basis + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh + - bash ci/build_codec_sanitizers_linux.sh + +build-codec-windows-cmake: + extends: + - .build-job-windows + - .rules-basis + script: + - *print-common-info-windows + - *activate-WX-windows + - cmake -G "Visual Studio 15 2017" . -Bbuild + - cmake --build build -j + +build-codec-windows-msbuild: + extends: + - .build-job-windows + - .rules-basis + script: + - *print-common-info-windows + - *activate-WX-windows + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug + +# --------------------------------------------------------------- +# Test jobs for merge requests +# --------------------------------------------------------------- + +# test that runs all modes with 1s input signals +codec-smoke-test: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + timeout: "20 minutes" + tags: + - ivas-linux-fast + stage: test + needs: + [ + "build-codec-linux-cmake", + "build-codec-linux-make", + "build-codec-instrumented-linux", + "build-codec-sanitizers-linux", + ] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + # LTV update needed as ltv ISM metadata files are used + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + + # for MRs to basop-ci-branch-pc, we want to test with the BASOP float reference build + # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros + # TODO: make manual for basop-CI merges + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/build-binaries-from-basop-repo.sh ivas-float-update + - bash ci/smoke_test.sh coverage + - else + - bash ci/smoke_test.sh + - fi + + ### analyze for failures + - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi + - ret_val=0 + - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without JBM failed"; ret_val=1; fi + - if cat smoke_test_output_jbm.txt | grep -c "failed"; then echo "Smoke test JBM part failed"; ret_val=1; fi + - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then echo "Smoke test with external hrtf files failed"; ret_val=1; fi + - exit $ret_val + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - smoke_test_output.txt + - smoke_test_output_jbm.txt + - smoke_test_output_hrtf.txt + expose_as: "Smoke test results" + +# code selftest testvectors with memory-sanitizer binaries +codec-msan: + extends: + - .sanitizer-selftest-on-mr + - .sanitizer-selftest-anchor + tags: + - ivas-linux-fast + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=1 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + +# code selftest testvectors with address-sanitizer binaries +codec-asan: + extends: + - .sanitizer-selftest-on-mr + - .sanitizer-selftest-anchor + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=2 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + +# code selftest testvectors with undefined-behaviour-sanitizer binaries +codec-usan: + extends: + - .sanitizer-selftest-on-mr + - .sanitizer-selftest-anchor + tags: + - ivas-linux-fast + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=3 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + +# compare bit-exactness between 5ms and 20 on the branch +pytest-compare-20ms-and-5ms-rendering: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: test + needs: + [ + "build-codec-linux-cmake", + "build-codec-linux-make", + "build-codec-instrumented-linux", + "build-codec-sanitizers-linux", + ] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh + - make clean + - make -j + ### prepare pytest + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + # create references + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + ### run pytest + - exit_code=0 + - exit_code5=0 + - exit_code10=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true + - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true + - zero_errors=1 + - if [ $zero_errors5 != 1 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors10 != 1 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $exit_code5 -ne 0 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi + - if [ $exit_code10 -ne 0 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi + - if [ $exit_code -ne 0 ]; then exit $EXIT_CODE_FAIL; fi + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + expose_as: "pytest 5ms and 10ms vs 20ms results" + paths: + - report-junit-5ms.xml + - report-5ms.html + - report-junit-10ms.xml + - report-10ms.html + reports: + junit: + - report-junit-5ms.xml + - report-junit-10ms.xml + +# test renderer executable +renderer-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j IVAS_rend + - testcase_timeout=60 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer make pytest results" + reports: + junit: + - report-junit.xml + +.renderer-sanitizer-job: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-cmake"] + stage: test + timeout: "90 minutes" + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + reports: + junit: + - report-junit.xml + before_script: + - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + - testcase_timeout=180 + +# test renderer executable with cmake + asan +renderer-asan: + extends: + - .renderer-sanitizer-job + variables: + SANITIZER_BUILD_STRING: "asan" + script: + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + artifacts: + expose_as: "renderer asan result" + +# test renderer executable with cmake + msan +renderer-msan: + extends: + - .renderer-sanitizer-job + variables: + SANITIZER_BUILD_STRING: "msan" + script: + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + artifacts: + expose_as: "renderer msan result" + +# test renderer executable with cmake + usan +renderer-usan: + extends: + - .renderer-sanitizer-job + variables: + SANITIZER_BUILD_STRING: "usan" + script: + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + artifacts: + expose_as: "renderer usan result" + +# compare renderer bitexactness between target and source branch +renderer-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + # TODO: set reasonable timeout, will most likely take less + timeout: "20 minutes" + stage: compare + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - *merge-request-comparison-setup-codec + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[rend\(erer\)*[ -]*non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + ### If ref_using_main is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + + ### Run test using branch scripts and input + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest renderer results" + reports: + junit: + - report-junit.xml + +# test split rendering +split-rendering-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j + - testcase_timeout=10 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "split rendering make pytest results" + reports: + junit: + - report-junit.xml + +# test split rendering in VoIP mode against BINAURAL output +split-rendering-voip-be-to-binaural: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j + - testcase_timeout=30 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_voip_be_splitrend_vs_binaural.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "VoIP split rendering vs BINAURAL make pytest results" + reports: + junit: + - report-junit.xml + +# test rtpdump +rtpdump-test: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j + - testcase_timeout=15 + - python3 -m pytest -q --tb=no -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/rtp/test_rtp.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "rtpdump pytest results" + reports: + junit: + - report-junit.xml + +lc3-wrapper-unit-test: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-cmake"] + stage: test + script: + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + +# compare split-rendering bitexactness between target and source branch +split-rendering-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + # TODO: set reasonable timeout, will most likely take less + timeout: "30 minutes" + stage: compare + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + # store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-fetch-target-branch.sh + - target_commit="$("${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-get-target-commit.sh)" + - git checkout $target_commit + - echo "Building reference codec at commit $target_commit" + + # build reference binaries + - make -j + - mv IVAS_cod IVAS_cod_ref + - mv IVAS_dec IVAS_dec_ref + - mv IVAS_rend IVAS_rend_ref + - mv ISAR_post_rend ISAR_post_rend_ref + + ### If ref_using_main is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + + # back to source branch + - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts + - git checkout $source_branch_commit_sha + - make clean + - make -j + + ### Run test using scripts and input from main + - if [ $ref_using_main == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 2 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest split rendering results" + reports: + junit: + - report-junit.xml + +# compare bit exactness between target and source branch +ivas-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + stage: compare + needs: ["build-codec-linux-cmake", "codec-smoke-test"] + timeout: "14 minutes" + parallel: + matrix: + # note: keep in sync with list in $TESTS_DIR_CODEC_BE_ON_MR + - PYTEST_SCRIPT: + - test_param_file.py + - test_sba.py + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - *merge-request-comparison-setup-codec + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + ### If ref_using_main is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + + ### prepare pytest + # create references + - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1 + + ### Run test using branch scripts and input + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + + ### run pytest + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest ivas results" + reports: + junit: + - report-junit.xml + +# Check interop IVAS_cod_test -> IVAS_dec_ref +ivas-interop-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-cmake"] + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - *merge-request-comparison-setup-codec + # the next line is dependent on ref-using-main flag in the other tests, but here the flag does not make sense + - git checkout $source_branch_commit_sha + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_interop_flag=$(grep -c --ignore-case "\[non[ -]*io\]" tmp.txt) || true + + ### prepare pytest + + # Run reference creation, using source branch encoder and main decoder (see .merge-request-comparison-setup-codec) + - exit_code=0 + - exit_code2=0 + # set timeout for individual testcase runs to 60 seconds + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true + + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "interop test results" + reports: + junit: + - report*-junit.xml + +evs-pytest-on-merge-request: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: compare + needs: ["build-codec-linux-cmake", "codec-smoke-test"] + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - *merge-request-comparison-setup-codec + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[evs[ -]*non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + ### If ref_using_main is not set, checkoug the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + + ### prepare pytest + # create references + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout + + ### Run test using branch scripts and input + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + + ### run pytest for EVS cases + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit-evs.xml + - report.html + expose_as: "pytest evs results" + reports: + junit: + - report-junit-evs.xml + +voip-be-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-make"] + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - make clean + - make -j + - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py + +clang-format-check: + extends: + - .test-job-linux + - .rules-merge-request-to-main + variables: + ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" + stage: prevalidate + needs: [] + timeout: "5 minutes" + script: + # Set up variables. This can't be done in the "variables" section because variables are not expanded properly there + - PATCH_FILE_NAME="$ARTIFACT_BASE_NAME".patch + - > + INSTRUCTIONS_GITLAB="To fix formatting issues:\n + - download the diff patch available as artifact of this job\n + - unzip the artifact and place the patch file in the root directory of your local IVAS repo\n + - run: git apply $PATCH_FILE_NAME\n + - commit new changes" + - > + INSTRUCTIONS_README="To fix formatting issues:\n + - place the patch file in the root directory of your local IVAS repo\n + - run: git apply $PATCH_FILE_NAME\n + - commit new changes" + + - format_problems=0 + - scripts/check-format.sh -af -p 8 || format_problems=$? + - if [ $format_problems == 0 ] ; then exit 0; fi + + - mkdir tmp-formatting-fix + - git diff > "tmp-formatting-fix/$PATCH_FILE_NAME" + + # Print instructions to job output + - echo -e "$INSTRUCTIONS_GITLAB" + + # Include readme in the artifact, in case someone misses the job printout (e.g. getting the artifact via MR interface) + - echo -e "$INSTRUCTIONS_README" > "tmp-formatting-fix/readme.txt" + + - exit $format_problems + artifacts: + expire_in: 1 day + paths: + - tmp-formatting-fix/ + when: on_failure + name: "$ARTIFACT_BASE_NAME" + expose_as: "formatting patch" + +# check for crashes if first received frame on decoder side is an SID +check-first-frame-is-sid: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-cmake"] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + # this rm makes check-for-testvectors only check for the signals we actually need in this test + - rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/check-for-testvectors.sh + - bash ci/run-first-frame-is-sid-test.sh + + artifacts: + paths: + - logs_enc + - logs_dec_msan + - logs_dec_asan + - logs_dec_usan + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--sidstart" + expose_as: "logs-sidstart" + expire_in: "5 days" + +lc3plus-ensure-no-code-changes: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: postvalidate + needs: [] + timeout: "5 minutes" + script: + # Replace code commited to repo with code downloaded from ETSI + - ./scripts/lc3plus_lib_setup/get_lc3plus.sh + + # Ensure git reports no changes + - modified_files=$(git status -su lib_lc3plus) + - if [[ $modified_files ]]; then printf 'LC3plus codebase was modified!\n\n'"$modified_files"'\n\n'; exit $EXIT_CODE_FAIL; fi + +check-bitexactness-hrtf-rom-and-file: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-cmake"] + timeout: "5 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - cmake . + - make -j + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html + artifacts: + paths: + - report.html + - report-junit.xml + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading" + expose_as: "logs-hrtf-loading" + expire_in: "5 days" + +check-bitexactness-ext-and-transport-format: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-cmake"] + timeout: "5 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - cmake . + - make -j + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest tests/test_be_for_ext_outputs.py --html=report.html --junit-xml=report-junit.xml --self-contained-html + artifacts: + paths: + - report.html + - report-junit.xml + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--ext-sanity-check" + expose_as: "logs-ext-sanity-check" + expire_in: "5 days" + +# --------------------------------------------------------------- +# Test jobs for main branch +# --------------------------------------------------------------- + +# check bitexactness to EVS windows binaries +be-2-evs-windows: + extends: + - .rules-merge-request-to-main + tags: + - ivas-windows + stage: test + needs: ["build-codec-windows-msbuild"] + timeout: "20 minutes" # To be revisited + script: + - *print-common-info-windows + + - $winoutdata = $null + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata + - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 + + # copy over to never change the testvector dir + - cp -r $EVS_BE_WIN_TEST_DIR ./evs_be_win_test + - cp IVAS_cod.exe ./evs_be_win_test/bin/IVAS_cod.exe + - cp IVAS_dec.exe ./evs_be_win_test/bin/IVAS_dec.exe + + - cd evs_be_win_test + - python ../ci/run_evs_be_win_test.py + +# TODO: turn into manual job if needed +# check bitexactness to EVS +# be-2-evs-linux: +# extends: +# - .test-job-linux +# - .rules-main-push +# tags: +# - be-2-evs-temp +# stage: test +# needs: ["build-codec-linux-cmake"] +# timeout: "20 minutes" # To be revisited +# script: +# - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh +# +# - mkdir build +# - cd build +# - cmake .. +# - make -j +# - cd .. +# +# # copy over to never change the testvector dir +# - cp -r $EVS_BE_TEST_DIR ./evs_be_test +# - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod +# - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec +# +# - cd evs_be_test +# - python3 ../ci/run_evs_be_test.py + +# TODO: do we still need this? +# codec-comparison-on-main-push: +# extends: +# - .test-job-linux-needs-testv-dir +# - .rules-main-push +# stage: compare +# needs: ["build-codec-linux-cmake"] +# timeout: "30 minutes" # To be revisited +# script: +# - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh +# - latest_commit=$(git rev-parse HEAD) # Latest commit +# - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) +# - echo "Comparing changes from $previous_merge_commit to $latest_commit" +# - git --no-pager diff --stat $previous_merge_commit..$latest_commit +# +# # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. +# +# ### build test binaries, initial clean for paranoia reasons +# - make clean +# - mkdir build +# - cd build +# - cmake .. +# - make -j +# - mv IVAS_cod ../IVAS_cod_test +# - mv IVAS_dec ../IVAS_dec_test +# - cd .. +# - rm -rf build/* +# +# ### compare to the previous merge commit in the main branch +# - git fetch origin main +# - git checkout $previous_merge_commit +# - echo "Building reference codec at commit $previous_merge_commit" +# +# ### build reference binaries +# - cd build +# - cmake .. +# - make -j +# - mv IVAS_cod ../IVAS_cod_ref +# - mv IVAS_dec ../IVAS_dec_ref +# - cd .. +# +# # helper variable - "|| true" to prevent failures from grep not finding anything +# # write to temporary file as workaround for failures observed with piping echo +# - echo $CI_COMMIT_MESSAGE > tmp.txt +# - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true +# - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true +# +# ### re-checkout the latest commit in the main branch, if ref_using_main is not set +# - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi +# +# ### prepare pytest +# # rename test binaries back +# - mv IVAS_cod_test IVAS_cod +# - mv IVAS_dec_test IVAS_dec +# # create references +# - testcase_timeout=60 +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout +# +# ### re-checkout the latest commit here, if ref_using_main is set +# - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi +# +# ### run pytest +# - exit_code=0 +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? +# - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi +# - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true +# - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi +# - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; +# allow_failure: +# exit_codes: +# - 123 +# artifacts: +# name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" +# expire_in: 1 week +# when: always +# paths: +# - report-junit.xml +# - report.html +# expose_as: "Results of comparison to previous merge commit" +# reports: +# junit: report-junit.xml + +# --------------------------------------------------------------- +# Manual jobs +# --------------------------------------------------------------- + +test-be-to-release: + stage: test + tags: + - ivas-windows + resource_group: ivas-be-to-release-test-resource + timeout: "20 minutes" + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' + script: + - echo "$CI_COMMIT_BRANCH" + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata + + # path to release candidate refs defined in config.toml + + - echo "Placeholder for BE test to release script" + - mkdir logs + + artifacts: + name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 1 week + paths: + - logs/ + expose_as: "test-be-to-release results" + +ivas-conformance: + tags: + - ivas-windows + stage: test + timeout: "90 minutes" + rules: + - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + allow_failure: + exit_codes: + - 123 + script: + - *print-common-info-windows + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug + - cp -force IVAS_cod.exe IVAS_cod_ref.exe + - cp -force IVAS_dec.exe IVAS_dec_ref.exe + - cp -force IVAS_rend.exe IVAS_rend_ref.exe + - cp -force ISAR_post_rend.exe ISAR_post_rend_ref.exe + + # Reference creation + - python scripts/prepare_combined_format_inputs.py + - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer_short/test_renderer.py", "tests/split_rendering/test_split_rendering.py" + - python -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html + - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt + + # Copy input data and output ref data + - if (Test-Path testvec) {rm -r -force testvec} + - if (Test-Path TMP_DEC) {rm -r -force TMP_DEC} + - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} + - if (Test-Path TMP_JBM) {rm -r -force TMP_JBM} + - if (Test-Path TMP_REND) {rm -r -force TMP_REND} + - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} + - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} + - mkdir testvec + - mkdir testvec/binauralRenderer_interface + - mkdir testvec/testv + - mkdir testvec/testv/renderer_short + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin + - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv + - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec + - cp -r -force -ErrorAction Ignore scripts/object_edit testvec + - cp -r -force -ErrorAction Ignore scripts/switchPaths testvec + - cp -r -force -ErrorAction Ignore scripts/trajectories testvec + - cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface + - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref + - cp -r -force -ErrorAction Ignore tests/renderer_short/ref testvec/testv/renderer_short/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/ref testvec/testv/split_rendering/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files + - python scripts/cleanup_26252.py + + # Copy test script files + - cp -r -force -ErrorAction Ignore tests/conformance-test testvec/ + - cp Readme_IVAS_dec.txt testvec + - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_rend.txt testvec + - cp Readme_IVAS_JBM_dec.txt testvec + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec + - cp IVAS_cod.exe testvec/bin + - cp IVAS_dec.exe testvec/bin + - cp IVAS_rend.exe testvec/bin + - cp ISAR_post_rend.exe testvec/bin + + # Test run generated scripts in testvec + - cd testvec + - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html + - mv report.html .. + - mv report-junit.xml .. + + artifacts: + name: "ivas-conformance-$CI_COMMIT_SHORT_SHA" + expire_in: 1 week + when: always + paths: + - report_cmd.html + - report-junit.xml + - report.html + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt + - Readme_IVAS_rend.txt + - Readme_IVAS_JBM_dec.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt + expose_as: "Draft IVAS conformance" + reports: + junit: report-junit.xml + +ivas-conformance-linux: + extends: + - .job-linux + stage: test + timeout: "90 minutes" + rules: + - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + - if: $MANUAL_PIPELINE_TYPE == "coverage" + - if: $COVERAGE_TEST + allow_failure: + exit_codes: + - 123 + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_CONFORMANCE" + COVERAGE_OUTPUT_DIR: "coverage_conformance" + COVERAGE_TITLE: "conformance test test_26252.py" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - make -j + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + - cp IVAS_rend IVAS_rend_ref + - cp ISAR_post_rend ISAR_post_rend_ref + + # Reference creation + - python3 scripts/prepare_combined_format_inputs.py + - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py" + - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html + - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt + + # Copy input data and output ref data + - rm -rf testvec + - mkdir testvec + - mkdir testvec/binauralRenderer_interface + - mkdir testvec/testv + - mkdir testvec/testv/renderer_short + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin + - cp -r scripts/testv/* testvec/testv + - cp -r scripts/ls_layouts testvec + - cp -r scripts/object_edit testvec + - cp -r scripts/switchPaths testvec + - cp -r scripts/trajectories testvec + - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface + - cp -r tests/ref testvec/testv/ref + - cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref + - cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref + - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files + - python3 scripts/cleanup_26252.py + + # Copy test script files + - cp -r tests/conformance-test testvec/ + - cp Readme_IVAS_dec.txt testvec + - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_rend.txt testvec + - cp Readme_IVAS_JBM_dec.txt testvec + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec + + # Create GCOV execs for coverage analysis + - make clean + - make GCOV=1 -j + + - cp IVAS_cod testvec/bin + - cp IVAS_dec testvec/bin + - cp IVAS_rend testvec/bin + - cp ISAR_post_rend testvec/bin + + # Test run generated scripts in testvec + - cd testvec + - exit_code=0 + - python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? + - mv report.html .. + - mv report-junit.xml .. + + # Collect coverage + - cd - + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE" + + # Check for failures + - if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi + + artifacts: + name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" + expire_in: 1 week + when: always + paths: + - report_cmd.html + - report-junit.xml + - report.html + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt + - Readme_IVAS_rend.txt + - Readme_IVAS_JBM_dec.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR + expose_as: "Draft IVAS conformance -- Linux" + reports: + junit: report-junit.xml + +test-long-self-test: + extends: + - .job-linux + stage: compare + resource_group: ivas-long-self-test-resource + timeout: "50 minutes" + tags: + - ivas-linux-fast + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' + allow_failure: + exit_codes: + - 123 + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + + ### build branch binaries + - cmake . -Bbuild-test + - cmake --build build-test -- -j + - mv build-test/IVAS_cod ./IVAS_cod + - mv build-test/IVAS_dec ./IVAS_dec + + ### store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### switch to main + - git checkout main + - git pull origin main + - echo "Building reference codec at commit $(git rev-parse HEAD)" + + ### build main (ref) binaries + - cmake . -Bbuild-ref + - cmake --build build-ref -- -j + - mv build-ref/IVAS_cod ./IVAS_cod_ref + - mv build-ref/IVAS_dec ./IVAS_dec_ref + + ### Switch back to branch, this could probably be removed later + - git checkout $source_branch_commit_sha + + ### prepare pytest + + # Copy test vectors from LTV to TESTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + + # create references + - exit_code_ref=0 + - testcase_timeout=300 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? + + ### run pytest self-test using long test vectors + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + artifacts: + name: "test-long-self-test--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 4 mos + paths: + - report-junit-ltv.xml + - report-ltv.html + expose_as: "test-long-self-test results" + reports: + junit: + - report-junit-ltv.xml + +check-clipping: + extends: + - .job-linux + stage: test + timeout: "30 minutes" + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' + allow_failure: + exit_codes: + - 123 + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/enable-debugging-macro.sh + - make -j + - tests/scale_pcm.py ./scripts/testv/ $SCALE_FACTOR # Default: 3.162 (+10 dB). Can be set in manual trigger. + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 scripts/parse_xml_report.py report-junit.xml report.csv --clipping + artifacts: + name: "check-clipping--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 2 weeks + paths: + - report-junit.xml + - report.html + - report.csv + expose_as: "check-clipping results" + reports: + junit: + - report-junit.xml + +test-branch-vs-input-passthrough: + extends: + - .job-linux + stage: compare + timeout: "30 minutes" # TBD + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + allow_failure: + exit_codes: + - 123 + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - make -j + + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py report-junit.xml report.csv + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ]; then echo "Differences encountered."; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + artifacts: + name: "test-branch-vs-input-passthrough--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 4 mos + paths: + - report-junit.xml + - report.html + - report.csv + expose_as: "test-branch-vs-input-passthrough results" + reports: + junit: + - report-junit.xml + +# --------------------------------------------------------------- +# Scheduled jobs on main +# --------------------------------------------------------------- + +# code selftest long testvectors with memory-sanitizer binaries +ltv-msan: + extends: + - .sanitizer-selftest-ltv + - .sanitizer-selftest-anchor + rules: + - if: $SANITIZER_SCHEDULE_E + timeout: 5 hours + tags: + - ivas-linux-fast + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=1 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" + - USE_LTV="--use_ltv" + +# code selftest long testvectors with address-sanitizer binaries +ltv-asan: + extends: + - .sanitizer-selftest-ltv + - .sanitizer-selftest-anchor + rules: + - if: $SANITIZER_SCHEDULE_E + when: delayed + start_in: 5 hours + tags: + - ivas-linux-fast + timeout: 5 hours + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=2 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" + - USE_LTV="--use_ltv" + +# code selftest long testvectors with undefined-behaviour-sanitizer binaries +ltv-usan: + extends: + - .sanitizer-selftest-ltv + - .sanitizer-selftest-anchor + rules: + - if: $SANITIZER_SCHEDULE_E + when: delayed + start_in: 8 hours + tags: + - ivas-linux-fast + timeout: 5 hours + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=3 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" + - USE_LTV="--use_ltv" + +.sanitizer-test-template: + extends: + - .test-job-linux + stage: test + tags: + - sanitizer_test_main + - ivas-linux-fast + artifacts: + name: "$CI_JOB_NAME--main--sha-$CI_COMMIT_SHORT_SHA" + expire_in: 7 weeks + when: always + paths: + - ep_015.g192 + - dly_profile.dat + - head_rot_traj.csv + - exof_traj.csv + - ./LOGS_PLC + - ./LOGS_noPLC + +### --- sanitizer schedule A --- + +.sanitizer-test-schedule-A: + extends: + - .sanitizer-test-template + +sanitizer-test-mono: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + timeout: 1.25 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS + +sanitizer-test-stereo: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 1.25 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED EXT --tests $SANITIZER_TESTS + +sanitizer-test-stereodmxevs: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 3.75 hours + timeout: 1.25 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py StereoDmxEVS mono --tests $SANITIZER_TESTS + +sanitizer-test-mc-5_1: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 5 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-mc-5_1_2: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 7.5 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-mc-5_1_4: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 10 hours + timeout: 3.75 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-mc-7_1: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 13.75 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-mc-7_1_4: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 16.25 hours + timeout: 5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-ism+1: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 21.25 hours + timeout: 1.25 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-ism+2: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 22.5 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-ism+3: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 25 hours + timeout: 3.75 hour + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-ism+4: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 28.75 hours + timeout: 5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-masa: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 33.75 hours + timeout: 10 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +### --- sanitizer schedule B --- + +.sanitizer-test-schedule-B: + extends: + - .sanitizer-test-template + timeout: 7.5 hours + +sanitizer-test-foa: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-hoa2: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 7.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-hoa3: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 15 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-foa-ism1: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 22.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-foa-ism2: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 30 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-foa-ism3: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 37.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-foa-ism4: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 45 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa2-ism1: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 52.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa2-ism2: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 60 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa2-ism3: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 67.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa2-ism4: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 75 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa3-ism1: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 82.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa3-ism2: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 90 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-omasa-ism4: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 97.5 hours + timeout: 10 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +### --- sanitizer schedule C --- + +.sanitizer-test-schedule-C: + extends: + - .sanitizer-test-template + timeout: 10 hours + +sanitizer-test-omasa-ism1: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-omasa-ism2: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + when: delayed + start_in: 10 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-omasa-ism3: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + when: delayed + start_in: 20 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa3-ism3: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + when: delayed + start_in: 30 hours + timeout: 7.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa3-ism4: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + when: delayed + start_in: 37.5 hours + timeout: 7.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +### --- sanitizer schedule D --- + +.sanitizer-test-schedule-D: + extends: + - .sanitizer-test-template + timeout: 7.5 hours + +sanitizer-test-planar-foa: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-planar-hoa2: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 7.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-planar-hoa3: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 15 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-foa-ism1: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 22.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-foa-ism2: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 30 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-foa-ism3: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 37.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-foa-ism4: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 45 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa2-ism1: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 52.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa2-ism2: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 60 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa2-ism3: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 67.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa2-ism4: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 75 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa3-ism1: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 82.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa3-ism2: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 90 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa3-ism3: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 97.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa3-ism4: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 105 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +# GCOV/LCOV coverage analysis of self_test suite +.coverage-test-template: + extends: + - .test-job-linux-needs-testv-dir + - .rules-main-scheduled + tags: + - ivas-linux-fast + stage: test + rules: + - if: $COVERAGE_TEST + - if: $MANUAL_PIPELINE_TYPE == "coverage" + timeout: 6 hours + before_script: + - !reference [.test-job-linux-needs-testv-dir, before_script] + - set -e + - 'trap ''echo "Command failed at line $LINENO: $BASH_COMMAND"'' ERR' + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + # compile with coverage enabled + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true -DGCOV=ON + - cmake --build cmake-build -- -j + # copy executables to ref to be able to run some --create_ref tests + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + - cp IVAS_rend IVAS_rend_ref + - cp ISAR_post_rend ISAR_post_rend_ref + after_script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir cmake-build --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE" + artifacts: + name: "main-$COVERAGE_OUTPUT_DIR-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week + +coverage-merge: + extends: + - .rules-main-scheduled + - .test-job-linux-needs-testv-dir + tags: + - ivas-linux-fast + stage: test + rules: + - if: $COVERAGE_TEST + - if: $MANUAL_PIPELINE_TYPE == "coverage" + needs: + - job: coverage-test-on-main-scheduled-stv + artifacts: true + - job: coverage-test-on-main-scheduled-ltv + artifacts: true + - job: ivas-conformance-linux + artifacts: true + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_MERGED" + COVERAGE_OUTPUT_DIR: "coverage-merged" + COVERAGE_TITLE: "merged total coverage" + script: + - echo "Merging coverage reports..." + - lcov -a $COVERAGE_OUTPUT_FILE_CONFORMANCE -a $COVERAGE_OUTPUT_FILE_STV -a $COVERAGE_OUTPUT_FILE_LTV -o "$COVERAGE_OUTPUT_FILE_MERGED" + - commit_sha=$(git rev-parse HEAD) + - genhtml $COVERAGE_OUTPUT_FILE_MERGED -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" + artifacts: + name: "merged-coverage-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week + paths: + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR + +coverage-test-on-main-scheduled-stv: + extends: .coverage-test-template + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_STV" + COVERAGE_OUTPUT_DIR: "coverage_stv" + COVERAGE_TITLE: "short test vectors" + script: + # -- Run short test vector suite to check coverage of that first + # codec smoke test + - bash ci/smoke_test.sh coverage + # default renderer framesize is 20ms + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true + # run renderer smoke test - needed for REFVEC tests + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py + # run renderer test only with REF creation stage - no CUT needed since executables are identical + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref + # run split rendering smoke test and REF creation + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref + # VoIP BE test + - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py + # split rendering VoIP BE test + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py + # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted + # - bash ci/run-first-frame-is-sid-test.sh + # TODO consider HRTF binary loading test, or probably already covered by PRM files + # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading + # run BE vs EXT test + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py + # run LC3plus wrapper unit test + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job + artifacts: + paths: + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR + +coverage-test-on-main-scheduled-ltv: + extends: .coverage-test-template + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_LTV" + COVERAGE_OUTPUT_DIR: "coverage_ltv" + COVERAGE_TITLE: "long test vectors" + script: + # -- Add extended coverage of EVS test and long test vectors + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # default renderer framesize is 20ms + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true + # run renderer smoke test - needed for REFVEC tests + - ls -altr scripts/testv + # LTV smoketest is the same as REF creation stage for renderer + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv + # split rendering test doesn't support LTV - skipped here + # VoIP BE tests don't support LTV - skipped here + artifacts: + paths: + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR + +# --------------------------------------------------------------- +# Complexity measurement jobs +# --------------------------------------------------------------- + +.complexity-measurements-setup: + &complexity-measurements-setup # create necessary environment + - mkdir -p wmops/logs + + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) + - echo $job_id + - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip + - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html + - public_dir="$CI_JOB_NAME-public" + + # if is needed to catch case when no artifact is there (first run), similarly as above + # 1. check for public_dir being there as this might not be the case when artifact download failed + # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts + - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then + - mv $public_dir/* wmops/ + # check here if we have the split-by-levels files present - if not, fake them up with the existing global one + # this is needed for the first run with split graphs on a branch where the global version did run previously + # NOTE: checking only for level_1 file here as this should already be sufficient + # NOTE2: also not chechking for RAM for same reason + - wmops_all_global="wmops/log_wmops_all.txt" + - ram_all_global="wmops/log_ram_all.txt" + - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then + - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") + - for suffix in "${suffixes[@]}"; do + - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt + - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt + - done + - fi + - fi + + - rm artifacts.zip + - rm -rf $public_dir + +.complexity-measurements-prepare-artifacts: + &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory + - public_dir="$CI_JOB_NAME-public" + - mkdir $public_dir + - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/ + # move logfiles for links + - mkdir $public_dir/logs + # first move logs + - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g") + - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done + # copy index page blueprint + - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html + # patch the format in the title + - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html + # do separately here to avoid overwrite complaints by mv + - mv -f ci/complexity_measurements/style.css ${public_dir}/ + +.complexity-template: + extends: + # still needed as long as no long MASA testvectors are there + - .test-job-linux-needs-testv-dir + tags: + - ivas-linux-fast + timeout: 3 hours 30 minutes + stage: test + before_script: + - !reference [.job-linux, before_script] + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - *complexity-measurements-setup + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA" + when: always + expire_in: 1 week + paths: + - $CI_JOB_NAME-public + +complexity-stereo-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + script: + - in_format=stereo + - out_format=ext + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-ism-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 1 hour + script: + - in_format=ISM + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-ism-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 2 hours + script: + - in_format=ISM + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-ism-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 3 hours + script: + - in_format=ISM + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-sba-hoa3-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 4 hours + script: + - in_format=HOA3 + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-sba-hoa3-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 5 hours + script: + - in_format=HOA3 + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-sba-hoa3-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 6 hours + script: + - in_format=HOA3 + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-mc-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 7 hours + script: + - in_format=MC + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-mc-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 8 hours + script: + - in_format=MC + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-mc-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 9 hours + script: + - in_format=MC + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 10 hours + script: + - in_format=MASA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 11 hours + script: + - in_format=MASA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-hoa3-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 12 hours + script: + - in_format=MASA + - out_format=HOA3 + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 13 hours + script: + - in_format=OMASA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 14 hours + script: + - in_format=OMASA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-hoa3-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 15 hours + script: + - in_format=OMASA + - out_format=HOA3 + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-StereoDmxEVS-stereo-in-mono-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 16 hours + script: + - in_format=StereoDmxEVS + - out_format=mono + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-osba-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 17 hours + script: + - in_format=OSBA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-osba-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 18 hours + script: + - in_format=OSBA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-osba-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 19 hours + script: + - in_format=OSBA + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +# --------------------------------------------------------------- +# Other jobs +# --------------------------------------------------------------- + +# job that sets up gitlab pages website +pages: + stage: deploy + extends: + - .job-linux + rules: + - if: $UPDATE_PAGES + script: + - python3 ci/setup_pages.py + artifacts: + paths: + - public + expire_in: 1 day + +# Pull state of a branch on 3GPP repo, push to a mirror repo. +pull-from-3gpp-forge: + stage: maintenance + rules: + - if: $MIRROR_ACCESS_TOKEN # Only run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) + script: + # Set up git LFS for mirroring (see: https://github.com/git-lfs/git-lfs/issues/1762) + - git lfs install --skip-smudge --local + + # Check out mirror branch - by default the runner checks out by commit hash, which results in detached head state + - git checkout $CI_COMMIT_BRANCH + + # Pull commits from upstream + - git remote add upstream https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec.git + - git pull --ff-only --tags upstream $MIRROR_SOURCE_BRANCH + - git lfs pull upstream + + # Push to mirror, include tags. Option `-o ci.skip` tells GitLab to skip CI for the pushed commits (assumed already tested upstream) + - git push --tags -o ci.skip "https://${GITLAB_USER_LOGIN}:${MIRROR_ACCESS_TOKEN}@${CI_REPOSITORY_URL#*@}" "HEAD:${CI_COMMIT_BRANCH}" diff --git a/main.yml b/main.yml index df75d2cbabdf9e7d6565ca8fa042e83e02956203..9062a0c8b0338dcfd470a71a9d88b70f95b7aa8f 100644 --- a/main.yml +++ b/main.yml @@ -7,14 +7,3 @@ include: default: # by default, we want all jobs to be interruptible, and we need to configure this explicitly interruptible: true - - # Note: currently only used for float -stages: - - .pre - - maintenance - - prevalidate - - build - - test - - compare - - postvalidate - - deploy