From b13b4e4739e394f85772d31e795fb21f0c0e5950 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 Sep 2025 15:59:16 +0200 Subject: [PATCH 1/5] move ci code from ivas-codec repo to here --- includes/default-variables-float.yml | 0 main-float.yml | 2662 ++++++++++++++++++++++++++ main.yml | 11 - 3 files changed, 2662 insertions(+), 11 deletions(-) create mode 100644 includes/default-variables-float.yml create mode 100644 main-float.yml diff --git a/includes/default-variables-float.yml b/includes/default-variables-float.yml new file mode 100644 index 0000000..e69de29 diff --git a/main-float.yml b/main-float.yml new file mode 100644 index 0000000..a035c6d --- /dev/null +++ b/main-float.yml @@ -0,0 +1,2662 @@ + +include: + - local: main.yml + - local: includes/default-variables-float.yml + +stages: + - .pre + - maintenance + - prevalidate + - build + - test + - compare + - postvalidate + - deploy + +# This sets when pipelines are created. Jobs have more specific rules to restrict them. +workflow: + name: '$IVAS_PIPELINE_NAME' + rules: + # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + variables: + IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main + variables: + IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main + variables: + IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing + variables: + IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' + variables: + IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' + variables: + IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' + variables: + IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'trigger' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + variables: + IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'trigger' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' + variables: + IVAS_PIPELINE_NAME: 'Check core input clipping: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + variables: + IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH' + +# --------------------------------------------------------------- +# Generic script anchors +# --------------------------------------------------------------- + +# These can be used later on to do common tasks + +.print-common-info-windows: &print-common-info-windows + - | + echo "Printing common information for build job." + echo "Current job is run on commit $CI_COMMIT_SHA" + echo "Commit time was $CI_COMMIT_TIMESTAMP" + ("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") | Invoke-Expression + +.get-basop-float-reference: &get-basop-float-reference + - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch --depth 1 ivas-basop + - cd ivas-basop + - git status + - make -j + - cd - + - cp ivas-basop/IVAS_cod ./ + - cp ivas-basop/IVAS_dec ./ + +.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec + ### build test binaries, initial clean for paranoia reasons + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh + - make clean + - mkdir build + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_test + - mv IVAS_dec ../IVAS_dec_test + - mv IVAS_rend ../IVAS_rend_test + - cd .. + - rm -rf build/* + - git restore . + + ### store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### checkout version to compare against + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-fetch-target-branch.sh + - target_commit="$("${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-get-target-commit.sh)" + - git checkout $target_commit + - echo "Building reference codec at commit $target_commit" + + ### build reference binaries + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_ref + - mv IVAS_dec ../IVAS_dec_ref + - mv IVAS_rend ../IVAS_rend_ref + - cd .. + - git restore . + + # rename test binaries back + - mv IVAS_cod_test IVAS_cod + - mv IVAS_dec_test IVAS_dec + - mv IVAS_rend_test IVAS_rend + +.merge-request-comparison-check: &merge-request-comparison-check + - echo "--------------- Running merge-request-comparison-check anchor ---------------" + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + +.update-ltv-repo-win: &update-ltv-repo-win + - Push-Location + - cd $LTV_DIR_WIN + - git pull + - Pop-Location + +.copy-ltv-files-to-testv-dir-win: ©-ltv-files-to-testv-dir-win + - cp $LTV_DIR_WIN\*.wav scripts\testv + - cp $LTV_DIR_WIN\*.met scripts\testv + - cp $LTV_DIR_WIN\*.csv scripts\testv + +.activate-WX-windows: &activate-WX-windows + - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" + - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } + +# to be reused in MR and LTV-scheduled sanitizer test jobs +# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section +.sanitizer-selftest-anchor: + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - make clean + - make -j CLANG=$CLANG_NUM + - testcase_timeout=$SELFTEST_SANITY_TIMEOUT + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 + + - exit_code20=0 + - exit_code10=0 + - exit_code5=0 + + - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi + + +# --------------------------------------------------------------- +# .pre jobs for setting up things +# --------------------------------------------------------------- + +# See: https://gitlab.com/gitlab-org/gitlab/-/issues/194023 +# Solution to make main branch pipelines uninterruptible while all other +# pipelines can be interrupted by default. This works because all jobs +# after uninterruptible jobs will be uninterruptible. Resource group +# setting avoids rare case where two fast merges could still interrupt +# pipeline. This should be revisited if there are updates to Gitlab. +uninterruptible: + stage: .pre + interruptible: false + resource_group: uninterruptible + script: + - echo "$CI_COMMIT_BRANCH is uninterruptible" + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: always + tags: + - ivas-linux + +# --------------------------------------------------------------- +# Validation jobs +# --------------------------------------------------------------- + +branch-is-up-to-date-with-main-pre: + extends: + - .rules-merge-request-to-main + - .job-linux + stage: prevalidate + needs: [] + script: + - | + commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + echo "Commits behind: $commits_behind_count" + if [ "$commits_behind_count" -ne 0 ]; then + echo "Your branch is behind main, run 'git merge origin/main' to update." + exit 1 + fi + +check-self-test-names-pre: + extends: + - .rules-merge-request-to-main + - .job-linux + stage: prevalidate + needs: [] + script: + - python3 ci/check_self_test_names.py scripts/config/self_test.prm 135 + +check-no-duplicates-in-self-tests: + extends: + - .rules-merge-request-to-main + - .job-linux + stage: prevalidate + needs: [] + script: + - python3 ci/find_duplicates_in_prm_files.py + +branch-is-up-to-date-with-main-post: + extends: + - .rules-merge-request-to-main + - .job-linux + stage: postvalidate + script: + - | + commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + echo "Commits behind: $commits_behind_count" + if [ "$commits_behind_count" -ne 0 ]; then + echo "Your branch is behind main, possibly main changed during your pipeline run, run 'git merge origin/main' to update." + exit 1 + fi + +.basop-ci-branch-compat-template: + extends: + - .test-job-linux + - .rules-merge-request-to-basop-ci-branch + tags: + - ivas-linux + before_script: + - !reference [.job-linux, before_script] + - python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm + - python3 tests/create_short_testvectors.py + - python3 scripts/prepare_combined_format_inputs.py + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + + - *get-basop-float-reference + + - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT + + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 1 month + paths: + - report-junit.xml + - report.html + + expose_as: "check-float-reference results" + reports: + junit: + - report-junit.xml + +basop-ref-compat-stv: + extends: + - .basop-ci-branch-compat-template + variables: + TEST_SUITE: "$SHORT_TEST_SUITE tests/renderer" + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS + script: + - exit 0 + +basop-ref-compat-ltv: + extends: + - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast + variables: + TEST_SUITE: $LONG_TEST_SUITE + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS + script: + - exit 0 + +basop-ref-compat-encoder-stv: + extends: + - .basop-ci-branch-compat-template + variables: + TEST_SUITE: $SHORT_TEST_SUITE_ENCODER + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS + script: + - exit 0 + +basop-ref-compat-encoder-ltv: + extends: + - .basop-ci-branch-compat-template + tags: + - ivas-linux-fast + variables: + TEST_SUITE: $LONG_TEST_SUITE_ENCODER + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS + script: + - exit 0 + +basop-ref-compat-encoder-dmx-comp: + extends: + - .basop-ci-branch-compat-template + variables: + TEST_SUITE: $SHORT_TEST_SUITE_ENCODER + # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option + PYTEST_ARGS: "--compare_enc_dmx" + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS + script: + - exit 0 + +# --------------------------------------------------------------- +# Build jobs +# --------------------------------------------------------------- + +build-codec-linux-make: + extends: + - .build-job-linux + - .rules-basis + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh + - make -j + +build-codec-linux-cmake: + extends: + - .build-job-linux + - .rules-basis + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh + - mkdir build + - cd build + - cmake .. + - cd .. + - make -C build -j + +build-codec-instrumented-linux: + extends: + - .build-job-linux + - .rules-basis + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - ./scripts/prepare_instrumentation.sh + - make -j -C scripts/c-code_instrument + artifacts: + when: always + access: all + expire_in: "30 days" + paths: + - wmc_tool_output.txt + + +# make sure that the codec builds with msan, asan and usan +build-codec-sanitizers-linux: + extends: + - .build-job-linux + - .rules-basis + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh + - bash ci/build_codec_sanitizers_linux.sh + +build-codec-windows-cmake: + extends: + - .build-job-windows + - .rules-basis + script: + - *print-common-info-windows + - *activate-WX-windows + - cmake -G "Visual Studio 15 2017" . -Bbuild + - cmake --build build -j + +build-codec-windows-msbuild: + extends: + - .build-job-windows + - .rules-basis + script: + - *print-common-info-windows + - *activate-WX-windows + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug + +# --------------------------------------------------------------- +# Test jobs for merge requests +# --------------------------------------------------------------- + +# test that runs all modes with 1s input signals +codec-smoke-test: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-basop-ci-branch-or-main + timeout: "20 minutes" + tags: + - ivas-linux-fast + stage: test + needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + # LTV update needed as ltv ISM metadata files are used + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + + # for MRs to basop-ci-branch, we want to test with the BASOP float reference build + # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then + - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline + - *get-basop-float-reference + - bash ci/smoke_test.sh coverage + - else + - bash ci/smoke_test.sh + - fi + + ### analyze for failures + - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi + - ret_val=0 + - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without JBM failed"; ret_val=1; fi + - if cat smoke_test_output_jbm.txt | grep -c "failed"; then echo "Smoke test JBM part failed"; ret_val=1; fi + - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then echo "Smoke test with external hrtf files failed"; ret_val=1; fi + - exit $ret_val + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - smoke_test_output.txt + - smoke_test_output_jbm.txt + - smoke_test_output_hrtf.txt + expose_as: "Smoke test results" + +# code selftest testvectors with memory-sanitizer binaries +codec-msan: + extends: + - .sanitizer-selftest-on-mr + - .sanitizer-selftest-anchor + tags: + - ivas-linux-fast + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=1 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + +# code selftest testvectors with address-sanitizer binaries +codec-asan: + extends: + - .sanitizer-selftest-on-mr + - .sanitizer-selftest-anchor + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=2 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + +# code selftest testvectors with undefined-behaviour-sanitizer binaries +codec-usan: + extends: + - .sanitizer-selftest-on-mr + - .sanitizer-selftest-anchor + tags: + - ivas-linux-fast + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=3 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + +# compare bit-exactness between 5ms and 20 on the branch +pytest-compare-20ms-and-5ms-rendering: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh + - make clean + - make -j + ### prepare pytest + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + # create references + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + ### run pytest + - exit_code=0 + - exit_code5=0 + - exit_code10=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true + - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true + - zero_errors=1 + - if [ $zero_errors5 != 1 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors10 != 1 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $exit_code5 -ne 0 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi + - if [ $exit_code10 -ne 0 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi + - if [ $exit_code -ne 0 ]; then exit $EXIT_CODE_FAIL; fi + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + expose_as: "pytest 5ms and 10ms vs 20ms results" + paths: + - report-junit-5ms.xml + - report-5ms.html + - report-junit-10ms.xml + - report-10ms.html + reports: + junit: + - report-junit-5ms.xml + - report-junit-10ms.xml + +# test renderer executable +renderer-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j IVAS_rend + - testcase_timeout=60 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer make pytest results" + reports: + junit: + - report-junit.xml + +.renderer-sanitizer-job: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-cmake"] + stage: test + timeout: "90 minutes" + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + reports: + junit: + - report-junit.xml + before_script: + - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + # rename files to fit naming convention + # en- and decoder needed because of split rendering testcases + - mv IVAS_cod IVAS_cod_ref + - mv IVAS_dec IVAS_dec_ref + - mv IVAS_rend IVAS_rend_ref + - mv ISAR_post_rend ISAR_post_rend_ref + - testcase_timeout=180 + +# test renderer executable with cmake + asan +renderer-asan: + extends: + - .renderer-sanitizer-job + variables: + SANITIZER_BUILD_STRING: "asan" + script: + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + artifacts: + expose_as: "renderer asan result" + + +# test renderer executable with cmake + msan +renderer-msan: + extends: + - .renderer-sanitizer-job + variables: + SANITIZER_BUILD_STRING: "msan" + script: + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + artifacts: + expose_as: "renderer msan result" + + +# test renderer executable with cmake + usan +renderer-usan: + extends: + - .renderer-sanitizer-job + variables: + SANITIZER_BUILD_STRING: "usan" + script: + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + artifacts: + expose_as: "renderer usan result" + + +# compare renderer bitexactness between target and source branch +renderer-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + # TODO: set reasonable timeout, will most likely take less + timeout: "20 minutes" + stage: compare + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - *merge-request-comparison-setup-codec + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[rend\(erer\)*[ -]*non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + ### If ref_using_main is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + + ### Run test using branch scripts and input + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest renderer results" + reports: + junit: + - report-junit.xml + +# test split rendering +split-rendering-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j + - testcase_timeout=10 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "split rendering make pytest results" + reports: + junit: + - report-junit.xml + +# test split rendering in VoIP mode against BINAURAL output +split-rendering-voip-be-to-binaural: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j + - testcase_timeout=30 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_voip_be_splitrend_vs_binaural.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "VoIP split rendering vs BINAURAL make pytest results" + reports: + junit: + - report-junit.xml + +lc3-wrapper-unit-test: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-cmake"] + stage: test + script: + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + +# compare split-rendering bitexactness between target and source branch +split-rendering-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + # TODO: set reasonable timeout, will most likely take less + timeout: "30 minutes" + stage: compare + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + # store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-fetch-target-branch.sh + - target_commit="$("${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-get-target-commit.sh)" + - git checkout $target_commit + - echo "Building reference codec at commit $target_commit" + + # build reference binaries + - make -j + - mv IVAS_cod IVAS_cod_ref + - mv IVAS_dec IVAS_dec_ref + - mv IVAS_rend IVAS_rend_ref + - mv ISAR_post_rend ISAR_post_rend_ref + + ### If ref_using_main is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + + # back to source branch + - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts + - git checkout $source_branch_commit_sha + - make clean + - make -j + + ### Run test using scripts and input from main + - if [ $ref_using_main == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 2 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest split rendering results" + reports: + junit: + - report-junit.xml + +# compare bit exactness between target and source branch +ivas-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + stage: compare + needs: ["build-codec-linux-cmake", "codec-smoke-test"] + timeout: "14 minutes" + parallel: + matrix: + # note: keep in sync with list in $TESTS_DIR_CODEC_BE_ON_MR + - PYTEST_SCRIPT: + - test_param_file.py + - test_sba.py + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - *merge-request-comparison-setup-codec + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + ### If ref_using_main is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + + ### prepare pytest + # create references + - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1 + + ### Run test using branch scripts and input + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + + ### run pytest + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest ivas results" + reports: + junit: + - report-junit.xml + + +# Check interop IVAS_cod_test -> IVAS_dec_ref +ivas-interop-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-cmake"] + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - *merge-request-comparison-setup-codec + # the next line is dependent on ref-using-main flag in the other tests, but here the flag does not make sense + - git checkout $source_branch_commit_sha + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_interop_flag=$(grep -c --ignore-case "\[non[ -]*io\]" tmp.txt) || true + + ### prepare pytest + + # Run reference creation, using source branch encoder and main decoder (see .merge-request-comparison-setup-codec) + - exit_code=0 + - exit_code2=0 + # set timeout for individual testcase runs to 60 seconds + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true + + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "interop test results" + reports: + junit: + - report*-junit.xml + +evs-pytest-on-merge-request: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: compare + needs: ["build-codec-linux-cmake", "codec-smoke-test"] + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - *merge-request-comparison-setup-codec + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[evs[ -]*non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + ### If ref_using_main is not set, checkoug the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + + ### prepare pytest + # create references + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout + + ### Run test using branch scripts and input + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + + ### run pytest for EVS cases + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit-evs.xml + - report.html + expose_as: "pytest evs results" + reports: + junit: + - report-junit-evs.xml + +voip-be-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-make"] + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - make clean + - make -j + - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py + +clang-format-check: + extends: + - .test-job-linux + - .rules-merge-request-to-main + variables: + ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" + stage: prevalidate + needs: [] + timeout: "5 minutes" + script: + # Set up variables. This can't be done in the "variables" section because variables are not expanded properly there + - PATCH_FILE_NAME="$ARTIFACT_BASE_NAME".patch + - > + INSTRUCTIONS_GITLAB="To fix formatting issues:\n + - download the diff patch available as artifact of this job\n + - unzip the artifact and place the patch file in the root directory of your local IVAS repo\n + - run: git apply $PATCH_FILE_NAME\n + - commit new changes" + - > + INSTRUCTIONS_README="To fix formatting issues:\n + - place the patch file in the root directory of your local IVAS repo\n + - run: git apply $PATCH_FILE_NAME\n + - commit new changes" + + - format_problems=0 + - scripts/check-format.sh -af -p 8 || format_problems=$? + - if [ $format_problems == 0 ] ; then exit 0; fi + + - mkdir tmp-formatting-fix + - git diff > "tmp-formatting-fix/$PATCH_FILE_NAME" + + # Print instructions to job output + - echo -e "$INSTRUCTIONS_GITLAB" + + # Include readme in the artifact, in case someone misses the job printout (e.g. getting the artifact via MR interface) + - echo -e "$INSTRUCTIONS_README" > "tmp-formatting-fix/readme.txt" + + - exit $format_problems + artifacts: + expire_in: 1 day + paths: + - tmp-formatting-fix/ + when: on_failure + name: "$ARTIFACT_BASE_NAME" + expose_as: "formatting patch" + +# check for crashes if first received frame on decoder side is an SID +check-first-frame-is-sid: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-cmake"] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + # this rm makes check-for-testvectors only check for the signals we actually need in this test + - rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/check-for-testvectors.sh + - bash ci/run-first-frame-is-sid-test.sh + + artifacts: + paths: + - logs_enc + - logs_dec_msan + - logs_dec_asan + - logs_dec_usan + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--sidstart" + expose_as: "logs-sidstart" + expire_in: "5 days" + +lc3plus-ensure-no-code-changes: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: postvalidate + needs: [] + timeout: "5 minutes" + script: + # Replace code commited to repo with code downloaded from ETSI + - ./scripts/lc3plus_lib_setup/get_lc3plus.sh + + # Ensure git reports no changes + - modified_files=$(git status -su lib_lc3plus) + - if [[ $modified_files ]]; then printf 'LC3plus codebase was modified!\n\n'"$modified_files"'\n\n'; exit $EXIT_CODE_FAIL; fi + +check-bitexactness-hrtf-rom-and-file: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-cmake"] + timeout: "5 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - cmake . + - make -j + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html + artifacts: + paths: + - report.html + - report-junit.xml + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading" + expose_as: "logs-hrtf-loading" + expire_in: "5 days" + +check-bitexactness-ext-and-transport-format: + extends: + - .test-job-linux + - .rules-merge-request-to-main + stage: test + needs: ["build-codec-linux-cmake"] + timeout: "5 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - cmake . + - make -j + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest tests/test_be_for_ext_outputs.py --html=report.html --junit-xml=report-junit.xml --self-contained-html + artifacts: + paths: + - report.html + - report-junit.xml + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--ext-sanity-check" + expose_as: "logs-ext-sanity-check" + expire_in: "5 days" + + +# --------------------------------------------------------------- +# Test jobs for main branch +# --------------------------------------------------------------- + +# check bitexactness to EVS windows binaries +be-2-evs-windows: + extends: + - .rules-merge-request-to-main + tags: + - ivas-windows + stage: test + needs: ["build-codec-windows-msbuild"] + timeout: "20 minutes" # To be revisited + script: + - *print-common-info-windows + + - $winoutdata = $null + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata + - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 + + # copy over to never change the testvector dir + - cp -r $EVS_BE_WIN_TEST_DIR ./evs_be_win_test + - cp IVAS_cod.exe ./evs_be_win_test/bin/IVAS_cod.exe + - cp IVAS_dec.exe ./evs_be_win_test/bin/IVAS_dec.exe + + - cd evs_be_win_test + - python ../ci/run_evs_be_win_test.py + +# check bitexactness to EVS +be-2-evs-linux: + extends: + - .test-job-linux + - .rules-main-push + tags: + - be-2-evs-temp + stage: test + needs: ["build-codec-linux-cmake"] + timeout: "20 minutes" # To be revisited + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + + - mkdir build + - cd build + - cmake .. + - make -j + - cd .. + + # copy over to never change the testvector dir + - cp -r $EVS_BE_TEST_DIR ./evs_be_test + - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod + - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec + + - cd evs_be_test + - python3 ../ci/run_evs_be_test.py + +codec-comparison-on-main-push: + extends: + - .test-job-linux-needs-testv-dir + - .rules-main-push + stage: compare + needs: ["build-codec-linux-cmake"] + timeout: "30 minutes" # To be revisited + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - latest_commit=$(git rev-parse HEAD) # Latest commit + - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) + - echo "Comparing changes from $previous_merge_commit to $latest_commit" + - git --no-pager diff --stat $previous_merge_commit..$latest_commit + + # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. + + ### build test binaries, initial clean for paranoia reasons + - make clean + - mkdir build + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_test + - mv IVAS_dec ../IVAS_dec_test + - cd .. + - rm -rf build/* + + ### compare to the previous merge commit in the main branch + - git fetch origin main + - git checkout $previous_merge_commit + - echo "Building reference codec at commit $previous_merge_commit" + + ### build reference binaries + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_ref + - mv IVAS_dec ../IVAS_dec_ref + - cd .. + + # helper variable - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_COMMIT_MESSAGE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + ### re-checkout the latest commit in the main branch, if ref_using_main is not set + - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi + + ### prepare pytest + # rename test binaries back + - mv IVAS_cod_test IVAS_cod + - mv IVAS_dec_test IVAS_dec + # create references + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout + + ### re-checkout the latest commit here, if ref_using_main is set + - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi + + ### run pytest + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi + - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; + allow_failure: + exit_codes: + - 123 + artifacts: + name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "Results of comparison to previous merge commit" + reports: + junit: report-junit.xml + + +# --------------------------------------------------------------- +# Manual jobs +# --------------------------------------------------------------- + +test-be-to-release: + stage: test + tags: + - ivas-windows + resource_group: ivas-be-to-release-test-resource + timeout: "20 minutes" + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' + script: + + - echo "$CI_COMMIT_BRANCH" + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata + + # path to release candidate refs defined in config.toml + + - echo "Placeholder for BE test to release script" + - mkdir logs + + artifacts: + name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 1 week + paths: + - logs/ + expose_as: "test-be-to-release results" + + +ivas-conformance: + tags: + - ivas-windows + stage: test + timeout: "90 minutes" + rules: + - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + allow_failure: + exit_codes: + - 123 + script: + - *print-common-info-windows + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug + - cp -force IVAS_cod.exe IVAS_cod_ref.exe + - cp -force IVAS_dec.exe IVAS_dec_ref.exe + - cp -force IVAS_rend.exe IVAS_rend_ref.exe + - cp -force ISAR_post_rend.exe ISAR_post_rend_ref.exe + + # Reference creation + - python scripts/prepare_combined_format_inputs.py + - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" + - python -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files + + # Output creation + - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt + + # Copy input data and output ref data + - if (Test-Path testvec) {rm -r -force testvec} + - if (Test-Path TMP_DEC) {rm -r -force TMP_DEC} + - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} + - if (Test-Path TMP_JBM) {rm -r -force TMP_JBM} + - if (Test-Path TMP_REND) {rm -r -force TMP_REND} + - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} + - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} + - mkdir testvec + - mkdir testvec/binauralRenderer_interface + - mkdir testvec/testv + - mkdir testvec/testv/renderer + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin + - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv + - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec + - cp -r -force -ErrorAction Ignore scripts/object_edit testvec + - cp -r -force -ErrorAction Ignore scripts/switchPaths testvec + - cp -r -force -ErrorAction Ignore scripts/trajectories testvec + - cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface + - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref + - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref + - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files + - python scripts/cleanup_26252.py + + # Copy test script files + - cp -r -force -ErrorAction Ignore tests/conformance-test testvec/ + - cp Readme_IVAS_dec.txt testvec + - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_rend.txt testvec + - cp Readme_IVAS_JBM_dec.txt testvec + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec + - cp IVAS_cod.exe testvec/bin + - cp IVAS_dec.exe testvec/bin + - cp IVAS_rend.exe testvec/bin + - cp ISAR_post_rend.exe testvec/bin + + + # Test run generated scripts in testvec + - cd testvec + - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html + - mv report.html .. + - mv report-junit.xml .. + + artifacts: + name: "ivas-conformance-$CI_COMMIT_SHORT_SHA" + expire_in: 1 week + when: always + paths: + - report_cmd.html + - report-junit.xml + - report.html + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt + - Readme_IVAS_rend.txt + - Readme_IVAS_JBM_dec.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt + expose_as: "Draft IVAS conformance" + reports: + junit: report-junit.xml + +ivas-conformance-linux: + extends: + - .job-linux + stage: test + timeout: "90 minutes" + rules: + - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + allow_failure: + exit_codes: + - 123 + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - make -j + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + - cp IVAS_rend IVAS_rend_ref + - cp ISAR_post_rend ISAR_post_rend_ref + + # Reference creation + - python3 scripts/prepare_combined_format_inputs.py + - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" + - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files + + # Output creation + - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt + + # Copy input data and output ref data + - rm -rf testvec + - mkdir testvec + - mkdir testvec/binauralRenderer_interface + - mkdir testvec/testv + - mkdir testvec/testv/renderer + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin + - cp -r scripts/testv/* testvec/testv + - cp -r scripts/ls_layouts testvec + - cp -r scripts/object_edit testvec + - cp -r scripts/switchPaths testvec + - cp -r scripts/trajectories testvec + - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface + - cp -r tests/ref testvec/testv/ref + - cp -r tests/dut/* testvec/testv/ref + - cp -r tests/renderer/cut testvec/testv/renderer/ref + - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref + - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files + - python3 scripts/cleanup_26252.py + + # Copy test script files + - cp -r tests/conformance-test testvec/ + - cp Readme_IVAS_dec.txt testvec + - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_rend.txt testvec + - cp Readme_IVAS_JBM_dec.txt testvec + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec + + # Create GCOV execs for coverage analysis + - make clean + - make GCOV=1 -j + + - cp IVAS_cod testvec/bin + - cp IVAS_dec testvec/bin + - cp IVAS_rend testvec/bin + - cp ISAR_post_rend testvec/bin + + # Test run generated scripts in testvec + - cd testvec + - exit_code=0 + - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? + - mv report.html .. + - mv report-junit.xml .. + + # Collect coverage + - cd - + - lcov -c -d obj -o coverage.info + - lcov -r coverage.info "*apps*" -o coverage.info + - lcov -r coverage.info "*lib_util*" -o coverage.info + - commit_sha=$(git rev-parse HEAD) + - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" + + # Check for failures + - if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi + + artifacts: + name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" + expire_in: 1 week + when: always + paths: + - report_cmd.html + - report-junit.xml + - report.html + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt + - Readme_IVAS_rend.txt + - Readme_IVAS_JBM_dec.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt + - coverage.info + - coverage + expose_as: "Draft IVAS conformance -- Linux" + reports: + junit: report-junit.xml + + +test-long-self-test: + extends: + - .job-linux + stage: compare + resource_group: ivas-long-self-test-resource + timeout: "50 minutes" + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' + allow_failure: + exit_codes: + - 123 + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + + ### build branch binaries + - cmake . -Bbuild-test + - cmake --build build-test -- -j + - mv build-test/IVAS_cod ./IVAS_cod + - mv build-test/IVAS_dec ./IVAS_dec + + ### store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### switch to main + - git checkout main + - git pull origin main + - echo "Building reference codec at commit $(git rev-parse HEAD)" + + ### build main (ref) binaries + - cmake . -Bbuild-ref + - cmake --build build-ref -- -j + - mv build-ref/IVAS_cod ./IVAS_cod_ref + - mv build-ref/IVAS_dec ./IVAS_dec_ref + + ### Switch back to branch, this could probably be removed later + - git checkout $source_branch_commit_sha + + ### prepare pytest + + # Copy test vectors from LTV to TESTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + + # create references + - exit_code_ref=0 + - testcase_timeout=300 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? + + ### run pytest self-test using long test vectors + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + + artifacts: + name: "test-long-self-test--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 4 mos + paths: + - report-junit-ltv.xml + - report-ltv.html + expose_as: "test-long-self-test results" + reports: + junit: + - report-junit-ltv.xml + +check-clipping: + extends: + - .job-linux + stage: test + timeout: "30 minutes" + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' + allow_failure: + exit_codes: + - 123 + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/enable-debugging-macro.sh + - make -j + - tests/scale_pcm.py ./scripts/testv/ $SCALE_FACTOR # Default: 3.162 (+10 dB). Can be set in manual trigger. + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 scripts/parse_xml_report.py report-junit.xml report.csv --clipping + artifacts: + name: "check-clipping--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 2 weeks + paths: + - report-junit.xml + - report.html + - report.csv + expose_as: "check-clipping results" + reports: + junit: + - report-junit.xml + +test-branch-vs-input-passthrough: + extends: + - .job-linux + stage: compare + timeout: "30 minutes" # TBD + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + allow_failure: + exit_codes: + - 123 + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - make -j + + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py report-junit.xml report.csv + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ]; then echo "Differences encountered."; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + artifacts: + name: "test-branch-vs-input-passthrough--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 4 mos + paths: + - report-junit.xml + - report.html + - report.csv + expose_as: "test-branch-vs-input-passthrough results" + reports: + junit: + - report-junit.xml + +# --------------------------------------------------------------- +# Scheduled jobs on main +# --------------------------------------------------------------- + +# code selftest long testvectors with memory-sanitizer binaries +ltv-msan: + extends: + - .sanitizer-selftest-ltv + - .sanitizer-selftest-anchor + rules: + - if: $SANITIZER_SCHEDULE_E + timeout: 5 hours + tags: + - ivas-linux-fast + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=1 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" + - USE_LTV="--use_ltv" + +# code selftest long testvectors with address-sanitizer binaries +ltv-asan: + extends: + - .sanitizer-selftest-ltv + - .sanitizer-selftest-anchor + rules: + - if: $SANITIZER_SCHEDULE_E + when: delayed + start_in: 5 hours + tags: + - ivas-linux-fast + timeout: 5 hours + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=2 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" + - USE_LTV="--use_ltv" + +# code selftest long testvectors with undefined-behaviour-sanitizer binaries +ltv-usan: + extends: + - .sanitizer-selftest-ltv + - .sanitizer-selftest-anchor + rules: + - if: $SANITIZER_SCHEDULE_E + when: delayed + start_in: 8 hours + tags: + - ivas-linux-fast + timeout: 5 hours + before_script: + - !reference [.job-linux, before_script] + - CLANG_NUM=3 + - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" + - USE_LTV="--use_ltv" + +.sanitizer-test-template: + extends: + - .test-job-linux + stage: test + tags: + - sanitizer_test_main + - ivas-linux-fast + artifacts: + name: "$CI_JOB_NAME--main--sha-$CI_COMMIT_SHORT_SHA" + expire_in: 7 weeks + when: always + paths: + - ep_015.g192 + - dly_profile.dat + - head_rot_traj.csv + - exof_traj.csv + - ./LOGS_PLC + - ./LOGS_noPLC + +### --- sanitizer schedule A --- + +.sanitizer-test-schedule-A: + extends: + - .sanitizer-test-template + +sanitizer-test-mono: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + timeout: 1.25 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS + +sanitizer-test-stereo: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 1.25 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED EXT --tests $SANITIZER_TESTS + +sanitizer-test-stereodmxevs: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 3.75 hours + timeout: 1.25 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py StereoDmxEVS mono --tests $SANITIZER_TESTS + +sanitizer-test-mc-5_1: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 5 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-mc-5_1_2: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 7.5 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-mc-5_1_4: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 10 hours + timeout: 3.75 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-mc-7_1: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 13.75 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-mc-7_1_4: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 16.25 hours + timeout: 5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-ism+1: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 21.25 hours + timeout: 1.25 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-ism+2: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 22.5 hours + timeout: 2.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-ism+3: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 25 hours + timeout: 3.75 hour + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-ism+4: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 28.75 hours + timeout: 5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-masa: + extends: .sanitizer-test-schedule-A + rules: + - if: $SANITIZER_SCHEDULE_A + when: delayed + start_in: 33.75 hours + timeout: 10 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +### --- sanitizer schedule B --- + +.sanitizer-test-schedule-B: + extends: + - .sanitizer-test-template + timeout: 7.5 hours + +sanitizer-test-foa: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-hoa2: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 7.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-hoa3: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 15 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-foa-ism1: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 22.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-foa-ism2: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 30 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-foa-ism3: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 37.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-foa-ism4: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 45 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa2-ism1: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 52.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa2-ism2: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 60 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa2-ism3: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 67.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa2-ism4: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 75 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa3-ism1: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 82.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa3-ism2: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 90 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-omasa-ism4: + extends: .sanitizer-test-schedule-B + rules: + - if: $SANITIZER_SCHEDULE_B + when: delayed + start_in: 97.5 hours + timeout: 10 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + + +### --- sanitizer schedule C --- + +.sanitizer-test-schedule-C: + extends: + - .sanitizer-test-template + timeout: 10 hours + +sanitizer-test-omasa-ism1: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-omasa-ism2: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + when: delayed + start_in: 10 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-omasa-ism3: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + when: delayed + start_in: 20 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa3-ism3: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + when: delayed + start_in: 30 hours + timeout: 7.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-hoa3-ism4: + extends: .sanitizer-test-schedule-C + rules: + - if: $SANITIZER_SCHEDULE_C + when: delayed + start_in: 37.5 hours + timeout: 7.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +### --- sanitizer schedule D --- + +.sanitizer-test-schedule-D: + extends: + - .sanitizer-test-template + timeout: 7.5 hours + +sanitizer-test-planar-foa: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-planar-hoa2: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 7.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-planar-hoa3: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 15 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-foa-ism1: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 22.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-foa-ism2: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 30 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-foa-ism3: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 37.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-foa-ism4: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 45 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa2-ism1: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 52.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa2-ism2: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 60 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa2-ism3: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 67.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa2-ism4: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 75 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa3-ism1: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 82.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa3-ism2: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 90 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa3-ism3: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 97.5 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + +sanitizer-test-osba-planar-hoa3-ism4: + extends: .sanitizer-test-schedule-D + rules: + - if: $SANITIZER_SCHEDULE_D + when: delayed + start_in: 105 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS + + + +# GCOV/LCOV coverage analysis of self_test suite +coverage-test-on-main-scheduled: + extends: + - .test-job-linux-needs-testv-dir + - .rules-main-scheduled + tags: + - ivas-linux-fast + stage: test + rules: + # only run in scheduled pipeline that passes this env vars + - if: $COVERAGE_TEST + timeout: 3 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - make GCOV=1 -j + - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script + + # -- Run short test vector suite to check coverage of that first + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true + - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref + - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut + - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here + # remove apps and lib_util files from coverage + - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info + - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info + + # -- Add extended coverage of EVS test and long test vectors + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - bash ci/smoke_test.sh coverage + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true + - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py + - lcov -c -d obj -o coverage.info + # remove apps and lib_util files from coverage + - lcov -r coverage.info "*apps*" -o coverage.info + - lcov -r coverage.info "*lib_util*" -o coverage.info + - commit_sha=$(git rev-parse HEAD) + - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" + - genhtml coverage_stv.info -o coverage_stv -t "Coverage on main -- short test vectors @ $commit_sha" + artifacts: + name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week + paths: + - coverage.info + - coverage_stv.info + - coverage + - coverage_stv + +# --------------------------------------------------------------- +# Complexity measurement jobs +# --------------------------------------------------------------- + +.complexity-measurements-setup: + &complexity-measurements-setup # create necessary environment + - mkdir -p wmops/logs + + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) + - echo $job_id + - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip + - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html + - public_dir="$CI_JOB_NAME-public" + + # if is needed to catch case when no artifact is there (first run), similarly as above + # 1. check for public_dir being there as this might not be the case when artifact download failed + # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts + - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then + - mv $public_dir/* wmops/ + # check here if we have the split-by-levels files present - if not, fake them up with the existing global one + # this is needed for the first run with split graphs on a branch where the global version did run previously + # NOTE: checking only for level_1 file here as this should already be sufficient + # NOTE2: also not chechking for RAM for same reason + - wmops_all_global="wmops/log_wmops_all.txt" + - ram_all_global="wmops/log_ram_all.txt" + - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then + - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") + - for suffix in "${suffixes[@]}"; do + - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt + - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt + - done + - fi + - fi + + - rm artifacts.zip + - rm -rf $public_dir + +.complexity-measurements-prepare-artifacts: + &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory + - public_dir="$CI_JOB_NAME-public" + - mkdir $public_dir + - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/ + # move logfiles for links + - mkdir $public_dir/logs + # first move logs + - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g") + - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done + # copy index page blueprint + - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html + # patch the format in the title + - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html + # do separately here to avoid overwrite complaints by mv + - mv -f ci/complexity_measurements/style.css ${public_dir}/ + +.complexity-template: + extends: + # still needed as long as no long MASA testvectors are there + - .test-job-linux-needs-testv-dir + tags: + - ivas-linux-fast + timeout: 3 hours 30 minutes + stage: test + before_script: + - !reference [.job-linux, before_script] + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - *complexity-measurements-setup + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA" + when: always + expire_in: 1 week + paths: + - $CI_JOB_NAME-public + +complexity-stereo-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + script: + - in_format=stereo + - out_format=ext + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-ism-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 1 hour + script: + - in_format=ISM + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-ism-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 2 hours + script: + - in_format=ISM + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-ism-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 3 hours + script: + - in_format=ISM + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-sba-hoa3-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 4 hours + script: + - in_format=HOA3 + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-sba-hoa3-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 5 hours + script: + - in_format=HOA3 + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-sba-hoa3-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 6 hours + script: + - in_format=HOA3 + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-mc-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 7 hours + script: + - in_format=MC + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-mc-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 8 hours + script: + - in_format=MC + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-mc-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 9 hours + script: + - in_format=MC + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 10 hours + script: + - in_format=MASA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 11 hours + script: + - in_format=MASA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-hoa3-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 12 hours + script: + - in_format=MASA + - out_format=HOA3 + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 13 hours + script: + - in_format=OMASA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 14 hours + script: + - in_format=OMASA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-hoa3-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 15 hours + script: + - in_format=OMASA + - out_format=HOA3 + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-StereoDmxEVS-stereo-in-mono-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 16 hours + script: + - in_format=StereoDmxEVS + - out_format=mono + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-osba-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 17 hours + script: + - in_format=OSBA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-osba-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 18 hours + script: + - in_format=OSBA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-osba-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 19 hours + script: + - in_format=OSBA + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +# --------------------------------------------------------------- +# Other jobs +# --------------------------------------------------------------- + +# job that sets up gitlab pages website +pages: + stage: deploy + extends: + - .job-linux + rules: + - if: $UPDATE_PAGES + script: + - python3 ci/setup_pages.py + artifacts: + paths: + - public + expire_in: 1 day + +# Pull state of a branch on 3GPP repo, push to a mirror repo. +pull-from-3gpp-forge: + stage: maintenance + rules: + - if: $MIRROR_ACCESS_TOKEN # Only run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) + script: + # Set up git LFS for mirroring (see: https://github.com/git-lfs/git-lfs/issues/1762) + - git lfs install --skip-smudge --local + + # Check out mirror branch - by default the runner checks out by commit hash, which results in detached head state + - git checkout $CI_COMMIT_BRANCH + + # Pull commits from upstream + - git remote add upstream https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec.git + - git pull --ff-only --tags upstream $MIRROR_SOURCE_BRANCH + - git lfs pull upstream + + # Push to mirror, include tags. Option `-o ci.skip` tells GitLab to skip CI for the pushed commits (assumed already tested upstream) + - git push --tags -o ci.skip "https://${GITLAB_USER_LOGIN}:${MIRROR_ACCESS_TOKEN}@${CI_REPOSITORY_URL#*@}" "HEAD:${CI_COMMIT_BRANCH}" diff --git a/main.yml b/main.yml index df75d2c..9062a0c 100644 --- a/main.yml +++ b/main.yml @@ -7,14 +7,3 @@ include: default: # by default, we want all jobs to be interruptible, and we need to configure this explicitly interruptible: true - - # Note: currently only used for float -stages: - - .pre - - maintenance - - prevalidate - - build - - test - - compare - - postvalidate - - deploy -- GitLab From 33c29b155b7ab096ba1ae3a9b7ae593a14547983 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 Sep 2025 16:00:40 +0200 Subject: [PATCH 2/5] add default variables for float --- includes/default-variables-float.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/includes/default-variables-float.yml b/includes/default-variables-float.yml index e69de29..50f6f40 100644 --- a/includes/default-variables-float.yml +++ b/includes/default-variables-float.yml @@ -0,0 +1,28 @@ +variables: + EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test" + EVS_BE_WIN_TEST_DIR: "C:/Users/gitlab-runner/testvec" + PROCESSING_SCRIPTS_BIN_DIR: "/test-bin" + SANITIZER_TESTS: "CLANG1 CLANG2 CLANG3" + OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4" + OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3" + OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB" + OUT_FORMATS_ALL: "$OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT" + MANUAL_PIPELINE_TYPE: + description: "Type for the manual pipeline run. Use 'test-be-release' to run BE test against release codec." + value: 'default' + options: + - 'default' + - 'test-be-release' + - 'test-long-self-test' + - 'ivas-conformance' + - 'ivas-conformance-linux' + - 'check-clipping' + - 'test-branch-vs-input-passthrough' + + PYTEST_ARGS: "" + LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" + LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" + LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" + TEST_SUITE: "" -- GitLab From 5347ccdd77c7d5a53fc710262ff8353e83d2b8f4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Nov 2025 16:48:27 +0100 Subject: [PATCH 3/5] update from current gitlab-ci.yml --- main-float.yml | 699 ++++++++++++++++++++++++++++--------------------- 1 file changed, 399 insertions(+), 300 deletions(-) diff --git a/main-float.yml b/main-float.yml index a035c6d..93d6597 100644 --- a/main-float.yml +++ b/main-float.yml @@ -15,43 +15,46 @@ stages: # This sets when pipelines are created. Jobs have more specific rules to restrict them. workflow: - name: '$IVAS_PIPELINE_NAME' + name: "$IVAS_PIPELINE_NAME" rules: # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' variables: - IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + IVAS_PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main variables: - IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Push pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main variables: - IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Scheduled pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing variables: - IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Web run pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' variables: - IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Test BE to release pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: - IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Test long self-test against main pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' variables: - IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Draft IVAS Conformance test: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'trigger' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' variables: - IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'trigger' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' variables: - IVAS_PIPELINE_NAME: 'Check core input clipping: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Check core input clipping: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' variables: - IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Pass-through comparison vs input: $CI_COMMIT_BRANCH" + - if: $MANUAL_PIPELINE_TYPE == 'coverage' + variables: + IVAS_PIPELINE_NAME: "Coverage measurement" # --------------------------------------------------------------- # Generic script anchors @@ -66,15 +69,6 @@ workflow: echo "Commit time was $CI_COMMIT_TIMESTAMP" ("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") | Invoke-Expression -.get-basop-float-reference: &get-basop-float-reference - - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch --depth 1 ivas-basop - - cd ivas-basop - - git status - - make -j - - cd - - - cp ivas-basop/IVAS_cod ./ - - cp ivas-basop/IVAS_dec ./ - .merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh @@ -129,9 +123,9 @@ workflow: - Pop-Location .copy-ltv-files-to-testv-dir-win: ©-ltv-files-to-testv-dir-win - - cp $LTV_DIR_WIN\*.wav scripts\testv - - cp $LTV_DIR_WIN\*.met scripts\testv - - cp $LTV_DIR_WIN\*.csv scripts\testv + - cp $LTV_DIR_WIN\*.wav scripts\testv + - cp $LTV_DIR_WIN\*.met scripts\testv + - cp $LTV_DIR_WIN\*.csv scripts\testv .activate-WX-windows: &activate-WX-windows - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" @@ -146,7 +140,7 @@ workflow: - make clean - make -j CLANG=$CLANG_NUM - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1 - exit_code20=0 - exit_code10=0 @@ -164,7 +158,6 @@ workflow: - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi - # --------------------------------------------------------------- # .pre jobs for setting up things # --------------------------------------------------------------- @@ -238,24 +231,42 @@ branch-is-up-to-date-with-main-post: exit 1 fi +### BASOP compatibility test jobs +### these jobs run the tests with the BASOP executables to check if added testcases work fine both with float and BASOP codec +### run on manual trigger only to not prolong MR pipelines too much .basop-ci-branch-compat-template: extends: - .test-job-linux - - .rules-merge-request-to-basop-ci-branch + stage: basop-compat + # allows for pipelines to pass when these were not started + allow_failure: true + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + when: manual + - if: $CI_PIPELINE_SOURCE == 'push' + when: never tags: - - ivas-linux + - ivas-linux-fast before_script: - !reference [.job-linux, before_script] - - python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm - - python3 tests/create_short_testvectors.py - python3 scripts/prepare_combined_format_inputs.py - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/build-binaries-from-basop-repo.sh main + + # long timeout for BASOP executables + - testcase_timeout=$TESTCASE_TIMEOUT_LTV_SANITIZERS - - *get-basop-float-reference + # hack for renderer tests to match the expected naming + - mv IVAS_rend IVAS_rend_ref - - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT + # BASOP renderer test uses cut input signals + - | + if [ "$TEST_SUITE" == "tests/renderer_short/test_renderer.py" ]; then + python3 tests/create_short_testvectors --inplace --all --cut_len 5.0 + fi + - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --create_ref --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" @@ -265,62 +276,56 @@ branch-is-up-to-date-with-main-post: - report-junit.xml - report.html - expose_as: "check-float-reference results" + expose_as: "basop-compat results" reports: junit: - report-junit.xml -basop-ref-compat-stv: +### The stv jobs are run for both main and ivas-float-update +basop-compat-stv: extends: - .basop-ci-branch-compat-template variables: - TEST_SUITE: "$SHORT_TEST_SUITE tests/renderer" TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS - script: - - exit 0 + parallel: + matrix: + - TEST_SUITE: + - tests/codec_be_on_mr_nonselection/test_param_file.py + - tests/renderer_short/test_renderer.py -basop-ref-compat-ltv: - extends: - - .basop-ci-branch-compat-template - tags: - - ivas-linux-fast - variables: - TEST_SUITE: $LONG_TEST_SUITE - TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 -basop-ref-compat-encoder-stv: +basop-compat-encoder-dmx-comp: extends: - .basop-ci-branch-compat-template variables: TEST_SUITE: $SHORT_TEST_SUITE_ENCODER + # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option + PYTEST_ARGS: "--compare_enc_dmx" TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS script: - exit 0 -basop-ref-compat-encoder-ltv: +basop-compat-ltv: extends: - .basop-ci-branch-compat-template - tags: - - ivas-linux-fast variables: - TEST_SUITE: $LONG_TEST_SUITE_ENCODER + TEST_SUITE: $LONG_TEST_SUITE TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 -basop-ref-compat-encoder-dmx-comp: +basop-compat-encoder-ltv: extends: - .basop-ci-branch-compat-template variables: - TEST_SUITE: $SHORT_TEST_SUITE_ENCODER - # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option - PYTEST_ARGS: "--compare_enc_dmx" - TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS + TEST_SUITE: $LONG_TEST_SUITE_ENCODER + TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS script: - exit 0 + # --------------------------------------------------------------- # Build jobs # --------------------------------------------------------------- @@ -363,7 +368,6 @@ build-codec-instrumented-linux: paths: - wmc_tool_output.txt - # make sure that the codec builds with msan, asan and usan build-codec-sanitizers-linux: extends: @@ -401,22 +405,28 @@ build-codec-windows-msbuild: codec-smoke-test: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request-to-basop-ci-branch-or-main + - .rules-merge-request-to-main timeout: "20 minutes" tags: - ivas-linux-fast stage: test - needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + needs: + [ + "build-codec-linux-cmake", + "build-codec-linux-make", + "build-codec-instrumented-linux", + "build-codec-sanitizers-linux", + ] script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh # LTV update needed as ltv ISM metadata files are used - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - # for MRs to basop-ci-branch, we want to test with the BASOP float reference build + # for MRs to basop-ci-branch-pc, we want to test with the BASOP float reference build # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros + # TODO: make manual for basop-CI merges - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then - - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline - - *get-basop-float-reference + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/build-binaries-from-basop-repo.sh ivas-float-update - bash ci/smoke_test.sh coverage - else - bash ci/smoke_test.sh @@ -485,7 +495,13 @@ pytest-compare-20ms-and-5ms-rendering: - .test-job-linux - .rules-merge-request-to-main stage: test - needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + needs: + [ + "build-codec-linux-cmake", + "build-codec-linux-make", + "build-codec-instrumented-linux", + "build-codec-sanitizers-linux", + ] script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh @@ -568,12 +584,6 @@ renderer-smoke-test: before_script: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - # rename files to fit naming convention - # en- and decoder needed because of split rendering testcases - - mv IVAS_cod IVAS_cod_ref - - mv IVAS_dec IVAS_dec_ref - - mv IVAS_rend IVAS_rend_ref - - mv ISAR_post_rend ISAR_post_rend_ref - testcase_timeout=180 # test renderer executable with cmake + asan @@ -583,11 +593,10 @@ renderer-asan: variables: SANITIZER_BUILD_STRING: "asan" script: - - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: expose_as: "renderer asan result" - # test renderer executable with cmake + msan renderer-msan: extends: @@ -595,11 +604,10 @@ renderer-msan: variables: SANITIZER_BUILD_STRING: "msan" script: - - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: expose_as: "renderer msan result" - # test renderer executable with cmake + usan renderer-usan: extends: @@ -607,11 +615,10 @@ renderer-usan: variables: SANITIZER_BUILD_STRING: "usan" script: - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: expose_as: "renderer usan result" - # compare renderer bitexactness between target and source branch renderer-pytest-on-merge-request: extends: @@ -707,6 +714,29 @@ split-rendering-voip-be-to-binaural: junit: - report-junit.xml +# test rtpdump +rtpdump-test: + extends: + - .test-job-linux + - .rules-merge-request-to-main + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j + - testcase_timeout=15 + - python3 -m pytest -q --tb=no -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/rtp/test_rtp.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "rtpdump pytest results" + reports: + junit: + - report-junit.xml + lc3-wrapper-unit-test: extends: - .test-job-linux @@ -850,7 +880,6 @@ ivas-pytest-on-merge-request: junit: - report-junit.xml - # Check interop IVAS_cod_test -> IVAS_dec_ref ivas-interop-on-merge-request: extends: @@ -1097,7 +1126,6 @@ check-bitexactness-ext-and-transport-format: expose_as: "logs-ext-sanity-check" expire_in: "5 days" - # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- @@ -1126,114 +1154,115 @@ be-2-evs-windows: - cd evs_be_win_test - python ../ci/run_evs_be_win_test.py +# TODO: turn into manual job if needed # check bitexactness to EVS -be-2-evs-linux: - extends: - - .test-job-linux - - .rules-main-push - tags: - - be-2-evs-temp - stage: test - needs: ["build-codec-linux-cmake"] - timeout: "20 minutes" # To be revisited - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - - mkdir build - - cd build - - cmake .. - - make -j - - cd .. - - # copy over to never change the testvector dir - - cp -r $EVS_BE_TEST_DIR ./evs_be_test - - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod - - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec - - - cd evs_be_test - - python3 ../ci/run_evs_be_test.py - -codec-comparison-on-main-push: - extends: - - .test-job-linux-needs-testv-dir - - .rules-main-push - stage: compare - needs: ["build-codec-linux-cmake"] - timeout: "30 minutes" # To be revisited - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - latest_commit=$(git rev-parse HEAD) # Latest commit - - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) - - echo "Comparing changes from $previous_merge_commit to $latest_commit" - - git --no-pager diff --stat $previous_merge_commit..$latest_commit - - # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. - - ### build test binaries, initial clean for paranoia reasons - - make clean - - mkdir build - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_test - - mv IVAS_dec ../IVAS_dec_test - - cd .. - - rm -rf build/* - - ### compare to the previous merge commit in the main branch - - git fetch origin main - - git checkout $previous_merge_commit - - echo "Building reference codec at commit $previous_merge_commit" - - ### build reference binaries - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_ref - - mv IVAS_dec ../IVAS_dec_ref - - cd .. - - # helper variable - "|| true" to prevent failures from grep not finding anything - # write to temporary file as workaround for failures observed with piping echo - - echo $CI_COMMIT_MESSAGE > tmp.txt - - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true - - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true - - ### re-checkout the latest commit in the main branch, if ref_using_main is not set - - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi - - ### prepare pytest - # rename test binaries back - - mv IVAS_cod_test IVAS_cod - - mv IVAS_dec_test IVAS_dec - # create references - - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout - - ### re-checkout the latest commit here, if ref_using_main is set - - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi - - ### run pytest - - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi - - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; - allow_failure: - exit_codes: - - 123 - artifacts: - name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "Results of comparison to previous merge commit" - reports: - junit: report-junit.xml - +# be-2-evs-linux: +# extends: +# - .test-job-linux +# - .rules-main-push +# tags: +# - be-2-evs-temp +# stage: test +# needs: ["build-codec-linux-cmake"] +# timeout: "20 minutes" # To be revisited +# script: +# - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh +# +# - mkdir build +# - cd build +# - cmake .. +# - make -j +# - cd .. +# +# # copy over to never change the testvector dir +# - cp -r $EVS_BE_TEST_DIR ./evs_be_test +# - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod +# - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec +# +# - cd evs_be_test +# - python3 ../ci/run_evs_be_test.py + +# TODO: do we still need this? +# codec-comparison-on-main-push: +# extends: +# - .test-job-linux-needs-testv-dir +# - .rules-main-push +# stage: compare +# needs: ["build-codec-linux-cmake"] +# timeout: "30 minutes" # To be revisited +# script: +# - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh +# - latest_commit=$(git rev-parse HEAD) # Latest commit +# - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) +# - echo "Comparing changes from $previous_merge_commit to $latest_commit" +# - git --no-pager diff --stat $previous_merge_commit..$latest_commit +# +# # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. +# +# ### build test binaries, initial clean for paranoia reasons +# - make clean +# - mkdir build +# - cd build +# - cmake .. +# - make -j +# - mv IVAS_cod ../IVAS_cod_test +# - mv IVAS_dec ../IVAS_dec_test +# - cd .. +# - rm -rf build/* +# +# ### compare to the previous merge commit in the main branch +# - git fetch origin main +# - git checkout $previous_merge_commit +# - echo "Building reference codec at commit $previous_merge_commit" +# +# ### build reference binaries +# - cd build +# - cmake .. +# - make -j +# - mv IVAS_cod ../IVAS_cod_ref +# - mv IVAS_dec ../IVAS_dec_ref +# - cd .. +# +# # helper variable - "|| true" to prevent failures from grep not finding anything +# # write to temporary file as workaround for failures observed with piping echo +# - echo $CI_COMMIT_MESSAGE > tmp.txt +# - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true +# - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true +# +# ### re-checkout the latest commit in the main branch, if ref_using_main is not set +# - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi +# +# ### prepare pytest +# # rename test binaries back +# - mv IVAS_cod_test IVAS_cod +# - mv IVAS_dec_test IVAS_dec +# # create references +# - testcase_timeout=60 +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout +# +# ### re-checkout the latest commit here, if ref_using_main is set +# - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi +# +# ### run pytest +# - exit_code=0 +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? +# - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi +# - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true +# - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi +# - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; +# allow_failure: +# exit_codes: +# - 123 +# artifacts: +# name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" +# expire_in: 1 week +# when: always +# paths: +# - report-junit.xml +# - report.html +# expose_as: "Results of comparison to previous merge commit" +# reports: +# junit: report-junit.xml # --------------------------------------------------------------- # Manual jobs @@ -1248,7 +1277,6 @@ test-be-to-release: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' script: - - echo "$CI_COMMIT_BRANCH" - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata @@ -1265,7 +1293,6 @@ test-be-to-release: - logs/ expose_as: "test-be-to-release results" - ivas-conformance: tags: - ivas-windows @@ -1278,36 +1305,33 @@ ivas-conformance: exit_codes: - 123 script: - - *print-common-info-windows - - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug + - *print-common-info-windows + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug - cp -force IVAS_cod.exe IVAS_cod_ref.exe - cp -force IVAS_dec.exe IVAS_dec_ref.exe - - cp -force IVAS_rend.exe IVAS_rend_ref.exe + - cp -force IVAS_rend.exe IVAS_rend_ref.exe - cp -force ISAR_post_rend.exe ISAR_post_rend_ref.exe - + # Reference creation - python scripts/prepare_combined_format_inputs.py - - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" - - python -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files - - # Output creation - - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer_short/test_renderer.py", "tests/split_rendering/test_split_rendering.py" + - python -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt - + # Copy input data and output ref data - if (Test-Path testvec) {rm -r -force testvec} - if (Test-Path TMP_DEC) {rm -r -force TMP_DEC} - - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} + - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} - if (Test-Path TMP_JBM) {rm -r -force TMP_JBM} - if (Test-Path TMP_REND) {rm -r -force TMP_REND} - - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} - - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} + - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} + - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_rendering - - mkdir testvec/bin + - mkdir testvec/testv/renderer_short + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec - cp -r -force -ErrorAction Ignore scripts/object_edit testvec @@ -1315,31 +1339,29 @@ ivas-conformance: - cp -r -force -ErrorAction Ignore scripts/trajectories testvec - cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref - - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns - - # Remove redundant files + - cp -r -force -ErrorAction Ignore tests/renderer_short/ref testvec/testv/renderer_short/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/ref testvec/testv/split_rendering/ref + - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files - python scripts/cleanup_26252.py # Copy test script files - cp -r -force -ErrorAction Ignore tests/conformance-test testvec/ - cp Readme_IVAS_dec.txt testvec - - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec - - cp Readme_IVAS_ISAR_dec.txt testvec - - cp Readme_IVAS_ISAR_post_rend.txt testvec + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec - cp IVAS_cod.exe testvec/bin - cp IVAS_dec.exe testvec/bin - cp IVAS_rend.exe testvec/bin - - cp ISAR_post_rend.exe testvec/bin - - + - cp ISAR_post_rend.exe testvec/bin + # Test run generated scripts in testvec - - cd testvec + - cd testvec - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html - mv report.html .. - mv report-junit.xml .. @@ -1352,12 +1374,12 @@ ivas-conformance: - report_cmd.html - report-junit.xml - report.html - - Readme_IVAS_dec.txt - - Readme_IVAS_enc.txt + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt - Readme_IVAS_rend.txt - Readme_IVAS_JBM_dec.txt - - Readme_IVAS_ISAR_dec.txt - - Readme_IVAS_ISAR_post_rend.txt + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt expose_as: "Draft IVAS conformance" reports: junit: report-junit.xml @@ -1369,9 +1391,15 @@ ivas-conformance-linux: timeout: "90 minutes" rules: - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + - if: $MANUAL_PIPELINE_TYPE == "coverage" + - if: $COVERAGE_TEST allow_failure: exit_codes: - 123 + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_CONFORMANCE" + COVERAGE_OUTPUT_DIR: "coverage_conformance" + COVERAGE_TITLE: "conformance test test_26252.py" script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - make -j @@ -1379,24 +1407,21 @@ ivas-conformance-linux: - cp IVAS_dec IVAS_dec_ref - cp IVAS_rend IVAS_rend_ref - cp ISAR_post_rend ISAR_post_rend_ref - + # Reference creation - python3 scripts/prepare_combined_format_inputs.py - - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files - - # Output creation - - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py" + - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt - + # Copy input data and output ref data - rm -rf testvec - mkdir testvec - mkdir testvec/binauralRenderer_interface - mkdir testvec/testv - - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_rendering - - mkdir testvec/bin + - mkdir testvec/testv/renderer_short + - mkdir testvec/testv/split_rendering + - mkdir testvec/bin - cp -r scripts/testv/* testvec/testv - cp -r scripts/ls_layouts testvec - cp -r scripts/object_edit testvec @@ -1404,51 +1429,46 @@ ivas-conformance-linux: - cp -r scripts/trajectories testvec - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - cp -r tests/ref testvec/testv/ref - - cp -r tests/dut/* testvec/testv/ref - - cp -r tests/renderer/cut testvec/testv/renderer/ref - - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref - - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns - - # Remove redundant files + - cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref + - cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref + - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs + - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns + + # Remove redundant files - python3 scripts/cleanup_26252.py - + # Copy test script files - cp -r tests/conformance-test testvec/ - cp Readme_IVAS_dec.txt testvec - - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec - - cp Readme_IVAS_ISAR_dec.txt testvec - - cp Readme_IVAS_ISAR_post_rend.txt testvec - + - cp Readme_IVAS_ISAR_dec.txt testvec + - cp Readme_IVAS_ISAR_post_rend.txt testvec + # Create GCOV execs for coverage analysis - make clean - make GCOV=1 -j - + - cp IVAS_cod testvec/bin - cp IVAS_dec testvec/bin - cp IVAS_rend testvec/bin - - cp ISAR_post_rend testvec/bin - + - cp ISAR_post_rend testvec/bin + # Test run generated scripts in testvec - - cd testvec - - exit_code=0 - - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? + - cd testvec + - exit_code=0 + - python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? - mv report.html .. - mv report-junit.xml .. # Collect coverage - cd - - - lcov -c -d obj -o coverage.info - - lcov -r coverage.info "*apps*" -o coverage.info - - lcov -r coverage.info "*lib_util*" -o coverage.info - - commit_sha=$(git rev-parse HEAD) - - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE" # Check for failures - if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi - + artifacts: name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" expire_in: 1 week @@ -1457,25 +1477,26 @@ ivas-conformance-linux: - report_cmd.html - report-junit.xml - report.html - - Readme_IVAS_dec.txt - - Readme_IVAS_enc.txt + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt - Readme_IVAS_rend.txt - Readme_IVAS_JBM_dec.txt - - Readme_IVAS_ISAR_dec.txt - - Readme_IVAS_ISAR_post_rend.txt - - coverage.info - - coverage + - Readme_IVAS_ISAR_dec.txt + - Readme_IVAS_ISAR_post_rend.txt + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR expose_as: "Draft IVAS conformance -- Linux" reports: junit: report-junit.xml - test-long-self-test: extends: - .job-linux stage: compare resource_group: ivas-long-self-test-resource timeout: "50 minutes" + tags: + - ivas-linux-fast rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' allow_failure: @@ -1527,7 +1548,6 @@ test-long-self-test: - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi - exit 0 - artifacts: name: "test-long-self-test--sha-$CI_COMMIT_SHORT_SHA--results" when: always @@ -1976,7 +1996,6 @@ sanitizer-test-omasa-ism4: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - ### --- sanitizer schedule C --- .sanitizer-test-schedule-C: @@ -2189,10 +2208,8 @@ sanitizer-test-osba-planar-hoa3-ism4: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - - # GCOV/LCOV coverage analysis of self_test suite -coverage-test-on-main-scheduled: +.coverage-test-template: extends: - .test-job-linux-needs-testv-dir - .rules-main-scheduled @@ -2200,53 +2217,135 @@ coverage-test-on-main-scheduled: - ivas-linux-fast stage: test rules: - # only run in scheduled pipeline that passes this env vars - if: $COVERAGE_TEST - timeout: 3 hours - script: + - if: $MANUAL_PIPELINE_TYPE == "coverage" + timeout: 6 hours + before_script: + - !reference [.test-job-linux-needs-testv-dir, before_script] + - set -e + - 'trap ''echo "Command failed at line $LINENO: $BASH_COMMAND"'' ERR' - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - make GCOV=1 -j - - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script + # compile with coverage enabled + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true -DGCOV=ON + - cmake --build cmake-build -- -j + # copy executables to ref to be able to run some --create_ref tests + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + - cp IVAS_rend IVAS_rend_ref + - cp ISAR_post_rend ISAR_post_rend_ref + after_script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir cmake-build --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE" + artifacts: + name: "main-$COVERAGE_OUTPUT_DIR-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week - # -- Run short test vector suite to check coverage of that first - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec +coverage-merge: + extends: + - .rules-main-scheduled + - .test-job-linux-needs-testv-dir + tags: + - ivas-linux-fast + stage: test + rules: + - if: $COVERAGE_TEST + - if: $MANUAL_PIPELINE_TYPE == "coverage" + needs: + - job: coverage-test-on-main-scheduled-stv + artifacts: true + - job: coverage-test-on-main-scheduled-ltv + artifacts: true + - job: ivas-conformance-linux + artifacts: true + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_MERGED" + COVERAGE_OUTPUT_DIR: "coverage-merged" + COVERAGE_TITLE: "merged total coverage" + script: + - echo "Merging coverage reports..." + - lcov -a $COVERAGE_OUTPUT_FILE_CONFORMANCE -a $COVERAGE_OUTPUT_FILE_STV -a $COVERAGE_OUTPUT_FILE_LTV -o "$COVERAGE_OUTPUT_FILE_MERGED" + - commit_sha=$(git rev-parse HEAD) + - genhtml $COVERAGE_OUTPUT_FILE_MERGED -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha" + artifacts: + name: "merged-coverage-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week + paths: + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR +coverage-test-on-main-scheduled-stv: + extends: .coverage-test-template + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_STV" + COVERAGE_OUTPUT_DIR: "coverage_stv" + COVERAGE_TITLE: "short test vectors" + script: + # -- Run short test vector suite to check coverage of that first + # codec smoke test + - bash ci/smoke_test.sh coverage + # default renderer framesize is 20ms + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut - - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here - # remove apps and lib_util files from coverage - - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info - - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true + # run renderer smoke test - needed for REFVEC tests + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py + # run renderer test only with REF creation stage - no CUT needed since executables are identical + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref + # run split rendering smoke test and REF creation + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref + # VoIP BE test + - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py + # split rendering VoIP BE test + - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py + # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted + # - bash ci/run-first-frame-is-sid-test.sh + # TODO consider HRTF binary loading test, or probably already covered by PRM files + # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading + # run BE vs EXT test + - python3 tests/create_short_testvectors.py --cut_len 1.0 + - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py + # run LC3plus wrapper unit test + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job + artifacts: + paths: + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR +coverage-test-on-main-scheduled-ltv: + extends: .coverage-test-template + variables: + COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_LTV" + COVERAGE_OUTPUT_DIR: "coverage_ltv" + COVERAGE_TITLE: "long test vectors" + script: # -- Add extended coverage of EVS test and long test vectors - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - bash ci/smoke_test.sh coverage - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py - - lcov -c -d obj -o coverage.info - # remove apps and lib_util files from coverage - - lcov -r coverage.info "*apps*" -o coverage.info - - lcov -r coverage.info "*lib_util*" -o coverage.info - - commit_sha=$(git rev-parse HEAD) - - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" - - genhtml coverage_stv.info -o coverage_stv -t "Coverage on main -- short test vectors @ $commit_sha" + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # default renderer framesize is 20ms + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only + # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true + # run renderer smoke test - needed for REFVEC tests + - ls -altr scripts/testv + # LTV smoketest is the same as REF creation stage for renderer + - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv + # split rendering test doesn't support LTV - skipped here + # VoIP BE tests don't support LTV - skipped here artifacts: - name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" - when: always - expire_in: 1 week paths: - - coverage.info - - coverage_stv.info - - coverage - - coverage_stv + - $COVERAGE_OUTPUT_FILE + - $COVERAGE_OUTPUT_DIR # --------------------------------------------------------------- # Complexity measurement jobs -- GitLab From e5c84eeb839d2944aa6d5afd6dafb553989def4e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Nov 2025 16:49:16 +0100 Subject: [PATCH 4/5] add missing stage --- main-float.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/main-float.yml b/main-float.yml index 93d6597..f17b861 100644 --- a/main-float.yml +++ b/main-float.yml @@ -9,6 +9,7 @@ stages: - prevalidate - build - test + - basop-compat - compare - postvalidate - deploy -- GitLab From cc6795712fe62a364b229232bd71cef3f6c9ec12 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Nov 2025 16:53:36 +0100 Subject: [PATCH 5/5] align with current ivas-codec main --- includes/default-variables-float.yml | 5 +++++ includes/rules.yml | 14 -------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/includes/default-variables-float.yml b/includes/default-variables-float.yml index 50f6f40..296f068 100644 --- a/includes/default-variables-float.yml +++ b/includes/default-variables-float.yml @@ -18,6 +18,7 @@ variables: - 'ivas-conformance-linux' - 'check-clipping' - 'test-branch-vs-input-passthrough' + - 'coverage' PYTEST_ARGS: "" LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" @@ -26,3 +27,7 @@ variables: SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" TEST_SUITE: "" + COVERAGE_OUTPUT_FILE_STV: "coverage-stv.info" + COVERAGE_OUTPUT_FILE_LTV: "coverage-ltv.info" + COVERAGE_OUTPUT_FILE_CONFORMANCE: "coverage-conformance.info" + COVERAGE_OUTPUT_FILE_MERGED: "coverage-merged.info" diff --git a/includes/rules.yml b/includes/rules.yml index 486257f..0a58899 100644 --- a/includes/rules.yml +++ b/includes/rules.yml @@ -28,20 +28,6 @@ - if: $CI_PIPELINE_SOURCE == 'push' when: never -.rules-merge-request-to-basop-ci-branch: - extends: .rules-basis - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - -.rules-merge-request-to-basop-ci-branch-or-main: - extends: .rules-basis - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main") - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - .rules-main-push: extends: .rules-basis rules: -- GitLab