From e636253fa547b0602d6474ce11446f6e6da97159 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 Sep 2025 16:04:03 +0200 Subject: [PATCH 1/4] move ci code to central repository --- .gitlab-ci.yml | 2651 +------------------------------------- .gitlab-ci/variables.yml | 30 - 2 files changed, 2 insertions(+), 2679 deletions(-) delete mode 100644 .gitlab-ci/variables.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a02bd69e5..5d9b506d38 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,2658 +1,11 @@ variables: # note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this # see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information - IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF main + IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF kiene/consolidate-float-ci-code include: - local: .gitlab-ci/variables.yml - project: ivas-codec-pc/ivas-codec-ci ref: *IVAS_CODEC_CI_REF - file: main.yml + file: main-float.yml -# This sets when pipelines are created. Jobs have more specific rules to restrict them. -workflow: - name: '$IVAS_PIPELINE_NAME' - rules: - # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - variables: - IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main - variables: - IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main - variables: - IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing - variables: - IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' - variables: - IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' - variables: - IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' - variables: - IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'trigger' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' - variables: - IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'trigger' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' - variables: - IVAS_PIPELINE_NAME: 'Check core input clipping: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' - variables: - IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH' - -# --------------------------------------------------------------- -# Generic script anchors -# --------------------------------------------------------------- - -# These can be used later on to do common tasks - -.print-common-info-windows: &print-common-info-windows - - | - echo "Printing common information for build job." - echo "Current job is run on commit $CI_COMMIT_SHA" - echo "Commit time was $CI_COMMIT_TIMESTAMP" - ("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") | Invoke-Expression - -.get-basop-float-reference: &get-basop-float-reference - - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch --depth 1 ivas-basop - - cd ivas-basop - - git status - - make -j - - cd - - - cp ivas-basop/IVAS_cod ./ - - cp ivas-basop/IVAS_dec ./ - -.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec - ### build test binaries, initial clean for paranoia reasons - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh - - make clean - - mkdir build - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_test - - mv IVAS_dec ../IVAS_dec_test - - mv IVAS_rend ../IVAS_rend_test - - cd .. - - rm -rf build/* - - git restore . - - ### store the current commit hash - - source_branch_commit_sha=$(git rev-parse HEAD) - - ### checkout version to compare against - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-fetch-target-branch.sh - - target_commit="$("${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-get-target-commit.sh)" - - git checkout $target_commit - - echo "Building reference codec at commit $target_commit" - - ### build reference binaries - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_ref - - mv IVAS_dec ../IVAS_dec_ref - - mv IVAS_rend ../IVAS_rend_ref - - cd .. - - git restore . - - # rename test binaries back - - mv IVAS_cod_test IVAS_cod - - mv IVAS_dec_test IVAS_dec - - mv IVAS_rend_test IVAS_rend - -.merge-request-comparison-check: &merge-request-comparison-check - - echo "--------------- Running merge-request-comparison-check anchor ---------------" - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -ne 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - -.update-ltv-repo-win: &update-ltv-repo-win - - Push-Location - - cd $LTV_DIR_WIN - - git pull - - Pop-Location - -.copy-ltv-files-to-testv-dir-win: ©-ltv-files-to-testv-dir-win - - cp $LTV_DIR_WIN\*.wav scripts\testv - - cp $LTV_DIR_WIN\*.met scripts\testv - - cp $LTV_DIR_WIN\*.csv scripts\testv - -.activate-WX-windows: &activate-WX-windows - - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } - -# to be reused in MR and LTV-scheduled sanitizer test jobs -# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section -.sanitizer-selftest-anchor: - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - make clean - - make -j CLANG=$CLANG_NUM - - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 - - - exit_code20=0 - - exit_code10=0 - - exit_code5=0 - - - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi - - # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? - # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option - # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active - # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - - - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi - - -# --------------------------------------------------------------- -# .pre jobs for setting up things -# --------------------------------------------------------------- - -# See: https://gitlab.com/gitlab-org/gitlab/-/issues/194023 -# Solution to make main branch pipelines uninterruptible while all other -# pipelines can be interrupted by default. This works because all jobs -# after uninterruptible jobs will be uninterruptible. Resource group -# setting avoids rare case where two fast merges could still interrupt -# pipeline. This should be revisited if there are updates to Gitlab. -uninterruptible: - stage: .pre - interruptible: false - resource_group: uninterruptible - script: - - echo "$CI_COMMIT_BRANCH is uninterruptible" - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: always - tags: - - ivas-linux - -# --------------------------------------------------------------- -# Validation jobs -# --------------------------------------------------------------- - -branch-is-up-to-date-with-main-pre: - extends: - - .rules-merge-request-to-main - - .job-linux - stage: prevalidate - needs: [] - script: - - | - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" - echo "Commits behind: $commits_behind_count" - if [ "$commits_behind_count" -ne 0 ]; then - echo "Your branch is behind main, run 'git merge origin/main' to update." - exit 1 - fi - -check-self-test-names-pre: - extends: - - .rules-merge-request-to-main - - .job-linux - stage: prevalidate - needs: [] - script: - - python3 ci/check_self_test_names.py scripts/config/self_test.prm 135 - -check-no-duplicates-in-self-tests: - extends: - - .rules-merge-request-to-main - - .job-linux - stage: prevalidate - needs: [] - script: - - python3 ci/find_duplicates_in_prm_files.py - -branch-is-up-to-date-with-main-post: - extends: - - .rules-merge-request-to-main - - .job-linux - stage: postvalidate - script: - - | - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" - echo "Commits behind: $commits_behind_count" - if [ "$commits_behind_count" -ne 0 ]; then - echo "Your branch is behind main, possibly main changed during your pipeline run, run 'git merge origin/main' to update." - exit 1 - fi - -.basop-ci-branch-compat-template: - extends: - - .test-job-linux - - .rules-merge-request-to-basop-ci-branch - tags: - - ivas-linux - before_script: - - !reference [.job-linux, before_script] - - python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm - - python3 tests/create_short_testvectors.py - - python3 scripts/prepare_combined_format_inputs.py - - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - - *get-basop-float-reference - - - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT - - artifacts: - name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" - when: always - expire_in: 1 month - paths: - - report-junit.xml - - report.html - - expose_as: "check-float-reference results" - reports: - junit: - - report-junit.xml - -basop-ref-compat-stv: - extends: - - .basop-ci-branch-compat-template - variables: - TEST_SUITE: "$SHORT_TEST_SUITE tests/renderer" - TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS - script: - - exit 0 - -basop-ref-compat-ltv: - extends: - - .basop-ci-branch-compat-template - tags: - - ivas-linux-fast - variables: - TEST_SUITE: $LONG_TEST_SUITE - TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS - script: - - exit 0 - -basop-ref-compat-encoder-stv: - extends: - - .basop-ci-branch-compat-template - variables: - TEST_SUITE: $SHORT_TEST_SUITE_ENCODER - TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS - script: - - exit 0 - -basop-ref-compat-encoder-ltv: - extends: - - .basop-ci-branch-compat-template - tags: - - ivas-linux-fast - variables: - TEST_SUITE: $LONG_TEST_SUITE_ENCODER - TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS - script: - - exit 0 - -basop-ref-compat-encoder-dmx-comp: - extends: - - .basop-ci-branch-compat-template - variables: - TEST_SUITE: $SHORT_TEST_SUITE_ENCODER - # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option - PYTEST_ARGS: "--compare_enc_dmx" - TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS - script: - - exit 0 - -# --------------------------------------------------------------- -# Build jobs -# --------------------------------------------------------------- - -build-codec-linux-make: - extends: - - .build-job-linux - - .rules-basis - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh - - make -j - -build-codec-linux-cmake: - extends: - - .build-job-linux - - .rules-basis - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh - - mkdir build - - cd build - - cmake .. - - cd .. - - make -C build -j - -build-codec-instrumented-linux: - extends: - - .build-job-linux - - .rules-basis - timeout: "10 minutes" - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - ./scripts/prepare_instrumentation.sh - - make -j -C scripts/c-code_instrument - artifacts: - when: always - access: all - expire_in: "30 days" - paths: - - wmc_tool_output.txt - - -# make sure that the codec builds with msan, asan and usan -build-codec-sanitizers-linux: - extends: - - .build-job-linux - - .rules-basis - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh - - bash ci/build_codec_sanitizers_linux.sh - -build-codec-windows-cmake: - extends: - - .build-job-windows - - .rules-basis - script: - - *print-common-info-windows - - *activate-WX-windows - - cmake -G "Visual Studio 15 2017" . -Bbuild - - cmake --build build -j - -build-codec-windows-msbuild: - extends: - - .build-job-windows - - .rules-basis - script: - - *print-common-info-windows - - *activate-WX-windows - - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug - -# --------------------------------------------------------------- -# Test jobs for merge requests -# --------------------------------------------------------------- - -# test that runs all modes with 1s input signals -codec-smoke-test: - extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request-to-basop-ci-branch-or-main - timeout: "20 minutes" - tags: - - ivas-linux-fast - stage: test - needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - # LTV update needed as ltv ISM metadata files are used - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - # for MRs to basop-ci-branch, we want to test with the BASOP float reference build - # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros - - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then - - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline - - *get-basop-float-reference - - bash ci/smoke_test.sh coverage - - else - - bash ci/smoke_test.sh - - fi - - ### analyze for failures - - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi - - ret_val=0 - - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without JBM failed"; ret_val=1; fi - - if cat smoke_test_output_jbm.txt | grep -c "failed"; then echo "Smoke test JBM part failed"; ret_val=1; fi - - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then echo "Smoke test with external hrtf files failed"; ret_val=1; fi - - exit $ret_val - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - smoke_test_output.txt - - smoke_test_output_jbm.txt - - smoke_test_output_hrtf.txt - expose_as: "Smoke test results" - -# code selftest testvectors with memory-sanitizer binaries -codec-msan: - extends: - - .sanitizer-selftest-on-mr - - .sanitizer-selftest-anchor - tags: - - ivas-linux-fast - before_script: - - !reference [.job-linux, before_script] - - CLANG_NUM=1 - - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS - - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - - USE_LTV="" - -# code selftest testvectors with address-sanitizer binaries -codec-asan: - extends: - - .sanitizer-selftest-on-mr - - .sanitizer-selftest-anchor - before_script: - - !reference [.job-linux, before_script] - - CLANG_NUM=2 - - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS - - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - - USE_LTV="" - -# code selftest testvectors with undefined-behaviour-sanitizer binaries -codec-usan: - extends: - - .sanitizer-selftest-on-mr - - .sanitizer-selftest-anchor - tags: - - ivas-linux-fast - before_script: - - !reference [.job-linux, before_script] - - CLANG_NUM=3 - - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS - - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" - - USE_LTV="" - -# compare bit-exactness between 5ms and 20 on the branch -pytest-compare-20ms-and-5ms-rendering: - extends: - - .test-job-linux - - .rules-merge-request-to-main - stage: test - needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh - - make clean - - make -j - ### prepare pytest - - cp IVAS_cod IVAS_cod_ref - - cp IVAS_dec IVAS_dec_ref - # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 - ### run pytest - - exit_code=0 - - exit_code5=0 - - exit_code10=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? - - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true - - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true - - zero_errors=1 - - if [ $zero_errors5 != 1 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi - - if [ $zero_errors10 != 1 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi - - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi - - if [ $exit_code5 -ne 0 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi - - if [ $exit_code10 -ne 0 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi - - if [ $exit_code -ne 0 ]; then exit $EXIT_CODE_FAIL; fi - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - expose_as: "pytest 5ms and 10ms vs 20ms results" - paths: - - report-junit-5ms.xml - - report-5ms.html - - report-junit-10ms.xml - - report-10ms.html - reports: - junit: - - report-junit-5ms.xml - - report-junit-10ms.xml - -# test renderer executable -renderer-smoke-test: - extends: - - .test-job-linux - - .rules-merge-request-to-main - needs: ["build-codec-linux-make"] - stage: test - script: - - make -j IVAS_rend - - testcase_timeout=60 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - expose_as: "renderer make pytest results" - reports: - junit: - - report-junit.xml - -.renderer-sanitizer-job: - extends: - - .test-job-linux - - .rules-merge-request-to-main - needs: ["build-codec-linux-cmake"] - stage: test - timeout: "90 minutes" - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - reports: - junit: - - report-junit.xml - before_script: - - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - - cmake --build cmake-build -- -j - # rename files to fit naming convention - # en- and decoder needed because of split rendering testcases - - mv IVAS_cod IVAS_cod_ref - - mv IVAS_dec IVAS_dec_ref - - mv IVAS_rend IVAS_rend_ref - - mv ISAR_post_rend ISAR_post_rend_ref - - testcase_timeout=180 - -# test renderer executable with cmake + asan -renderer-asan: - extends: - - .renderer-sanitizer-job - variables: - SANITIZER_BUILD_STRING: "asan" - script: - - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout - artifacts: - expose_as: "renderer asan result" - - -# test renderer executable with cmake + msan -renderer-msan: - extends: - - .renderer-sanitizer-job - variables: - SANITIZER_BUILD_STRING: "msan" - script: - - python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout - artifacts: - expose_as: "renderer msan result" - - -# test renderer executable with cmake + usan -renderer-usan: - extends: - - .renderer-sanitizer-job - variables: - SANITIZER_BUILD_STRING: "usan" - script: - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest -q --tb=no -n auto --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout - artifacts: - expose_as: "renderer usan result" - - -# compare renderer bitexactness between target and source branch -renderer-pytest-on-merge-request: - extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request-to-main - needs: ["build-codec-linux-make"] - # TODO: set reasonable timeout, will most likely take less - timeout: "20 minutes" - stage: compare - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh - - *merge-request-comparison-setup-codec - - # some helper variables - "|| true" to prevent failures from grep not finding anything - # write to temporary file as workaround for failures observed with piping echo - - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - - non_be_flag=$(grep -c --ignore-case "\[rend\(erer\)*[ -]*non[ -]*be\]" tmp.txt) || true - - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true - - ### If ref_using_main is not set, checkout the source branch to use scripts and input from there - - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi - - - exit_code=0 - - testcase_timeout=60 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? - - ### Run test using branch scripts and input - - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi - - # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - *merge-request-comparison-check - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "pytest renderer results" - reports: - junit: - - report-junit.xml - -# test split rendering -split-rendering-smoke-test: - extends: - - .test-job-linux - - .rules-merge-request-to-main - needs: ["build-codec-linux-make"] - stage: test - script: - - make -j - - testcase_timeout=10 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - expose_as: "split rendering make pytest results" - reports: - junit: - - report-junit.xml - -# test split rendering in VoIP mode against BINAURAL output -split-rendering-voip-be-to-binaural: - extends: - - .test-job-linux - - .rules-merge-request-to-main - needs: ["build-codec-linux-make"] - stage: test - script: - - make -j - - testcase_timeout=30 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_voip_be_splitrend_vs_binaural.py --testcase_timeout=$testcase_timeout - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - expose_as: "VoIP split rendering vs BINAURAL make pytest results" - reports: - junit: - - report-junit.xml - -lc3-wrapper-unit-test: - extends: - - .test-job-linux - - .rules-merge-request-to-main - needs: ["build-codec-linux-cmake"] - stage: test - script: - - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - - cmake --build cmake-build -- -j - - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test - -# compare split-rendering bitexactness between target and source branch -split-rendering-pytest-on-merge-request: - extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request-to-main - needs: ["build-codec-linux-make"] - # TODO: set reasonable timeout, will most likely take less - timeout: "30 minutes" - stage: compare - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh - - # some helper variables - "|| true" to prevent failures from grep not finding anything - # write to temporary file as workaround for failures observed with piping echo - - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true - - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true - - # store the current commit hash - - source_branch_commit_sha=$(git rev-parse HEAD) - - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-fetch-target-branch.sh - - target_commit="$("${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/mr-get-target-commit.sh)" - - git checkout $target_commit - - echo "Building reference codec at commit $target_commit" - - # build reference binaries - - make -j - - mv IVAS_cod IVAS_cod_ref - - mv IVAS_dec IVAS_dec_ref - - mv IVAS_rend IVAS_rend_ref - - mv ISAR_post_rend ISAR_post_rend_ref - - ### If ref_using_main is not set, checkout the source branch to use scripts and input from there - - if [ $ref_using_main == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi - - exit_code=0 - - testcase_timeout=60 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? - - # back to source branch - - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts - - git checkout $source_branch_commit_sha - - make clean - - make -j - - ### Run test using scripts and input from main - - if [ $ref_using_main == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi - - # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - *merge-request-comparison-check - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 2 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "pytest split rendering results" - reports: - junit: - - report-junit.xml - -# compare bit exactness between target and source branch -ivas-pytest-on-merge-request: - extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request-to-main - stage: compare - needs: ["build-codec-linux-cmake", "codec-smoke-test"] - timeout: "14 minutes" - parallel: - matrix: - # note: keep in sync with list in $TESTS_DIR_CODEC_BE_ON_MR - - PYTEST_SCRIPT: - - test_param_file.py - - test_sba.py - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh - - *merge-request-comparison-setup-codec - - # some helper variables - "|| true" to prevent failures from grep not finding anything - # write to temporary file as workaround for failures observed with piping echo - - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true - - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true - - ### If ref_using_main is not set, checkout the source branch to use scripts and input from there - - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi - - ### prepare pytest - # create references - - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1 - - ### Run test using branch scripts and input - - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi - - ### run pytest - - exit_code=0 - - testcase_timeout=60 - - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - *merge-request-comparison-check - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "pytest ivas results" - reports: - junit: - - report-junit.xml - - -# Check interop IVAS_cod_test -> IVAS_dec_ref -ivas-interop-on-merge-request: - extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request-to-main - stage: test - needs: ["build-codec-linux-cmake"] - timeout: "10 minutes" - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh - - *merge-request-comparison-setup-codec - # the next line is dependent on ref-using-main flag in the other tests, but here the flag does not make sense - - git checkout $source_branch_commit_sha - - # some helper variables - "|| true" to prevent failures from grep not finding anything - # write to temporary file as workaround for failures observed with piping echo - - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - - non_interop_flag=$(grep -c --ignore-case "\[non[ -]*io\]" tmp.txt) || true - - ### prepare pytest - - # Run reference creation, using source branch encoder and main decoder (see .merge-request-comparison-setup-codec) - - exit_code=0 - - exit_code2=0 - # set timeout for individual testcase runs to 60 seconds - - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? - - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true - - - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "interop test results" - reports: - junit: - - report*-junit.xml - -evs-pytest-on-merge-request: - extends: - - .test-job-linux - - .rules-merge-request-to-main - stage: compare - needs: ["build-codec-linux-cmake", "codec-smoke-test"] - timeout: "10 minutes" - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh - - *merge-request-comparison-setup-codec - - # some helper variables - "|| true" to prevent failures from grep not finding anything - # write to temporary file as workaround for failures observed with piping echo - - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - - non_be_flag=$(grep -c --ignore-case "\[evs[ -]*non[ -]*be\]" tmp.txt) || true - - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true - - ### If ref_using_main is not set, checkoug the source branch to use scripts and input from there - - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi - - ### prepare pytest - # create references - - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout - - ### Run test using branch scripts and input - - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi - - ### run pytest for EVS cases - - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml --testcase_timeout=$testcase_timeout || exit_code=$? - - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - - - *merge-request-comparison-check - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit-evs.xml - - report.html - expose_as: "pytest evs results" - reports: - junit: - - report-junit-evs.xml - -voip-be-on-merge-request: - extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request-to-main - stage: test - needs: ["build-codec-linux-make"] - timeout: "10 minutes" - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - make clean - - make -j - - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py - -clang-format-check: - extends: - - .test-job-linux - - .rules-merge-request-to-main - variables: - ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" - stage: prevalidate - needs: [] - timeout: "5 minutes" - script: - # Set up variables. This can't be done in the "variables" section because variables are not expanded properly there - - PATCH_FILE_NAME="$ARTIFACT_BASE_NAME".patch - - > - INSTRUCTIONS_GITLAB="To fix formatting issues:\n - - download the diff patch available as artifact of this job\n - - unzip the artifact and place the patch file in the root directory of your local IVAS repo\n - - run: git apply $PATCH_FILE_NAME\n - - commit new changes" - - > - INSTRUCTIONS_README="To fix formatting issues:\n - - place the patch file in the root directory of your local IVAS repo\n - - run: git apply $PATCH_FILE_NAME\n - - commit new changes" - - - format_problems=0 - - scripts/check-format.sh -af -p 8 || format_problems=$? - - if [ $format_problems == 0 ] ; then exit 0; fi - - - mkdir tmp-formatting-fix - - git diff > "tmp-formatting-fix/$PATCH_FILE_NAME" - - # Print instructions to job output - - echo -e "$INSTRUCTIONS_GITLAB" - - # Include readme in the artifact, in case someone misses the job printout (e.g. getting the artifact via MR interface) - - echo -e "$INSTRUCTIONS_README" > "tmp-formatting-fix/readme.txt" - - - exit $format_problems - artifacts: - expire_in: 1 day - paths: - - tmp-formatting-fix/ - when: on_failure - name: "$ARTIFACT_BASE_NAME" - expose_as: "formatting patch" - -# check for crashes if first received frame on decoder side is an SID -check-first-frame-is-sid: - extends: - - .test-job-linux-needs-testv-dir - - .rules-merge-request-to-main - stage: test - needs: ["build-codec-linux-cmake"] - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - # this rm makes check-for-testvectors only check for the signals we actually need in this test - - rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/check-for-testvectors.sh - - bash ci/run-first-frame-is-sid-test.sh - - artifacts: - paths: - - logs_enc - - logs_dec_msan - - logs_dec_asan - - logs_dec_usan - when: always - name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--sidstart" - expose_as: "logs-sidstart" - expire_in: "5 days" - -lc3plus-ensure-no-code-changes: - extends: - - .test-job-linux - - .rules-merge-request-to-main - stage: postvalidate - needs: [] - timeout: "5 minutes" - script: - # Replace code commited to repo with code downloaded from ETSI - - ./scripts/lc3plus_lib_setup/get_lc3plus.sh - - # Ensure git reports no changes - - modified_files=$(git status -su lib_lc3plus) - - if [[ $modified_files ]]; then printf 'LC3plus codebase was modified!\n\n'"$modified_files"'\n\n'; exit $EXIT_CODE_FAIL; fi - -check-bitexactness-hrtf-rom-and-file: - extends: - - .test-job-linux - - .rules-merge-request-to-main - stage: test - needs: ["build-codec-linux-cmake"] - timeout: "5 minutes" - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - cmake . - - make -j - - python3 tests/create_short_testvectors.py --cut_len 1.0 - - python3 -m pytest tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html - artifacts: - paths: - - report.html - - report-junit.xml - when: always - name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading" - expose_as: "logs-hrtf-loading" - expire_in: "5 days" - -check-bitexactness-ext-and-transport-format: - extends: - - .test-job-linux - - .rules-merge-request-to-main - stage: test - needs: ["build-codec-linux-cmake"] - timeout: "5 minutes" - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - cmake . - - make -j - - python3 tests/create_short_testvectors.py --cut_len 1.0 - - python3 -m pytest tests/test_be_for_ext_outputs.py --html=report.html --junit-xml=report-junit.xml --self-contained-html - artifacts: - paths: - - report.html - - report-junit.xml - when: always - name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--ext-sanity-check" - expose_as: "logs-ext-sanity-check" - expire_in: "5 days" - - -# --------------------------------------------------------------- -# Test jobs for main branch -# --------------------------------------------------------------- - -# check bitexactness to EVS windows binaries -be-2-evs-windows: - extends: - - .rules-merge-request-to-main - tags: - - ivas-windows - stage: test - needs: ["build-codec-windows-msbuild"] - timeout: "20 minutes" # To be revisited - script: - - *print-common-info-windows - - - $winoutdata = $null - - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata - - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 - - # copy over to never change the testvector dir - - cp -r $EVS_BE_WIN_TEST_DIR ./evs_be_win_test - - cp IVAS_cod.exe ./evs_be_win_test/bin/IVAS_cod.exe - - cp IVAS_dec.exe ./evs_be_win_test/bin/IVAS_dec.exe - - - cd evs_be_win_test - - python ../ci/run_evs_be_win_test.py - -# check bitexactness to EVS -be-2-evs-linux: - extends: - - .test-job-linux - - .rules-main-push - tags: - - be-2-evs-temp - stage: test - needs: ["build-codec-linux-cmake"] - timeout: "20 minutes" # To be revisited - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - - mkdir build - - cd build - - cmake .. - - make -j - - cd .. - - # copy over to never change the testvector dir - - cp -r $EVS_BE_TEST_DIR ./evs_be_test - - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod - - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec - - - cd evs_be_test - - python3 ../ci/run_evs_be_test.py - -codec-comparison-on-main-push: - extends: - - .test-job-linux-needs-testv-dir - - .rules-main-push - stage: compare - needs: ["build-codec-linux-cmake"] - timeout: "30 minutes" # To be revisited - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - latest_commit=$(git rev-parse HEAD) # Latest commit - - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) - - echo "Comparing changes from $previous_merge_commit to $latest_commit" - - git --no-pager diff --stat $previous_merge_commit..$latest_commit - - # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. - - ### build test binaries, initial clean for paranoia reasons - - make clean - - mkdir build - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_test - - mv IVAS_dec ../IVAS_dec_test - - cd .. - - rm -rf build/* - - ### compare to the previous merge commit in the main branch - - git fetch origin main - - git checkout $previous_merge_commit - - echo "Building reference codec at commit $previous_merge_commit" - - ### build reference binaries - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_ref - - mv IVAS_dec ../IVAS_dec_ref - - cd .. - - # helper variable - "|| true" to prevent failures from grep not finding anything - # write to temporary file as workaround for failures observed with piping echo - - echo $CI_COMMIT_MESSAGE > tmp.txt - - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true - - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true - - ### re-checkout the latest commit in the main branch, if ref_using_main is not set - - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi - - ### prepare pytest - # rename test binaries back - - mv IVAS_cod_test IVAS_cod - - mv IVAS_dec_test IVAS_dec - # create references - - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout - - ### re-checkout the latest commit here, if ref_using_main is set - - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi - - ### run pytest - - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi - - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; - allow_failure: - exit_codes: - - 123 - artifacts: - name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - expose_as: "Results of comparison to previous merge commit" - reports: - junit: report-junit.xml - - -# --------------------------------------------------------------- -# Manual jobs -# --------------------------------------------------------------- - -test-be-to-release: - stage: test - tags: - - ivas-windows - resource_group: ivas-be-to-release-test-resource - timeout: "20 minutes" - rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' - script: - - - echo "$CI_COMMIT_BRANCH" - - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata - - # path to release candidate refs defined in config.toml - - - echo "Placeholder for BE test to release script" - - mkdir logs - - artifacts: - name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" - when: always - expire_in: 1 week - paths: - - logs/ - expose_as: "test-be-to-release results" - - -ivas-conformance: - tags: - - ivas-windows - stage: test - timeout: "90 minutes" - rules: - - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - allow_failure: - exit_codes: - - 123 - script: - - *print-common-info-windows - - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug - - cp -force IVAS_cod.exe IVAS_cod_ref.exe - - cp -force IVAS_dec.exe IVAS_dec_ref.exe - - cp -force IVAS_rend.exe IVAS_rend_ref.exe - - cp -force ISAR_post_rend.exe ISAR_post_rend_ref.exe - - # Reference creation - - python scripts/prepare_combined_format_inputs.py - - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py" - - python -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files - - # Output creation - - python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt - - # Copy input data and output ref data - - if (Test-Path testvec) {rm -r -force testvec} - - if (Test-Path TMP_DEC) {rm -r -force TMP_DEC} - - if (Test-Path TMP_ENC) {rm -r -force TMP_ENC} - - if (Test-Path TMP_JBM) {rm -r -force TMP_JBM} - - if (Test-Path TMP_REND) {rm -r -force TMP_REND} - - if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND} - - if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR} - - mkdir testvec - - mkdir testvec/binauralRenderer_interface - - mkdir testvec/testv - - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_rendering - - mkdir testvec/bin - - cp -force -ErrorAction Ignore scripts/testv/* testvec/testv - - cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec - - cp -r -force -ErrorAction Ignore scripts/object_edit testvec - - cp -r -force -ErrorAction Ignore scripts/switchPaths testvec - - cp -r -force -ErrorAction Ignore scripts/trajectories testvec - - cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref - - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/cut testvec/testv/split_rendering/ref - - cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - - cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns - - # Remove redundant files - - python scripts/cleanup_26252.py - - # Copy test script files - - cp -r -force -ErrorAction Ignore tests/conformance-test testvec/ - - cp Readme_IVAS_dec.txt testvec - - cp Readme_IVAS_enc.txt testvec - - cp Readme_IVAS_rend.txt testvec - - cp Readme_IVAS_JBM_dec.txt testvec - - cp Readme_IVAS_ISAR_dec.txt testvec - - cp Readme_IVAS_ISAR_post_rend.txt testvec - - cp IVAS_cod.exe testvec/bin - - cp IVAS_dec.exe testvec/bin - - cp IVAS_rend.exe testvec/bin - - cp ISAR_post_rend.exe testvec/bin - - - # Test run generated scripts in testvec - - cd testvec - - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html - - mv report.html .. - - mv report-junit.xml .. - - artifacts: - name: "ivas-conformance-$CI_COMMIT_SHORT_SHA" - expire_in: 1 week - when: always - paths: - - report_cmd.html - - report-junit.xml - - report.html - - Readme_IVAS_dec.txt - - Readme_IVAS_enc.txt - - Readme_IVAS_rend.txt - - Readme_IVAS_JBM_dec.txt - - Readme_IVAS_ISAR_dec.txt - - Readme_IVAS_ISAR_post_rend.txt - expose_as: "Draft IVAS conformance" - reports: - junit: report-junit.xml - -ivas-conformance-linux: - extends: - - .job-linux - stage: test - timeout: "90 minutes" - rules: - - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' - allow_failure: - exit_codes: - - 123 - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - make -j - - cp IVAS_cod IVAS_cod_ref - - cp IVAS_dec IVAS_dec_ref - - cp IVAS_rend IVAS_rend_ref - - cp ISAR_post_rend ISAR_post_rend_ref - - # Reference creation - - python3 scripts/prepare_combined_format_inputs.py - - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py" - - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files - - # Output creation - - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt - - # Copy input data and output ref data - - rm -rf testvec - - mkdir testvec - - mkdir testvec/binauralRenderer_interface - - mkdir testvec/testv - - mkdir testvec/testv/renderer - - mkdir testvec/testv/split_rendering - - mkdir testvec/bin - - cp -r scripts/testv/* testvec/testv - - cp -r scripts/ls_layouts testvec - - cp -r scripts/object_edit testvec - - cp -r scripts/switchPaths testvec - - cp -r scripts/trajectories testvec - - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface - - cp -r tests/ref testvec/testv/ref - - cp -r tests/dut/* testvec/testv/ref - - cp -r tests/renderer/cut testvec/testv/renderer/ref - - cp -r tests/split_rendering/cut testvec/testv/split_rendering/ref - - cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs - - cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns - - # Remove redundant files - - python3 scripts/cleanup_26252.py - - # Copy test script files - - cp -r tests/conformance-test testvec/ - - cp Readme_IVAS_dec.txt testvec - - cp Readme_IVAS_enc.txt testvec - - cp Readme_IVAS_rend.txt testvec - - cp Readme_IVAS_JBM_dec.txt testvec - - cp Readme_IVAS_ISAR_dec.txt testvec - - cp Readme_IVAS_ISAR_post_rend.txt testvec - - # Create GCOV execs for coverage analysis - - make clean - - make GCOV=1 -j - - - cp IVAS_cod testvec/bin - - cp IVAS_dec testvec/bin - - cp IVAS_rend testvec/bin - - cp ISAR_post_rend testvec/bin - - # Test run generated scripts in testvec - - cd testvec - - exit_code=0 - - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? - - mv report.html .. - - mv report-junit.xml .. - - # Collect coverage - - cd - - - lcov -c -d obj -o coverage.info - - lcov -r coverage.info "*apps*" -o coverage.info - - lcov -r coverage.info "*lib_util*" -o coverage.info - - commit_sha=$(git rev-parse HEAD) - - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" - - # Check for failures - - if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi - - artifacts: - name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" - expire_in: 1 week - when: always - paths: - - report_cmd.html - - report-junit.xml - - report.html - - Readme_IVAS_dec.txt - - Readme_IVAS_enc.txt - - Readme_IVAS_rend.txt - - Readme_IVAS_JBM_dec.txt - - Readme_IVAS_ISAR_dec.txt - - Readme_IVAS_ISAR_post_rend.txt - - coverage.info - - coverage - expose_as: "Draft IVAS conformance -- Linux" - reports: - junit: report-junit.xml - - -test-long-self-test: - extends: - - .job-linux - stage: compare - resource_group: ivas-long-self-test-resource - timeout: "50 minutes" - rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' - allow_failure: - exit_codes: - - 123 - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - ### build branch binaries - - cmake . -Bbuild-test - - cmake --build build-test -- -j - - mv build-test/IVAS_cod ./IVAS_cod - - mv build-test/IVAS_dec ./IVAS_dec - - ### store the current commit hash - - source_branch_commit_sha=$(git rev-parse HEAD) - - ### switch to main - - git checkout main - - git pull origin main - - echo "Building reference codec at commit $(git rev-parse HEAD)" - - ### build main (ref) binaries - - cmake . -Bbuild-ref - - cmake --build build-ref -- -j - - mv build-ref/IVAS_cod ./IVAS_cod_ref - - mv build-ref/IVAS_dec ./IVAS_dec_ref - - ### Switch back to branch, this could probably be removed later - - git checkout $source_branch_commit_sha - - ### prepare pytest - - # Copy test vectors from LTV to TESTV - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - # create references - - exit_code_ref=0 - - testcase_timeout=300 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? - - ### run pytest self-test using long test vectors - - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? - - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - - artifacts: - name: "test-long-self-test--sha-$CI_COMMIT_SHORT_SHA--results" - when: always - expire_in: 4 mos - paths: - - report-junit-ltv.xml - - report-ltv.html - expose_as: "test-long-self-test results" - reports: - junit: - - report-junit-ltv.xml - -check-clipping: - extends: - - .job-linux - stage: test - timeout: "30 minutes" - rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' - allow_failure: - exit_codes: - - 123 - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/enable-debugging-macro.sh - - make -j - - tests/scale_pcm.py ./scripts/testv/ $SCALE_FACTOR # Default: 3.162 (+10 dB). Can be set in manual trigger. - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 scripts/parse_xml_report.py report-junit.xml report.csv --clipping - artifacts: - name: "check-clipping--sha-$CI_COMMIT_SHORT_SHA--results" - when: always - expire_in: 2 weeks - paths: - - report-junit.xml - - report.html - - report.csv - expose_as: "check-clipping results" - reports: - junit: - - report-junit.xml - -test-branch-vs-input-passthrough: - extends: - - .job-linux - stage: compare - timeout: "30 minutes" # TBD - rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' - allow_failure: - exit_codes: - - 123 - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - make -j - - - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_xml_report.py report-junit.xml report.csv - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -ne 0 ]; then echo "Differences encountered."; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - artifacts: - name: "test-branch-vs-input-passthrough--sha-$CI_COMMIT_SHORT_SHA--results" - when: always - expire_in: 4 mos - paths: - - report-junit.xml - - report.html - - report.csv - expose_as: "test-branch-vs-input-passthrough results" - reports: - junit: - - report-junit.xml - -# --------------------------------------------------------------- -# Scheduled jobs on main -# --------------------------------------------------------------- - -# code selftest long testvectors with memory-sanitizer binaries -ltv-msan: - extends: - - .sanitizer-selftest-ltv - - .sanitizer-selftest-anchor - rules: - - if: $SANITIZER_SCHEDULE_E - timeout: 5 hours - tags: - - ivas-linux-fast - before_script: - - !reference [.job-linux, before_script] - - CLANG_NUM=1 - - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS - - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" - - USE_LTV="--use_ltv" - -# code selftest long testvectors with address-sanitizer binaries -ltv-asan: - extends: - - .sanitizer-selftest-ltv - - .sanitizer-selftest-anchor - rules: - - if: $SANITIZER_SCHEDULE_E - when: delayed - start_in: 5 hours - tags: - - ivas-linux-fast - timeout: 5 hours - before_script: - - !reference [.job-linux, before_script] - - CLANG_NUM=2 - - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS - - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" - - USE_LTV="--use_ltv" - -# code selftest long testvectors with undefined-behaviour-sanitizer binaries -ltv-usan: - extends: - - .sanitizer-selftest-ltv - - .sanitizer-selftest-anchor - rules: - - if: $SANITIZER_SCHEDULE_E - when: delayed - start_in: 8 hours - tags: - - ivas-linux-fast - timeout: 5 hours - before_script: - - !reference [.job-linux, before_script] - - CLANG_NUM=3 - - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS - - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm" - - USE_LTV="--use_ltv" - -.sanitizer-test-template: - extends: - - .test-job-linux - stage: test - tags: - - sanitizer_test_main - - ivas-linux-fast - artifacts: - name: "$CI_JOB_NAME--main--sha-$CI_COMMIT_SHORT_SHA" - expire_in: 7 weeks - when: always - paths: - - ep_015.g192 - - dly_profile.dat - - head_rot_traj.csv - - exof_traj.csv - - ./LOGS_PLC - - ./LOGS_noPLC - -### --- sanitizer schedule A --- - -.sanitizer-test-schedule-A: - extends: - - .sanitizer-test-template - -sanitizer-test-mono: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - timeout: 1.25 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS - -sanitizer-test-stereo: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 1.25 hours - timeout: 2.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED EXT --tests $SANITIZER_TESTS - -sanitizer-test-stereodmxevs: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 3.75 hours - timeout: 1.25 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py StereoDmxEVS mono --tests $SANITIZER_TESTS - -sanitizer-test-mc-5_1: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 5 hours - timeout: 2.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-mc-5_1_2: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 7.5 hours - timeout: 2.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-mc-5_1_4: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 10 hours - timeout: 3.75 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-mc-7_1: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 13.75 hours - timeout: 2.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-mc-7_1_4: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 16.25 hours - timeout: 5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-ism+1: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 21.25 hours - timeout: 1.25 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-ism+2: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 22.5 hours - timeout: 2.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-ism+3: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 25 hours - timeout: 3.75 hour - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-ism+4: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 28.75 hours - timeout: 5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-masa: - extends: .sanitizer-test-schedule-A - rules: - - if: $SANITIZER_SCHEDULE_A - when: delayed - start_in: 33.75 hours - timeout: 10 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -### --- sanitizer schedule B --- - -.sanitizer-test-schedule-B: - extends: - - .sanitizer-test-template - timeout: 7.5 hours - -sanitizer-test-foa: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py FOA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-hoa2: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 7.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-hoa3: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 15 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-foa-ism1: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 22.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-foa-ism2: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 30 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-foa-ism3: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 37.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-foa-ism4: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 45 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py FOA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-hoa2-ism1: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 52.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-hoa2-ism2: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 60 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-hoa2-ism3: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 67.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-hoa2-ism4: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 75 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA2-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-hoa3-ism1: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 82.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-hoa3-ism2: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 90 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-omasa-ism4: - extends: .sanitizer-test-schedule-B - rules: - - if: $SANITIZER_SCHEDULE_B - when: delayed - start_in: 97.5 hours - timeout: 10 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - - -### --- sanitizer schedule C --- - -.sanitizer-test-schedule-C: - extends: - - .sanitizer-test-template - timeout: 10 hours - -sanitizer-test-omasa-ism1: - extends: .sanitizer-test-schedule-C - rules: - - if: $SANITIZER_SCHEDULE_C - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-omasa-ism2: - extends: .sanitizer-test-schedule-C - rules: - - if: $SANITIZER_SCHEDULE_C - when: delayed - start_in: 10 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-omasa-ism3: - extends: .sanitizer-test-schedule-C - rules: - - if: $SANITIZER_SCHEDULE_C - when: delayed - start_in: 20 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-hoa3-ism3: - extends: .sanitizer-test-schedule-C - rules: - - if: $SANITIZER_SCHEDULE_C - when: delayed - start_in: 30 hours - timeout: 7.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-hoa3-ism4: - extends: .sanitizer-test-schedule-C - rules: - - if: $SANITIZER_SCHEDULE_C - when: delayed - start_in: 37.5 hours - timeout: 7.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py HOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -### --- sanitizer schedule D --- - -.sanitizer-test-schedule-D: - extends: - - .sanitizer-test-template - timeout: 7.5 hours - -sanitizer-test-planar-foa: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-planar-hoa2: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 7.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-planar-hoa3: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 15 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-foa-ism1: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 22.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-foa-ism2: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 30 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-foa-ism3: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 37.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-foa-ism4: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 45 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarFOA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-hoa2-ism1: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 52.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-hoa2-ism2: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 60 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-hoa2-ism3: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 67.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-hoa2-ism4: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 75 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA2-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-hoa3-ism1: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 82.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM1 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-hoa3-ism2: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 90 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM2 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-hoa3-ism3: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 97.5 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM3 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - -sanitizer-test-osba-planar-hoa3-ism4: - extends: .sanitizer-test-schedule-D - rules: - - if: $SANITIZER_SCHEDULE_D - when: delayed - start_in: 105 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS - - - -# GCOV/LCOV coverage analysis of self_test suite -coverage-test-on-main-scheduled: - extends: - - .test-job-linux-needs-testv-dir - - .rules-main-scheduled - tags: - - ivas-linux-fast - stage: test - rules: - # only run in scheduled pipeline that passes this env vars - - if: $COVERAGE_TEST - timeout: 3 hours - script: - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - make GCOV=1 -j - - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script - - # -- Run short test vector suite to check coverage of that first - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref - - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut - - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here - # remove apps and lib_util files from coverage - - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info - - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info - - # -- Add extended coverage of EVS test and long test vectors - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - bash ci/smoke_test.sh coverage - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true - - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py - - lcov -c -d obj -o coverage.info - # remove apps and lib_util files from coverage - - lcov -r coverage.info "*apps*" -o coverage.info - - lcov -r coverage.info "*lib_util*" -o coverage.info - - commit_sha=$(git rev-parse HEAD) - - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" - - genhtml coverage_stv.info -o coverage_stv -t "Coverage on main -- short test vectors @ $commit_sha" - artifacts: - name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" - when: always - expire_in: 1 week - paths: - - coverage.info - - coverage_stv.info - - coverage - - coverage_stv - -# --------------------------------------------------------------- -# Complexity measurement jobs -# --------------------------------------------------------------- - -.complexity-measurements-setup: - &complexity-measurements-setup # create necessary environment - - mkdir -p wmops/logs - - - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) - - echo $job_id - - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - - public_dir="$CI_JOB_NAME-public" - - # if is needed to catch case when no artifact is there (first run), similarly as above - # 1. check for public_dir being there as this might not be the case when artifact download failed - # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts - - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then - - mv $public_dir/* wmops/ - # check here if we have the split-by-levels files present - if not, fake them up with the existing global one - # this is needed for the first run with split graphs on a branch where the global version did run previously - # NOTE: checking only for level_1 file here as this should already be sufficient - # NOTE2: also not chechking for RAM for same reason - - wmops_all_global="wmops/log_wmops_all.txt" - - ram_all_global="wmops/log_ram_all.txt" - - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then - - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") - - for suffix in "${suffixes[@]}"; do - - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt - - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt - - done - - fi - - fi - - - rm artifacts.zip - - rm -rf $public_dir - -.complexity-measurements-prepare-artifacts: - &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory - - public_dir="$CI_JOB_NAME-public" - - mkdir $public_dir - - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/ - # move logfiles for links - - mkdir $public_dir/logs - # first move logs - - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g") - - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done - # copy index page blueprint - - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html - # patch the format in the title - - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html - # do separately here to avoid overwrite complaints by mv - - mv -f ci/complexity_measurements/style.css ${public_dir}/ - -.complexity-template: - extends: - # still needed as long as no long MASA testvectors are there - - .test-job-linux-needs-testv-dir - tags: - - ivas-linux-fast - timeout: 3 hours 30 minutes - stage: test - before_script: - - !reference [.job-linux, before_script] - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - *complexity-measurements-setup - allow_failure: - exit_codes: - - 123 - artifacts: - name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA" - when: always - expire_in: 1 week - paths: - - $CI_JOB_NAME-public - -complexity-stereo-in-ext-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - script: - - in_format=stereo - - out_format=ext - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-ism-in-binaural-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 1 hour - script: - - in_format=ISM - - out_format=BINAURAL - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-ism-in-binaural_room_ir-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 2 hours - script: - - in_format=ISM - - out_format=BINAURAL_ROOM_IR - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-ism-in-ext-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 3 hours - script: - - in_format=ISM - - out_format=EXT - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-sba-hoa3-in-ext-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 4 hours - script: - - in_format=HOA3 - - out_format=EXT - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-sba-hoa3-in-binaural-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 5 hours - script: - - in_format=HOA3 - - out_format=BINAURAL - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-sba-hoa3-in-binaural_room_ir-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 6 hours - script: - - in_format=HOA3 - - out_format=BINAURAL_ROOM_IR - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-mc-in-ext-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 7 hours - script: - - in_format=MC - - out_format=EXT - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-mc-in-binaural-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 8 hours - script: - - in_format=MC - - out_format=BINAURAL - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-mc-in-binaural_room_ir-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 9 hours - script: - - in_format=MC - - out_format=BINAURAL_ROOM_IR - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-masa-in-ext-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 10 hours - script: - - in_format=MASA - - out_format=EXT - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-masa-in-binaural-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 11 hours - script: - - in_format=MASA - - out_format=BINAURAL - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-masa-in-hoa3-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 12 hours - script: - - in_format=MASA - - out_format=HOA3 - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-omasa-in-ext-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 13 hours - script: - - in_format=OMASA - - out_format=EXT - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-omasa-in-binaural-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 14 hours - script: - - in_format=OMASA - - out_format=BINAURAL - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-omasa-in-hoa3-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 15 hours - script: - - in_format=OMASA - - out_format=HOA3 - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-StereoDmxEVS-stereo-in-mono-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 16 hours - script: - - in_format=StereoDmxEVS - - out_format=mono - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-osba-in-ext-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 17 hours - script: - - in_format=OSBA - - out_format=EXT - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-osba-in-binaural-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 18 hours - script: - - in_format=OSBA - - out_format=BINAURAL - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -complexity-osba-in-binaural_room_ir-out: - extends: - - .complexity-template - rules: - - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 19 hours - script: - - in_format=OSBA - - out_format=BINAURAL_ROOM_IR - - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - - *complexity-measurements-prepare-artifacts - - exit $ret_val - -# --------------------------------------------------------------- -# Other jobs -# --------------------------------------------------------------- - -# job that sets up gitlab pages website -pages: - stage: deploy - extends: - - .job-linux - rules: - - if: $UPDATE_PAGES - script: - - python3 ci/setup_pages.py - artifacts: - paths: - - public - expire_in: 1 day - -# Pull state of a branch on 3GPP repo, push to a mirror repo. -pull-from-3gpp-forge: - stage: maintenance - rules: - - if: $MIRROR_ACCESS_TOKEN # Only run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) - script: - # Set up git LFS for mirroring (see: https://github.com/git-lfs/git-lfs/issues/1762) - - git lfs install --skip-smudge --local - - # Check out mirror branch - by default the runner checks out by commit hash, which results in detached head state - - git checkout $CI_COMMIT_BRANCH - - # Pull commits from upstream - - git remote add upstream https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec.git - - git pull --ff-only --tags upstream $MIRROR_SOURCE_BRANCH - - git lfs pull upstream - - # Push to mirror, include tags. Option `-o ci.skip` tells GitLab to skip CI for the pushed commits (assumed already tested upstream) - - git push --tags -o ci.skip "https://${GITLAB_USER_LOGIN}:${MIRROR_ACCESS_TOKEN}@${CI_REPOSITORY_URL#*@}" "HEAD:${CI_COMMIT_BRANCH}" diff --git a/.gitlab-ci/variables.yml b/.gitlab-ci/variables.yml deleted file mode 100644 index 9e60194a6c..0000000000 --- a/.gitlab-ci/variables.yml +++ /dev/null @@ -1,30 +0,0 @@ -variables: - EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test" - EVS_BE_WIN_TEST_DIR: "C:/Users/gitlab-runner/testvec" - PROCESSING_SCRIPTS_BIN_DIR: "/test-bin" - SANITIZER_TESTS: "CLANG1 CLANG2 CLANG3" - OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4" - OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3" - OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB" - OUT_FORMATS_ALL: "$OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT" - MANUAL_PIPELINE_TYPE: - description: "Type for the manual pipeline run. Use 'test-be-release' to run BE test against release codec." - value: 'default' - options: - - 'default' - - 'test-be-release' - - 'test-long-self-test' - - 'ivas-conformance' - - 'ivas-conformance-linux' - - 'check-clipping' - - 'test-branch-vs-input-passthrough' - - PYTEST_ARGS: "" - LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" - LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" - SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" - SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" - LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" - TEST_SUITE: "" - # note: currently overwrites default value from ci repo - TESTCASE_TIMEOUT_STV_SANITIZERS: 240 -- GitLab From 74edcf12ce1528641c41656c448d3b4616b7bf0f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 Sep 2025 16:08:35 +0200 Subject: [PATCH 2/4] remove now obsolete include --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5d9b506d38..0f1db44ff7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,6 @@ variables: IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF kiene/consolidate-float-ci-code include: - - local: .gitlab-ci/variables.yml - project: ivas-codec-pc/ivas-codec-ci ref: *IVAS_CODEC_CI_REF file: main-float.yml -- GitLab From d0ec3f76ba58c3641b8267fb3183302345a1372e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 25 Nov 2025 16:41:11 +0100 Subject: [PATCH 3/4] add comments in gitlab-ci yml --- .gitlab-ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f1db44ff7..2f0de28dca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,9 +2,16 @@ variables: # note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this # see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF kiene/consolidate-float-ci-code + # If you need to set some config variable only in a local branch, then add an overwrite here + # One example is DISABLE_HRTF - this will be set on a branch which is about to be merged and will be removed in a subsequent second MR + # this is more easily done directly here in the child repo + # these lines are suposed to stay commented out to serve as an example + # # set this to true to skip the external HRTF testcases in pytest calls + # DISABLE_HRTF: "true" + +# all CI code and config is included from https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec-ci include: - project: ivas-codec-pc/ivas-codec-ci ref: *IVAS_CODEC_CI_REF file: main-float.yml - -- GitLab From c4a19ef93331426118e9c3c84c1d744a843d66d9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 26 Nov 2025 09:18:27 +0100 Subject: [PATCH 4/4] change CI ref back --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f0de28dca..81e3d0a3d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ variables: # note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this # see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information - IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF kiene/consolidate-float-ci-code + IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF main # If you need to set some config variable only in a local branch, then add an overwrite here # One example is DISABLE_HRTF - this will be set on a branch which is about to be merged and will be removed in a subsequent second MR # this is more easily done directly here in the child repo -- GitLab